VED-1253: CloudWatch Dashboard Additions

infrastructure/account/fhir_api_perf_alerts_slack_chatbot.tf

@@ -0,0 +1,24 @@
+resource "aws_chatbot_slack_channel_configuration" "fhir_api_perf_alerts" {
+  configuration_name = "${var.environment}-fhir-api-perf-alerts-slack-channel-config"
+  iam_role_arn       = aws_iam_role.fhir_api_perf_alerts_chatbot.arn
+  slack_channel_id   = var.environment == "prod" ? "C0B11MJPQ6A" : "C0B1GKZ5S4R"
+  slack_team_id      = "TJ00QR03U"
+  sns_topic_arns     = [aws_sns_topic.fhir_api_perf_alerts.arn]
+}
+
+resource "aws_iam_role" "fhir_api_perf_alerts_chatbot" {
+  name = "${var.environment}-fhir-api-perf-alerts-chatbot-channel-role"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = "AssumeChatbotRole"
+        Principal = {
+          Service = "chatbot.amazonaws.com"
+        }
+      },
+    ]
+  })
+}

infrastructure/account/fhir_api_perf_alerts_sns_topic.tf

@@ -0,0 +1,22 @@
+resource "aws_sns_topic" "fhir_api_perf_alerts" {
+  name              = "${var.environment}-fhir-api-perf-alerts"
+  kms_master_key_id = aws_kms_key.error_alerts_sns_encryption_key.arn
+}
+
+resource "aws_sns_topic_policy" "fhir_api_perf_alerts_topic_policy" {
+  arn = aws_sns_topic.fhir_api_perf_alerts.arn
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Sid    = "AllowCloudWatchToPublish",
+        Effect = "Allow",
+        Principal = {
+          Service = "cloudwatch.amazonaws.com"
+        },
+        Action   = "SNS:Publish",
+        Resource = aws_sns_topic.fhir_api_perf_alerts.arn
+      }
+    ]
+  })
+}

infrastructure/account/kms.tf

@@ -225,3 +225,8 @@ resource "aws_kms_alias" "fhir_api_errors_sns_encryption_key" {
   name          = "alias/${var.environment}-fhir-api-errors-imms-sns-encryption"
   target_key_id = aws_kms_key.error_alerts_sns_encryption_key.key_id
 }
+
+resource "aws_kms_alias" "fhir_api_perf_alerts_sns_encryption_key" {
+  name          = "alias/${var.environment}-fhir-api-perf-alerts-imms-sns-encryption"
+  target_key_id = aws_kms_key.error_alerts_sns_encryption_key.key_id
+}

infrastructure/account/shield_protection.tf

@@ -34,13 +34,18 @@ locals {
   }
 }
 
+# Topic to publish alerts to when alarm is triggered
+data "aws_sns_topic" "fhir_api_perf_alerts" {
+  name = "${var.environment}-fhir-api-perf-alerts"
+}
 
 # Create Metric Alarms for each of those resources
 resource "aws_cloudwatch_metric_alarm" "ddos_protection_regional" {
   for_each = local.regional_shield_arn
 
   alarm_name        = "imms-${var.environment}-shield_ddos_${each.key}"
   alarm_description = "Alarm when Shield detects DDoS on ${each.key}"
+  alarm_actions     = [data.aws_sns_topic.fhir_api_perf_alerts.arn]
 
   namespace           = "AWS/DDoSProtection"
   metric_name         = "DDoSDetected"

infrastructure/instance/ack_lambda.tf

@@ -154,3 +154,33 @@ resource "aws_lambda_event_source_mapping" "sqs_to_lambda" {
   batch_size       = 1 # VED-734 - forwarder lambda already sends a list of up to 100 messages in the body
   enabled          = true
 }
+
+resource "aws_cloudwatch_log_metric_filter" "ack_lambda_error_logs" {
+  count = var.error_alarm_notifications_enabled ? 1 : 0
+
+  name           = "${local.short_prefix}-AckLambdaErrorLogsFilter"
+  pattern        = "%\\[ERROR\\]%"
+  log_group_name = aws_cloudwatch_log_group.ack_lambda_log_group.name
+
+  metric_transformation {
+    name      = "${local.short_prefix}-AckLambdaErrorLogs"
+    namespace = "${local.short_prefix}-AckLambda"
+    value     = "1"
+  }
+}
+
+resource "aws_cloudwatch_metric_alarm" "ack_lambda_error_alarm" {
+  count = var.error_alarm_notifications_enabled ? 1 : 0
+
+  alarm_name          = "${local.ack_lambda_name}-error"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 1
+  metric_name         = aws_cloudwatch_log_metric_filter.ack_lambda_error_logs[count.index].metric_transformation[0].name
+  namespace           = aws_cloudwatch_log_metric_filter.ack_lambda_error_logs[count.index].metric_transformation[0].namespace
+  period              = 120
+  statistic           = "Sum"
+  threshold           = 1
+  alarm_description   = "This sets off an alarm for any error logs found in the ack Lambda function"
+  alarm_actions       = [data.aws_sns_topic.imms_system_alert_errors.arn]
+  treat_missing_data  = "notBreaching"
+}

infrastructure/instance/forwarder_lambda.tf

@@ -178,3 +178,33 @@ resource "aws_cloudwatch_log_group" "forwarding_lambda_log_group" {
   name              = "/aws/lambda/${local.forwarder_lambda_name}"
   retention_in_days = 30
 }
+
+resource "aws_cloudwatch_log_metric_filter" "forwarding_lambda_error_logs" {
+  count = var.error_alarm_notifications_enabled ? 1 : 0
+
+  name           = "${local.short_prefix}-ForwardingLambdaErrorLogsFilter"
+  pattern        = "%\\[ERROR\\]%"
+  log_group_name = aws_cloudwatch_log_group.forwarding_lambda_log_group.name
+
+  metric_transformation {
+    name      = "${local.short_prefix}-ForwardingLambdaErrorLogs"
+    namespace = "${local.short_prefix}-ForwardingLambda"
+    value     = "1"
+  }
+}
+
+resource "aws_cloudwatch_metric_alarm" "forwarding_lambda_error_alarm" {
+  count = var.error_alarm_notifications_enabled ? 1 : 0
+
+  alarm_name          = "${local.forwarder_lambda_name}-error"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 1
+  metric_name         = aws_cloudwatch_log_metric_filter.forwarding_lambda_error_logs[count.index].metric_transformation[0].name
+  namespace           = aws_cloudwatch_log_metric_filter.forwarding_lambda_error_logs[count.index].metric_transformation[0].namespace
+  period              = 120
+  statistic           = "Sum"
+  threshold           = 1
+  alarm_description   = "This sets off an alarm for any error logs found in the forwarding Lambda function"
+  alarm_actions       = [data.aws_sns_topic.imms_system_alert_errors.arn]
+  treat_missing_data  = "notBreaching"
+}

infrastructure/instance/modules/lambda/lambda.tf

@@ -24,6 +24,10 @@ module "lambda_function_container_image" {
   image_config_command  = ["${var.function_name}_handler.${var.function_name}_handler"]
 }
 
+data "aws_sns_topic" "fhir_api_perf_alerts" {
+  name = "${var.environment}-fhir-api-perf-alerts"
+}
+
 resource "aws_cloudwatch_metric_alarm" "memory_alarm" {
   alarm_name                = "${var.short_prefix}_${var.function_name} memory alarm"
   comparison_operator       = "GreaterThanOrEqualToThreshold"
@@ -34,6 +38,7 @@ resource "aws_cloudwatch_metric_alarm" "memory_alarm" {
   statistic                 = "Maximum"
   threshold                 = 256
   alarm_description         = "This metric monitors Lambda memory usage"
+  alarm_actions             = [data.aws_sns_topic.fhir_api_perf_alerts.arn]
   insufficient_data_actions = []
 
 }