[Observability] Add alarm on missing clustermgtd heartbeat.

Author: gmarciani

CHANGELOG.md

@@ -9,6 +9,7 @@ CHANGELOG
 - Upgrade jmespath to ~=1.0 (from ~=0.10).
 - Upgrade tabulate to <=0.9.0 (from <=0.8.10).
 - Add permission `cloudwatch:PutMetricData` to the head node policy so that clustermgtd is able to emit metrics.
+- Add alarm on missing clustermgtd heartbeat.
 
 **BUG FIXES**
 - Add validation to block updates that change tag order. Blocking such change prevents update failures.

cli/src/pcluster/templates/cluster_stack.py

@@ -64,6 +64,9 @@
     CW_ALARM_PERIOD_DEFAULT,
     CW_LOG_GROUP_NAME_PREFIX,
     CW_LOGS_CFN_PARAM_NAME,
+    CW_METRICS_CLUSTERMGTD_HEARTBEAT,
+    CW_METRICS_DIMENSION_CLUSTER_NAME,
+    CW_METRICS_NAMESPACE,
     DEFAULT_EPHEMERAL_DIR,
     EFS_PORT,
     FSX_PORTS,
@@ -365,37 +368,77 @@ def _add_head_node_alarms(self):
         self.head_node_alarms = []
 
         metrics_for_alarms = {
-            "Health": self._cw_metric_head_node("AWS/EC2", "StatusCheckFailed"),
-            "Cpu": self._cw_metric_head_node("AWS/EC2", "CPUUtilization"),
-            "Mem": self._cw_metric_head_node("CWAgent", "mem_used_percent"),
-            "Disk": self._cw_metric_head_node("CWAgent", "disk_used_percent", extra_dimensions={"path": "/"}),
+            "Health": {
+                "metric": self._cw_metric_head_node("AWS/EC2", "StatusCheckFailed"),
+                "evaluation_periods": CW_ALARM_EVALUATION_PERIODS_DEFAULT,
+                "datapoints_to_alarm": CW_ALARM_DATAPOINTS_TO_ALARM_DEFAULT,
+                "comparison_operator": cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
+                "threshold": 0,
+                "treat_missing_data": cloudwatch.TreatMissingData.MISSING,
+            },
+            "Cpu": {
+                "metric": self._cw_metric_head_node("AWS/EC2", "CPUUtilization"),
+                "evaluation_periods": CW_ALARM_EVALUATION_PERIODS_DEFAULT,
+                "datapoints_to_alarm": CW_ALARM_DATAPOINTS_TO_ALARM_DEFAULT,
+                "comparison_operator": cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
+                "threshold": CW_ALARM_PERCENT_THRESHOLD_DEFAULT,
+                "treat_missing_data": cloudwatch.TreatMissingData.MISSING,
+            },
+            "Mem": {
+                "metric": self._cw_metric_head_node("CWAgent", "mem_used_percent"),
+                "evaluation_periods": CW_ALARM_EVALUATION_PERIODS_DEFAULT,
+                "datapoints_to_alarm": CW_ALARM_DATAPOINTS_TO_ALARM_DEFAULT,
+                "comparison_operator": cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
+                "threshold": CW_ALARM_PERCENT_THRESHOLD_DEFAULT,
+                "treat_missing_data": cloudwatch.TreatMissingData.MISSING,
+            },
+            "Disk": {
+                "metric": self._cw_metric_head_node("CWAgent", "disk_used_percent", extra_dimensions={"path": "/"}),
+                "evaluation_periods": CW_ALARM_EVALUATION_PERIODS_DEFAULT,
+                "datapoints_to_alarm": CW_ALARM_DATAPOINTS_TO_ALARM_DEFAULT,
+                "comparison_operator": cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
+                "threshold": CW_ALARM_PERCENT_THRESHOLD_DEFAULT,
+                "treat_missing_data": cloudwatch.TreatMissingData.MISSING,
+            },
+            "ClustermgtdHeartbeat": {
+                "metric": self._cw_metric_head_node(
+                    CW_METRICS_NAMESPACE,
+                    CW_METRICS_CLUSTERMGTD_HEARTBEAT,
+                    extra_dimensions={CW_METRICS_DIMENSION_CLUSTER_NAME: self.config.cluster_name},
+                ),
+                "evaluation_periods": 10,
+                "datapoints_to_alarm": 10,
+                "comparison_operator": cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
+                "threshold": 1,
+                "treat_missing_data": cloudwatch.TreatMissingData.BREACHING,
+            },
         }
 
-        for metric_key, metric in metrics_for_alarms.items():
+        for metric_key, alarm_details in metrics_for_alarms.items():
             alarm_id = f"HeadNode{metric_key}Alarm"
             alarm_name = f"{self.stack.stack_name}-HeadNode-{metric_key}"
-            threshold = 0 if metric_key == "Health" else CW_ALARM_PERCENT_THRESHOLD_DEFAULT
-            self.head_node_alarms.append(
-                cloudwatch.Alarm(
-                    scope=self.stack,
-                    id=alarm_id,
-                    alarm_name=alarm_name,
-                    metric=metric,
-                    evaluation_periods=CW_ALARM_EVALUATION_PERIODS_DEFAULT,
-                    threshold=threshold,
-                    comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
-                    datapoints_to_alarm=CW_ALARM_DATAPOINTS_TO_ALARM_DEFAULT,
-                )
-            )
-
-        self.head_node_alarms.append(
-            cloudwatch.CompositeAlarm(
+            alarm = cloudwatch.Alarm(
                 scope=self.stack,
-                id="HeadNodeAlarm",
-                composite_alarm_name=f"{self.stack.stack_name}-HeadNode",
-                alarm_rule=cloudwatch.AlarmRule.any_of(*self.head_node_alarms),
+                id=alarm_id,
+                alarm_name=alarm_name,
+                metric=alarm_details["metric"],
+                evaluation_periods=alarm_details["evaluation_periods"],
+                threshold=alarm_details["threshold"],
+                comparison_operator=alarm_details["comparison_operator"],
+                datapoints_to_alarm=alarm_details["datapoints_to_alarm"],
+                treat_missing_data=alarm_details["treat_missing_data"],
             )
+            alarm.node.add_dependency(self.wait_condition)
+            self.head_node_alarms.append(alarm)
+
+        composite_alarm = cloudwatch.CompositeAlarm(
+            scope=self.stack,
+            id="HeadNodeAlarm",
+            composite_alarm_name=f"{self.stack.stack_name}-HeadNode",
+            alarm_rule=cloudwatch.AlarmRule.any_of(*self.head_node_alarms),
         )
+        composite_alarm.node.add_dependency(self.wait_condition)
+        self.head_node_alarms.append(composite_alarm)
 
     def _add_iam_resources(self):
         head_node_iam_resources = HeadNodeIamResources(

cli/src/pcluster/templates/cw_dashboard_builder.py

@@ -19,7 +19,12 @@
 
 from pcluster.config.cluster_config import BaseClusterConfig, ExistingFileCache, SharedFsxLustre
 from pcluster.config.common import SharedStorageType
-from pcluster.constants import Feature
+from pcluster.constants import (
+    CW_METRICS_CLUSTERMGTD_HEARTBEAT,
+    CW_METRICS_DIMENSION_CLUSTER_NAME,
+    CW_METRICS_NAMESPACE,
+    Feature,
+)
 from pcluster.utils import is_feature_supported
 
 MAX_WIDTH = 24
@@ -567,9 +572,19 @@ def _add_head_node_instance_metrics_graphs(self):
             new_pcluster_metric(title="Memory Used Percent", metrics=["mem_used_percent"], namespace="CWAgent"),
         ]
 
+        # Custom Metrics
+        pcluster_metrics = [
+            new_pcluster_metric(
+                title="Daemons Heartbeats",
+                metrics=[CW_METRICS_CLUSTERMGTD_HEARTBEAT],
+                namespace=CW_METRICS_NAMESPACE,
+                additional_dimensions={CW_METRICS_DIMENSION_CLUSTER_NAME: self.config.cluster_name},
+            ),
+        ]
+
         # Create graphs for EC2 metrics and CW Agent metrics and update coordinates
         widgets_list = []
-        for metrics_param in ec2_metrics + cwagent_metrics:
+        for metrics_param in ec2_metrics + cwagent_metrics + pcluster_metrics:
             metrics_list = self._generate_metrics_list(metrics_param)
             graph_widget = self._generate_graph_widget(metrics_param.title, metrics_list)
             widgets_list.append(graph_widget)

cli/tests/pcluster/templates/test_cluster_stack.py

@@ -281,7 +281,7 @@ def test_add_alarms(mocker, config_file_name):
             "comparison_operator": "GreaterThanThreshold",
             "evaluation_periods": 1,
             "datapoints_to_alarm": 1,
-            "treat_missing_data": None,
+            "treat_missing_data": "missing",
         },
         "Cpu": {
             "name": "clustername-HeadNode-Cpu",
@@ -291,7 +291,7 @@ def test_add_alarms(mocker, config_file_name):
             "comparison_operator": "GreaterThanThreshold",
             "evaluation_periods": 1,
             "datapoints_to_alarm": 1,
-            "treat_missing_data": None,
+            "treat_missing_data": "missing",
         },
         "Mem": {
             "name": "clustername-HeadNode-Mem",
@@ -301,7 +301,7 @@ def test_add_alarms(mocker, config_file_name):
             "comparison_operator": "GreaterThanThreshold",
             "evaluation_periods": 1,
             "datapoints_to_alarm": 1,
-            "treat_missing_data": None,
+            "treat_missing_data": "missing",
         },
         "Disk": {
             "name": "clustername-HeadNode-Disk",
@@ -311,7 +311,17 @@ def test_add_alarms(mocker, config_file_name):
             "comparison_operator": "GreaterThanThreshold",
             "evaluation_periods": 1,
             "datapoints_to_alarm": 1,
-            "treat_missing_data": None,
+            "treat_missing_data": "missing",
+        },
+        "Clustermgtd-Heartbeat": {
+            "name": "clustername-HeadNode-ClustermgtdHeartbeat",
+            "metric_name": "ClustermgtdHeartbeat",
+            "namespace": "ParallelCluster",
+            "threshold": 1,
+            "comparison_operator": "LessThanThreshold",
+            "evaluation_periods": 10,
+            "datapoints_to_alarm": 10,
+            "treat_missing_data": "breaching",
         },
     }
 

cli/tests/pcluster/templates/test_cw_dashboard_builder.py

@@ -101,6 +101,9 @@ def _verify_alarms(output_yaml, alarms_enabled):
         assert_that(output_yaml).contains("HeadNodeDiskAlarm")
         assert_that(output_yaml).contains("disk_used_percent")
 
+        assert_that(output_yaml).contains("HeadNodeClustermgtdAlarm")
+        assert_that(output_yaml).contains("ClustermgtdHeartbeat")
+
     else:
         assert_that(output_yaml).does_not_contain("Cluster Alarms")
         assert_that(output_yaml).does_not_contain("AWS::CloudWatch::Alarm")
@@ -139,6 +142,8 @@ def _verify_head_node_instance_metrics_graphs(output_yaml):
     assert_that(output_yaml).contains("Disk Read/Write Ops")
     assert_that(output_yaml).contains("Disk Used Percent")
     assert_that(output_yaml).contains("Memory Used Percent")
+    assert_that(output_yaml).contains("Daemons Heartbeats")
+    assert_that(output_yaml).contains("ClustermgtdHeartbeat")
 
 
 def _verify_ec2_metrics_conditions(cluster_config, output_yaml):