diff --git a/config/telemetry/alerts/gateways.yaml b/config/telemetry/alerts/gateways.yaml index beff8344..203ee344 100644 --- a/config/telemetry/alerts/gateways.yaml +++ b/config/telemetry/alerts/gateways.yaml @@ -50,6 +50,19 @@ spec: summary: "Gateway {{ $labels.resource_name }} has been degraded for over 60 seconds" description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold." + - alert: TrafficProtectionPolicyApplyLagSLOViolation + expr: | + max by (cluster, name, exported_namespace) (nso_extension_tpp_cache_generation) + > + max by (cluster, name, exported_namespace) (nso_extension_tpp_applied_generation) + for: 30s + labels: + severity: critical + slo_violation: "true" + annotations: + summary: "TrafficProtectionPolicy {{ $labels.name }} apply lag on {{ $labels.cluster }} exceeds 30s" + description: "Edge {{ $labels.cluster }} has TrafficProtectionPolicy {{ $labels.exported_namespace }}/{{ $labels.name }} cached at a generation ahead of the last applied generation for over 30 seconds. The edge may be serving stale WAF config for this policy." + # TLS certificate health alerts fire on nso_* metrics emitted directly by the # NSO operator and extension server. They are available in the same Prometheus # that loads this rule, alongside the envoy_gateway_* metrics above. diff --git a/test/prometheus-rules/gateways/nso-slo-rules.yaml b/test/prometheus-rules/gateways/nso-slo-rules.yaml index 7aabf34a..a8ba4286 100644 --- a/test/prometheus-rules/gateways/nso-slo-rules.yaml +++ b/test/prometheus-rules/gateways/nso-slo-rules.yaml @@ -32,4 +32,16 @@ groups: slo_violation: "true" annotations: summary: "Gateway {{ $labels.resource_name }} has been degraded for over 60 seconds" - description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold." \ No newline at end of file + description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold." + - alert: TrafficProtectionPolicyApplyLagSLOViolation + expr: | + max by (cluster, name, exported_namespace) (nso_extension_tpp_cache_generation) + > + max by (cluster, name, exported_namespace) (nso_extension_tpp_applied_generation) + for: 30s + labels: + severity: critical + slo_violation: "true" + annotations: + summary: "TrafficProtectionPolicy {{ $labels.name }} apply lag on {{ $labels.cluster }} exceeds 30s" + description: "Edge {{ $labels.cluster }} has TrafficProtectionPolicy {{ $labels.exported_namespace }}/{{ $labels.name }} cached at a generation ahead of the last applied generation for over 30 seconds. The edge may be serving stale WAF config for this policy." diff --git a/test/prometheus-rules/gateways/nso-slo-tests.yaml b/test/prometheus-rules/gateways/nso-slo-tests.yaml index 6376d6e8..f4d6d621 100644 --- a/test/prometheus-rules/gateways/nso-slo-tests.yaml +++ b/test/prometheus-rules/gateways/nso-slo-tests.yaml @@ -98,4 +98,52 @@ tests: alert_rule_test: - eval_time: 1m alertname: GatewayNotReadySLOViolation - exp_alerts: [] \ No newline at end of file + exp_alerts: [] + # TrafficProtectionPolicyApplyLagSLOViolation — cache ahead of applied for >30s + - interval: 30s + input_series: + - series: 'nso_extension_tpp_cache_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}' + values: '5+0x4' + - series: 'nso_extension_tpp_cache_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-1"}' + values: '5+0x4' + - series: 'nso_extension_tpp_applied_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}' + values: '4+0x4' + - series: 'nso_extension_tpp_applied_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-1"}' + values: '4+0x4' + alert_rule_test: + - eval_time: 1m + alertname: TrafficProtectionPolicyApplyLagSLOViolation + exp_alerts: + - exp_labels: + severity: critical + slo_violation: "true" + cluster: edge-a + name: waf-pol + exported_namespace: ns-tenant + exp_annotations: + summary: "TrafficProtectionPolicy waf-pol apply lag on edge-a exceeds 30s" + description: "Edge edge-a has TrafficProtectionPolicy ns-tenant/waf-pol cached at a generation ahead of the last applied generation for over 30 seconds. The edge may be serving stale WAF config for this policy." + + # TrafficProtectionPolicyApplyLagSLOViolation — in sync (should NOT alert) + - interval: 30s + input_series: + - series: 'nso_extension_tpp_cache_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}' + values: '5+0x4' + - series: 'nso_extension_tpp_applied_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}' + values: '5+0x4' + alert_rule_test: + - eval_time: 1m + alertname: TrafficProtectionPolicyApplyLagSLOViolation + exp_alerts: [] + + # TrafficProtectionPolicyApplyLagSLOViolation — brief skew clears before 30s (should NOT alert) + - interval: 15s + input_series: + - series: 'nso_extension_tpp_cache_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}' + values: '5 5 5 5' + - series: 'nso_extension_tpp_applied_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}' + values: '4 4 5 5' + alert_rule_test: + - eval_time: 45s + alertname: TrafficProtectionPolicyApplyLagSLOViolation + exp_alerts: []