fix: Alert when TPP edge apply lag exceeds 30s

config/telemetry/alerts/gateways.yaml

@@ -50,6 +50,19 @@ spec:
         summary: "Gateway {{ $labels.resource_name }} has been degraded for over 60 seconds"
         description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."
 
+    - alert: TrafficProtectionPolicyApplyLagSLOViolation
+      expr: |
+        max by (cluster, name, exported_namespace) (nso_extension_tpp_cache_generation)
+          >
+        max by (cluster, name, exported_namespace) (nso_extension_tpp_applied_generation)
+      for: 30s
+      labels:
+        severity: critical
+        slo_violation: "true"
+      annotations:
+        summary: "TrafficProtectionPolicy {{ $labels.name }} apply lag on {{ $labels.cluster }} exceeds 30s"
+        description: "Edge {{ $labels.cluster }} has TrafficProtectionPolicy {{ $labels.exported_namespace }}/{{ $labels.name }} cached at a generation ahead of the last applied generation for over 30 seconds. The edge may be serving stale WAF config for this policy."
+
   # TLS certificate health alerts fire on nso_* metrics emitted directly by the
   # NSO operator and extension server. They are available in the same Prometheus
   # that loads this rule, alongside the envoy_gateway_* metrics above.

test/prometheus-rules/gateways/nso-slo-rules.yaml

@@ -32,4 +32,16 @@ groups:
       slo_violation: "true"
     annotations:
       summary: "Gateway {{ $labels.resource_name }} has been degraded for over 60 seconds"
-      description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."
+      description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."
+  - alert: TrafficProtectionPolicyApplyLagSLOViolation
+    expr: |
+      max by (cluster, name, exported_namespace) (nso_extension_tpp_cache_generation)
+        >
+      max by (cluster, name, exported_namespace) (nso_extension_tpp_applied_generation)
+    for: 30s
+    labels:
+      severity: critical
+      slo_violation: "true"
+    annotations:
+      summary: "TrafficProtectionPolicy {{ $labels.name }} apply lag on {{ $labels.cluster }} exceeds 30s"
+      description: "Edge {{ $labels.cluster }} has TrafficProtectionPolicy {{ $labels.exported_namespace }}/{{ $labels.name }} cached at a generation ahead of the last applied generation for over 30 seconds. The edge may be serving stale WAF config for this policy."

test/prometheus-rules/gateways/nso-slo-tests.yaml

@@ -98,4 +98,52 @@ tests:
     alert_rule_test:
       - eval_time: 1m
         alertname: GatewayNotReadySLOViolation
-        exp_alerts: []
+        exp_alerts: []
+  # TrafficProtectionPolicyApplyLagSLOViolation — cache ahead of applied for >30s
+  - interval: 30s
+    input_series:
+      - series: 'nso_extension_tpp_cache_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}'
+        values: '5+0x4'
+      - series: 'nso_extension_tpp_cache_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-1"}'
+        values: '5+0x4'
+      - series: 'nso_extension_tpp_applied_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}'
+        values: '4+0x4'
+      - series: 'nso_extension_tpp_applied_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-1"}'
+        values: '4+0x4'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: TrafficProtectionPolicyApplyLagSLOViolation
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              slo_violation: "true"
+              cluster: edge-a
+              name: waf-pol
+              exported_namespace: ns-tenant
+            exp_annotations:
+              summary: "TrafficProtectionPolicy waf-pol apply lag on edge-a exceeds 30s"
+              description: "Edge edge-a has TrafficProtectionPolicy ns-tenant/waf-pol cached at a generation ahead of the last applied generation for over 30 seconds. The edge may be serving stale WAF config for this policy."
+
+  # TrafficProtectionPolicyApplyLagSLOViolation — in sync (should NOT alert)
+  - interval: 30s
+    input_series:
+      - series: 'nso_extension_tpp_cache_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}'
+        values: '5+0x4'
+      - series: 'nso_extension_tpp_applied_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}'
+        values: '5+0x4'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: TrafficProtectionPolicyApplyLagSLOViolation
+        exp_alerts: []
+
+  # TrafficProtectionPolicyApplyLagSLOViolation — brief skew clears before 30s (should NOT alert)
+  - interval: 15s
+    input_series:
+      - series: 'nso_extension_tpp_cache_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}'
+        values: '5 5 5 5'
+      - series: 'nso_extension_tpp_applied_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}'
+        values: '4 4 5 5'
+    alert_rule_test:
+      - eval_time: 45s
+        alertname: TrafficProtectionPolicyApplyLagSLOViolation
+        exp_alerts: []