From 950b6150722c3dbe4e2ef0b088e93f610c7338ba Mon Sep 17 00:00:00 2001
From: Matt Jenkinson <75292329+mattdjenkinson@users.noreply.github.com>
Date: Wed, 5 Aug 2026 17:00:20 +0100
Subject: [PATCH] fix: alert when TPP edge apply lag exceeds 30s
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Measured upstream→edge TPP generation convergence is ~1.5s in
staging and ~3.3s in prod. Adopt a 30s SLO (~10x headroom) and alert
when cache generation stays ahead of applied generation on an edge.

Key features:
- TrafficProtectionPolicyApplyLagSLOViolation on nso_extension_tpp_*
  generation gauges, aggregated per cluster/policy
- promtool coverage for lag, in-sync, and brief-skew cases

Related to #266.
---
 config/telemetry/alerts/gateways.yaml         | 13 +++++
 .../gateways/nso-slo-rules.yaml               | 14 +++++-
 .../gateways/nso-slo-tests.yaml               | 50 ++++++++++++++++++-
 3 files changed, 75 insertions(+), 2 deletions(-)

diff --git a/config/telemetry/alerts/gateways.yaml b/config/telemetry/alerts/gateways.yaml
index beff8344..203ee344 100644
--- a/config/telemetry/alerts/gateways.yaml
+++ b/config/telemetry/alerts/gateways.yaml
@@ -50,6 +50,19 @@ spec:
         summary: "Gateway {{ $labels.resource_name }} has been degraded for over 60 seconds"
         description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."
 
+    - alert: TrafficProtectionPolicyApplyLagSLOViolation
+      expr: |
+        max by (cluster, name, exported_namespace) (nso_extension_tpp_cache_generation)
+          >
+        max by (cluster, name, exported_namespace) (nso_extension_tpp_applied_generation)
+      for: 30s
+      labels:
+        severity: critical
+        slo_violation: "true"
+      annotations:
+        summary: "TrafficProtectionPolicy {{ $labels.name }} apply lag on {{ $labels.cluster }} exceeds 30s"
+        description: "Edge {{ $labels.cluster }} has TrafficProtectionPolicy {{ $labels.exported_namespace }}/{{ $labels.name }} cached at a generation ahead of the last applied generation for over 30 seconds. The edge may be serving stale WAF config for this policy."
+
   # TLS certificate health alerts fire on nso_* metrics emitted directly by the
   # NSO operator and extension server. They are available in the same Prometheus
   # that loads this rule, alongside the envoy_gateway_* metrics above.
diff --git a/test/prometheus-rules/gateways/nso-slo-rules.yaml b/test/prometheus-rules/gateways/nso-slo-rules.yaml
index 7aabf34a..a8ba4286 100644
--- a/test/prometheus-rules/gateways/nso-slo-rules.yaml
+++ b/test/prometheus-rules/gateways/nso-slo-rules.yaml
@@ -32,4 +32,16 @@ groups:
       slo_violation: "true"
     annotations:
       summary: "Gateway {{ $labels.resource_name }} has been degraded for over 60 seconds"
-      description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."
\ No newline at end of file
+      description: "Gateway {{ $labels.resource_name }} in namespace {{ $labels.resource_namespace }} has been in a degraded state for over 60 seconds without recovering, which exceeds the 60-second SLO threshold."
+  - alert: TrafficProtectionPolicyApplyLagSLOViolation
+    expr: |
+      max by (cluster, name, exported_namespace) (nso_extension_tpp_cache_generation)
+        >
+      max by (cluster, name, exported_namespace) (nso_extension_tpp_applied_generation)
+    for: 30s
+    labels:
+      severity: critical
+      slo_violation: "true"
+    annotations:
+      summary: "TrafficProtectionPolicy {{ $labels.name }} apply lag on {{ $labels.cluster }} exceeds 30s"
+      description: "Edge {{ $labels.cluster }} has TrafficProtectionPolicy {{ $labels.exported_namespace }}/{{ $labels.name }} cached at a generation ahead of the last applied generation for over 30 seconds. The edge may be serving stale WAF config for this policy."
diff --git a/test/prometheus-rules/gateways/nso-slo-tests.yaml b/test/prometheus-rules/gateways/nso-slo-tests.yaml
index 6376d6e8..f4d6d621 100644
--- a/test/prometheus-rules/gateways/nso-slo-tests.yaml
+++ b/test/prometheus-rules/gateways/nso-slo-tests.yaml
@@ -98,4 +98,52 @@ tests:
     alert_rule_test:
       - eval_time: 1m
         alertname: GatewayNotReadySLOViolation
-        exp_alerts: []
\ No newline at end of file
+        exp_alerts: []
+  # TrafficProtectionPolicyApplyLagSLOViolation — cache ahead of applied for >30s
+  - interval: 30s
+    input_series:
+      - series: 'nso_extension_tpp_cache_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}'
+        values: '5+0x4'
+      - series: 'nso_extension_tpp_cache_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-1"}'
+        values: '5+0x4'
+      - series: 'nso_extension_tpp_applied_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}'
+        values: '4+0x4'
+      - series: 'nso_extension_tpp_applied_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-1"}'
+        values: '4+0x4'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: TrafficProtectionPolicyApplyLagSLOViolation
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              slo_violation: "true"
+              cluster: edge-a
+              name: waf-pol
+              exported_namespace: ns-tenant
+            exp_annotations:
+              summary: "TrafficProtectionPolicy waf-pol apply lag on edge-a exceeds 30s"
+              description: "Edge edge-a has TrafficProtectionPolicy ns-tenant/waf-pol cached at a generation ahead of the last applied generation for over 30 seconds. The edge may be serving stale WAF config for this policy."
+
+  # TrafficProtectionPolicyApplyLagSLOViolation — in sync (should NOT alert)
+  - interval: 30s
+    input_series:
+      - series: 'nso_extension_tpp_cache_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}'
+        values: '5+0x4'
+      - series: 'nso_extension_tpp_applied_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}'
+        values: '5+0x4'
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: TrafficProtectionPolicyApplyLagSLOViolation
+        exp_alerts: []
+
+  # TrafficProtectionPolicyApplyLagSLOViolation — brief skew clears before 30s (should NOT alert)
+  - interval: 15s
+    input_series:
+      - series: 'nso_extension_tpp_cache_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}'
+        values: '5 5 5 5'
+      - series: 'nso_extension_tpp_applied_generation{cluster="edge-a", name="waf-pol", exported_namespace="ns-tenant", pod="ext-0"}'
+        values: '4 4 5 5'
+    alert_rule_test:
+      - eval_time: 45s
+        alertname: TrafficProtectionPolicyApplyLagSLOViolation
+        exp_alerts: []