feat(infra): porting prometheus alert

test-infra/docker-compose.yml

@@ -20,6 +20,7 @@ services:
     image: prom/prometheus:latest
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./rules.yml:/etc/prometheus/rules.yml
       - prometheus-data:/prometheus
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'

test-infra/prometheus.yml

@@ -2,6 +2,10 @@ global:
   scrape_interval: 15s
   evaluation_interval: 15s
 
+# Cluster-health alert rules (the smoke oracle). Any firing alert = unhealthy run.
+rule_files:
+  - /etc/prometheus/rules.yml
+
 scrape_configs:
   - job_name: 'local-exporter'
     static_configs:

test-infra/rules.yml

@@ -0,0 +1,44 @@
+groups:
+- name: pluto
+  rules:
+  - alert: Pluto Down
+    expr: up == 0
+    for: 15s
+    annotations:
+      description: "Pluto {{ $labels.job }} is down"
+
+  - alert: Error Log Rate
+    expr: app_log_error_total > 0
+    for: 15s
+    annotations:
+      description: "Pluto {{ $labels.job }} has a high error rate"
+
+  - alert: Warn Log Rate
+    expr: increase(app_log_warn_total[30s]) > 2
+    for: 15s
+    annotations:
+      description: "Pluto {{ $labels.job }} has a high warning rate"
+
+  - alert: Validator API Error Rate
+    expr: increase(core_validatorapi_request_error_total{endpoint!="proxy"}[30s]) > 1
+    for: 15s
+    annotations:
+      description: "Pluto {{ $labels.job }} validator API a high error rate"
+
+  - alert: Proxy API Error Rate
+    expr: increase(core_validatorapi_request_error_total{endpoint="proxy"}[30s]) > 5
+    for: 15s
+    annotations:
+      description: "Pluto {{ $labels.job }} proxy API a high error rate"
+
+  - alert: Broadcast Duty Rate
+    expr: increase(core_bcast_broadcast_total[30s]) < 0.5
+    for: 15s
+    annotations:
+      description: "Pluto {{ $labels.job }} is not broadcasting enough duties"
+
+  - alert: Outstanding Duty Rate
+    expr: core_bcast_broadcast_total - core_scheduler_duty_total > 50
+    for: 15s
+    annotations:
+      description: "Pluto {{ $labels.job }} has too many outstanding duties"