Add --use-state-labels to override the exit state for firing alerts with labels

If this flag is set the plugin looks for warning/critical/ok in the label 'icingaState'

Author: martialblog

README.md

@@ -168,6 +168,9 @@ Flags:
                                     If no name is given, all alerts will be evaluated
   -T, --no-alerts-state string      State to assign when no alerts are found (0, 1, 2, 3, OK, WARNING, CRITICAL, UNKNOWN). If not set this defaults to OK (default "OK")
   -P, --problems                    Display only alerts which status is not inactive/OK. Note that in combination with the --name flag this might result in no alerts being displayed
+  -K, --state-labels-key string     Label key for the --use-state-labels. Can be used to override the default label key 'icingaState' (default "icingaState")
+  -S, --use-state-labels            Use the AlertRule label 'icingaState' to override the exit state for firing alerts.
+                                    If this flag is set the plugin looks for warning/critical/ok in the label 'icingaState'
 ```
 
 #### Checking all defined alerts

cmd/alert.go

@@ -16,13 +16,15 @@ import (
 )
 
 type AlertConfig struct {
-	AlertName     []string
-	Group         []string
-	ExcludeAlerts []string
-	ExcludeLabels []string
-	IncludeLabels []string
-	ProblemsOnly  bool
-	NoAlertsState string
+	AlertName           []string
+	Group               []string
+	ExcludeAlerts       []string
+	ExcludeLabels       []string
+	IncludeLabels       []string
+	ProblemsOnly        bool
+	HandleStateLabels   bool
+	HandleStateLabelKey string
+	NoAlertsState       string
 }
 
 var cliAlertConfig AlertConfig
@@ -144,8 +146,9 @@ inactive = 0`,
 
 			// Handle Inactive Alerts
 			if len(rl.AlertingRule.Alerts) == 0 {
-				// Counting states for perfdata
-				switch rl.GetStatus() {
+				// Counting states for perfdata. We don't use the state-label override here
+				// to have the acutal count from Prometheus
+				switch rl.GetStatus(false, "") {
 				case 0:
 					counterInactive++
 				case 1:
@@ -156,7 +159,7 @@ inactive = 0`,
 
 				sc := result.NewPartialResult()
 
-				_ = sc.SetState(rl.GetStatus())
+				_ = sc.SetState(rl.GetStatus(cliAlertConfig.HandleStateLabels, cliAlertConfig.HandleStateLabelKey))
 				sc.Output = rl.GetOutput()
 				overall.AddSubcheck(sc)
 			}
@@ -165,8 +168,9 @@ inactive = 0`,
 			if len(rl.AlertingRule.Alerts) > 0 {
 				// Handle Pending or Firing Alerts
 				for _, alert := range rl.AlertingRule.Alerts {
-					// Counting states for perfdata
-					switch rl.GetStatus() {
+					// Counting states for perfdata. We don't use the state-label override here
+					// to have the acutal count from Prometheus
+					switch rl.GetStatus(false, "") {
 					case 0:
 						counterInactive++
 					case 1:
@@ -177,7 +181,7 @@ inactive = 0`,
 
 					sc := result.NewPartialResult()
 
-					_ = sc.SetState(rl.GetStatus())
+					_ = sc.SetState(rl.GetStatus(cliAlertConfig.HandleStateLabels, cliAlertConfig.HandleStateLabelKey))
 					// Set the alert in the internal Type to generate the output
 					rl.Alert = alert
 					sc.Output = rl.GetOutput()
@@ -248,6 +252,13 @@ func init() {
 
 	fs.BoolVarP(&cliAlertConfig.ProblemsOnly, "problems", "P", false,
 		"Display only alerts which status is not inactive/OK. Note that in combination with the --name flag this might result in no alerts being displayed")
+
+	fs.BoolVarP(&cliAlertConfig.HandleStateLabels, "use-state-labels", "S", false,
+		"Use the AlertRule label 'icingaState' to override the exit state for firing alerts."+
+			"\nIf this flag is set the plugin looks for warning/critical/ok in the label 'icingaState'")
+
+	fs.StringVarP(&cliAlertConfig.HandleStateLabelKey, "state-labels-key", "K", "icingaState",
+		"Label key for the --use-state-labels. Can be used to override the default label key 'icingaState'")
 }
 
 // Function to convert state to integer.

internal/alert/alert.go

@@ -58,8 +58,10 @@ func FlattenRules(groups []v1.RuleGroup, wantedGroups []string) []Rule {
 	return rules
 }
 
-func (a *Rule) GetStatus() (status int) {
-	switch a.AlertingRule.State {
+func (a *Rule) GetStatus(handleStateLabels bool, labelKey string) (status int) {
+	state := a.AlertingRule.State
+
+	switch state {
 	case string(v1.AlertStateFiring):
 		status = check.Critical
 	case string(v1.AlertStatePending):
@@ -70,6 +72,24 @@ func (a *Rule) GetStatus() (status int) {
 		status = check.Unknown
 	}
 
+	if state == string(v1.AlertStateFiring) && handleStateLabels {
+		stateLabel, ok := a.AlertingRule.Labels[model.LabelName(labelKey)]
+
+		if !ok {
+			return status
+		}
+
+		lb := strings.ToLower(string(stateLabel))
+		switch lb {
+		case "warning":
+			status = check.Warning
+		case "critical":
+			status = check.Critical
+		case "ok":
+			status = check.OK
+		}
+	}
+
 	return status
 }
 

internal/alert/alert_test.go

@@ -10,7 +10,6 @@ import (
 )
 
 func TestGetStatus(t *testing.T) {
-
 	testTime := time.Now()
 
 	ar := v1.AlertingRule{
@@ -49,19 +48,67 @@ func TestGetStatus(t *testing.T) {
 		Alert:        ar.Alerts[0],
 	}
 
-	actual := r.GetStatus()
+	actual := r.GetStatus(false, "")
 	if actual != check.Critical {
 		t.Error("\nActual: ", actual, "\nExpected: ", check.Critical)
 	}
 
 	r.AlertingRule.State = "pending"
-	actual = r.GetStatus()
+	actual = r.GetStatus(false, "")
 	if actual != check.Warning {
 		t.Error("\nActual: ", actual, "\nExpected: ", check.Warning)
 	}
 
 }
 
+func TestGetStatus_WithLabel(t *testing.T) {
+	ar := v1.AlertingRule{
+		Alerts: []*v1.Alert{
+			{
+				Annotations: model.LabelSet{
+					"summary": "High request latency",
+				},
+				Labels: model.LabelSet{
+					"alertname": "HighRequestLatency",
+					"severity":  "page",
+				},
+				State: v1.AlertStateFiring,
+				Value: "1e+00",
+			},
+		},
+		Annotations: model.LabelSet{
+			"summary": "High request latency",
+		},
+		Labels: model.LabelSet{
+			"severity":    "page",
+			"icingaState": "OK",
+		},
+		Duration:       600,
+		Health:         v1.RuleHealthGood,
+		Name:           "HighRequestLatency",
+		Query:          "job:request_latency_seconds:mean5m{job=\"myjob\"} > 0.5",
+		LastError:      "",
+		EvaluationTime: 0.5,
+		State:          "firing",
+	}
+
+	r := Rule{
+		AlertingRule: ar,
+		Alert:        ar.Alerts[0],
+	}
+
+	actual := r.GetStatus(true, "icingaState")
+	if actual != check.OK {
+		t.Error("\nActual: ", actual, "\nExpected: ", check.Critical)
+	}
+
+	r.AlertingRule.State = "pending"
+	actual = r.GetStatus(true, "icingaState")
+	if actual != check.Warning {
+		t.Error("\nActual: ", actual, "\nExpected: ", check.Warning)
+	}
+}
+
 func TestGetOutput(t *testing.T) {
 
 	testTime := time.Now()

testdata/alertmanager/alert.rules

@@ -7,6 +7,7 @@ groups:
     for: 0m
     labels:
       severity: critical
+      icingaState: warning
     annotations:
       summary: Prometheus target missing (instance {{ $labels.instance }})
       description: "A Prometheus target has disappeared. An exporter might be crashed.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
@@ -16,6 +17,7 @@ groups:
     for: 0m
     labels:
       severity: low
+      icingaState: warning
     annotations:
       summary: Prometheus AlertManager job missing (instance {{ $labels.instance }})
       description: "A Prometheus AlertManager job has disappeared\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"