From 1b26a923364242556189a1f3e740676e9b65ed56 Mon Sep 17 00:00:00 2001 From: Shirly Radco Date: Thu, 12 Mar 2026 19:43:49 +0200 Subject: [PATCH 1/5] k8s: add Prometheus query layer and GET /alerts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Prometheus, Thanos, and Alertmanager query support with GET /api/v1/alerting/ alerts endpoint including alert component matching and alerting health status. Signed-off-by: Shirly Radco Signed-off-by: João Vilaça Signed-off-by: Aviv Litman Co-authored-by: AI Assistant --- internal/managementrouter/alerts_get.go | 109 +++ internal/managementrouter/alerts_get_test.go | 380 ++++++++ internal/managementrouter/query_filters.go | 35 + internal/managementrouter/router.go | 3 + pkg/alertcomponent/matcher.go | 381 ++++++++ pkg/k8s/alerting_health.go | 127 +++ pkg/k8s/client.go | 20 + pkg/k8s/const.go | 28 +- pkg/k8s/prometheus_alerts.go | 949 +++++++++++++++++++ pkg/k8s/relabeled_rules.go | 57 +- pkg/k8s/relabeled_rules_test.go | 157 --- pkg/k8s/rule_label_matchers.go | 91 ++ pkg/k8s/rule_label_matchers_test.go | 58 ++ pkg/k8s/types.go | 45 + pkg/management/get_alerting_health.go | 21 + pkg/management/get_alerts.go | 308 ++++++ pkg/management/get_alerts_test.go | 465 +++++++++ pkg/management/management_suite_test.go | 15 + pkg/management/testutils/k8s_client_mock.go | 53 +- pkg/management/types.go | 8 + pkg/management/update_classification.go | 25 - test/e2e/framework/framework.go | 58 ++ test/e2e/get_alerts_test.go | 297 ++++++ 23 files changed, 3457 insertions(+), 233 deletions(-) create mode 100644 internal/managementrouter/alerts_get.go create mode 100644 internal/managementrouter/alerts_get_test.go create mode 100644 internal/managementrouter/query_filters.go create mode 100644 pkg/alertcomponent/matcher.go create mode 100644 pkg/k8s/alerting_health.go create mode 100644 pkg/k8s/prometheus_alerts.go delete mode 100644 pkg/k8s/relabeled_rules_test.go create mode 100644 pkg/k8s/rule_label_matchers.go create mode 100644 pkg/k8s/rule_label_matchers_test.go create mode 100644 pkg/management/get_alerting_health.go create mode 100644 pkg/management/get_alerts.go create mode 100644 pkg/management/get_alerts_test.go create mode 100644 pkg/management/management_suite_test.go create mode 100644 test/e2e/get_alerts_test.go diff --git a/internal/managementrouter/alerts_get.go b/internal/managementrouter/alerts_get.go new file mode 100644 index 000000000..abb0ab462 --- /dev/null +++ b/internal/managementrouter/alerts_get.go @@ -0,0 +1,109 @@ +package managementrouter + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/openshift/monitoring-plugin/pkg/k8s" +) + +type GetAlertsResponse struct { + Data GetAlertsResponseData `json:"data"` + Warnings []string `json:"warnings,omitempty"` +} + +type GetAlertsResponseData struct { + Alerts []k8s.PrometheusAlert `json:"alerts"` +} + +func (hr *httpRouter) GetAlerts(w http.ResponseWriter, req *http.Request) { + state, labels, err := parseStateAndLabels(req.URL.Query()) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + ctx := req.Context() + + alerts, err := hr.managementClient.GetAlerts(ctx, k8s.GetAlertsRequest{ + Labels: labels, + State: state, + }) + if err != nil { + handleError(w, err) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(GetAlertsResponse{ + Data: GetAlertsResponseData{ + Alerts: alerts, + }, + Warnings: hr.alertWarnings(ctx), + }); err != nil { + log.WithError(err).Warn("failed to encode alerts response") + } +} + +func (hr *httpRouter) alertWarnings(ctx context.Context) []string { + health, ok := hr.alertingHealth(ctx) + if !ok { + return nil + } + + warnings := []string{} + if health.UserWorkloadEnabled && health.UserWorkload != nil { + warnings = append(warnings, buildRouteWarnings(health.UserWorkload.Prometheus, k8s.UserWorkloadRouteName, "user workload Prometheus")...) + warnings = append(warnings, buildRouteWarnings(health.UserWorkload.Alertmanager, k8s.UserWorkloadAlertmanagerRouteName, "user workload Alertmanager")...) + } + + return warnings +} + +//nolint:unused // used by the rules listing handler in a subsequent branch +func (hr *httpRouter) rulesWarnings(ctx context.Context) []string { + health, ok := hr.alertingHealth(ctx) + if !ok { + return nil + } + + if health.UserWorkloadEnabled && health.UserWorkload != nil { + return buildRouteWarnings(health.UserWorkload.Prometheus, k8s.UserWorkloadRouteName, "user workload Prometheus") + } + + return nil +} + +func (hr *httpRouter) alertingHealth(ctx context.Context) (k8s.AlertingHealth, bool) { + if hr.managementClient == nil { + return k8s.AlertingHealth{}, false + } + + health, err := hr.managementClient.GetAlertingHealth(ctx) + if err != nil { + log.WithError(err).Warn("alerting health unavailable") + return k8s.AlertingHealth{}, false + } + + return health, true +} + +func buildRouteWarnings(route k8s.AlertingRouteHealth, expectedName string, friendlyName string) []string { + if route.Name != "" && route.Name != expectedName { + return nil + } + if route.FallbackReachable { + return nil + } + + switch route.Status { + case k8s.RouteNotFound: + return []string{friendlyName + " route is missing"} + case k8s.RouteUnreachable: + return []string{friendlyName + " route is unreachable"} + default: + return nil + } +} diff --git a/internal/managementrouter/alerts_get_test.go b/internal/managementrouter/alerts_get_test.go new file mode 100644 index 000000000..1c931a00b --- /dev/null +++ b/internal/managementrouter/alerts_get_test.go @@ -0,0 +1,380 @@ +package managementrouter_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus/prometheus/model/relabel" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/openshift/monitoring-plugin/internal/managementrouter" + alertrule "github.com/openshift/monitoring-plugin/pkg/alert_rule" + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management" + "github.com/openshift/monitoring-plugin/pkg/management/testutils" + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +// agFixture holds mocks and the router for GetAlerts handler tests. +type agFixture struct { + router http.Handler + mockK8s *testutils.MockClient + mockPrometheusAlerts *testutils.MockPrometheusAlertsInterface +} + +func newAGFixture(t *testing.T) *agFixture { + t.Helper() + f := &agFixture{ + mockPrometheusAlerts: &testutils.MockPrometheusAlertsInterface{}, + } + f.mockK8s = &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return f.mockPrometheusAlerts + }, + } + f.rebuild() + return f +} + +func (f *agFixture) rebuild() { + mgmt := management.New(context.Background(), f.mockK8s) + f.router = managementrouter.New(mgmt) +} + +func (f *agFixture) get(t *testing.T, url string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, url, nil) + req.Header.Set("Authorization", "Bearer test-token") + w := httptest.NewRecorder() + f.router.ServeHTTP(w, req) + return w +} + +func decodeAlertsResp(t *testing.T, w *httptest.ResponseRecorder) managementrouter.GetAlertsResponse { + t.Helper() + var resp managementrouter.GetAlertsResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + return resp +} + +func TestGetAlerts_ParsesFlatQueryParams(t *testing.T) { + f := newAGFixture(t) + var captured k8s.GetAlertsRequest + f.mockPrometheusAlerts.GetAlertsFunc = func(_ context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + captured = req + return []k8s.PrometheusAlert{}, nil + } + + w := f.get(t, "/api/v1/alerting/alerts?namespace=ns1&severity=critical&state=firing&team=sre") + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + if captured.State != "firing" { + t.Errorf("expected state=firing, got %q", captured.State) + } + if captured.Labels["namespace"] != "ns1" { + t.Errorf("expected namespace=ns1, got %q", captured.Labels["namespace"]) + } + if captured.Labels["severity"] != "critical" { + t.Errorf("expected severity=critical, got %q", captured.Labels["severity"]) + } + if captured.Labels["team"] != "sre" { + t.Errorf("expected team=sre, got %q", captured.Labels["team"]) + } +} + +func TestGetAlerts_ReturnsAllAlerts(t *testing.T) { + f := newAGFixture(t) + testAlerts := []k8s.PrometheusAlert{ + { + Labels: map[string]string{managementlabels.AlertNameLabel: "HighCPUUsage", "severity": "warning", "namespace": "default"}, + Annotations: map[string]string{"description": "CPU usage is high"}, + State: "firing", + ActiveAt: time.Now(), + }, + { + Labels: map[string]string{managementlabels.AlertNameLabel: "LowMemory", "severity": "critical", "namespace": "monitoring"}, + Annotations: map[string]string{"description": "Memory is running low"}, + State: "firing", + ActiveAt: time.Now(), + }, + } + f.mockPrometheusAlerts.SetActiveAlerts(testAlerts) + + w := f.get(t, "/api/v1/alerting/alerts") + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + if ct := w.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("expected Content-Type application/json, got %q", ct) + } + resp := decodeAlertsResp(t, w) + if len(resp.Data.Alerts) != 2 { + t.Fatalf("expected 2 alerts, got %d", len(resp.Data.Alerts)) + } + if resp.Data.Alerts[0].Labels[managementlabels.AlertNameLabel] != "HighCPUUsage" { + t.Errorf("alert[0] name mismatch: %s", resp.Data.Alerts[0].Labels[managementlabels.AlertNameLabel]) + } + if resp.Data.Alerts[1].Labels[managementlabels.AlertNameLabel] != "LowMemory" { + t.Errorf("alert[1] name mismatch: %s", resp.Data.Alerts[1].Labels[managementlabels.AlertNameLabel]) + } +} + +func TestGetAlerts_WarningsWhenUserWorkloadRoutesMissing(t *testing.T) { + f := newAGFixture(t) + f.mockK8s.AlertingHealthFunc = func(_ context.Context) (k8s.AlertingHealth, error) { + return k8s.AlertingHealth{ + UserWorkloadEnabled: true, + UserWorkload: &k8s.AlertingStackHealth{ + Prometheus: k8s.AlertingRouteHealth{Status: k8s.RouteNotFound}, + Alertmanager: k8s.AlertingRouteHealth{Status: k8s.RouteNotFound}, + }, + }, nil + } + f.rebuild() + + w := f.get(t, "/api/v1/alerting/alerts") + resp := decodeAlertsResp(t, w) + + warnSet := make(map[string]bool) + for _, w := range resp.Warnings { + warnSet[w] = true + } + if !warnSet["user workload Prometheus route is missing"] { + t.Errorf("expected Prometheus route warning, got: %v", resp.Warnings) + } + if !warnSet["user workload Alertmanager route is missing"] { + t.Errorf("expected Alertmanager route warning, got: %v", resp.Warnings) + } +} + +func TestGetAlerts_SuppressesWarningsWhenFallbacksHealthy(t *testing.T) { + f := newAGFixture(t) + f.mockK8s.AlertingHealthFunc = func(_ context.Context) (k8s.AlertingHealth, error) { + return k8s.AlertingHealth{ + UserWorkloadEnabled: true, + UserWorkload: &k8s.AlertingStackHealth{ + Prometheus: k8s.AlertingRouteHealth{Status: k8s.RouteUnreachable, FallbackReachable: true}, + Alertmanager: k8s.AlertingRouteHealth{Status: k8s.RouteUnreachable, FallbackReachable: true}, + }, + }, nil + } + f.rebuild() + + w := f.get(t, "/api/v1/alerting/alerts") + resp := decodeAlertsResp(t, w) + if len(resp.Warnings) != 0 { + t.Errorf("expected no warnings, got: %v", resp.Warnings) + } +} + +func TestGetAlerts_ReturnsEmptyWhenNoAlerts(t *testing.T) { + f := newAGFixture(t) + f.mockPrometheusAlerts.SetActiveAlerts([]k8s.PrometheusAlert{}) + + w := f.get(t, "/api/v1/alerting/alerts") + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + resp := decodeAlertsResp(t, w) + if len(resp.Data.Alerts) != 0 { + t.Errorf("expected empty alerts, got %d", len(resp.Data.Alerts)) + } +} + +func TestGetAlerts_Returns500OnError(t *testing.T) { + f := newAGFixture(t) + f.mockPrometheusAlerts.GetAlertsFunc = func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return nil, fmt.Errorf("connection error") + } + + w := f.get(t, "/api/v1/alerting/alerts") + if w.Code != http.StatusInternalServerError { + t.Fatalf("expected 500, got %d: %s", w.Code, w.Body) + } + if body := w.Body.String(); !strings.Contains(body, "An unexpected error occurred") { + t.Errorf("expected error message, got: %s", body) + } +} + +func TestGetAlerts_ForwardsBearerToken(t *testing.T) { + f := newAGFixture(t) + var capturedCtx context.Context + f.mockPrometheusAlerts.GetAlertsFunc = func(ctx context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + capturedCtx = ctx + return []k8s.PrometheusAlert{}, nil + } + + req := httptest.NewRequest(http.MethodGet, "/api/v1/alerting/alerts", nil) + req.Header.Set("Authorization", "Bearer test-token-abc123") + w := httptest.NewRecorder() + f.router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + if token := k8s.BearerTokenFromContext(capturedCtx); token != "test-token-abc123" { + t.Errorf("expected token test-token-abc123, got %q", token) + } +} + +func TestGetAlerts_MissingAuthHeaderReturns401(t *testing.T) { + f := newAGFixture(t) + req := httptest.NewRequest(http.MethodGet, "/api/v1/alerting/alerts", nil) + w := httptest.NewRecorder() + f.router.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("expected 401, got %d: %s", w.Code, w.Body) + } +} + +func TestGetAlerts_EnrichesAlertWithRuleId(t *testing.T) { + f := newAGFixture(t) + baseRule := monitoringv1.Rule{ + Alert: "HighCPU", + Expr: intstr.FromString("node_cpu > 0.9"), + Labels: map[string]string{"severity": "critical"}, + } + ruleId := alertrule.GetAlertingRuleId(&baseRule) + + relabeledRule := monitoringv1.Rule{ + Alert: "HighCPU", + Expr: intstr.FromString("node_cpu > 0.9"), + Labels: map[string]string{ + managementlabels.AlertNameLabel: "HighCPU", + "severity": "critical", + k8s.AlertRuleLabelId: ruleId, + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "cluster-cpu-rules", + managementlabels.AlertingRuleLabelName: "my-alerting-rule", + }, + } + + f.mockK8s.RelabeledRulesFunc = func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{relabeledRule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == ruleId { + return relabeledRule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + } + f.mockK8s.NamespaceFunc = func() k8s.NamespaceInterface { + return &testutils.MockNamespaceInterface{ + IsClusterMonitoringNamespaceFunc: func(name string) bool { return name == "openshift-monitoring" }, + } + } + f.mockPrometheusAlerts.SetActiveAlerts([]k8s.PrometheusAlert{ + { + Labels: map[string]string{ + managementlabels.AlertNameLabel: "HighCPU", + "severity": "critical", + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.AlertBackendLabel: "alertmanager", + }, + Annotations: map[string]string{"summary": "CPU is high"}, + State: "firing", + ActiveAt: time.Now(), + }, + }) + f.rebuild() + + w := f.get(t, "/api/v1/alerting/alerts") + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + resp := decodeAlertsResp(t, w) + if len(resp.Data.Alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(resp.Data.Alerts)) + } + alert := resp.Data.Alerts[0] + if alert.AlertRuleId != ruleId { + t.Errorf("expected ruleId %s, got %s", ruleId, alert.AlertRuleId) + } + if alert.AlertComponent == "" { + t.Error("expected non-empty AlertComponent") + } + if alert.AlertLayer == "" { + t.Error("expected non-empty AlertLayer") + } +} + +func TestGetAlerts_EnrichesWithoutAlertingRuleCR(t *testing.T) { + f := newAGFixture(t) + baseRule := monitoringv1.Rule{ + Alert: "KubePodCrashLooping", + Expr: intstr.FromString("rate(kube_pod_restart_total[5m]) > 0"), + Labels: map[string]string{"severity": "warning"}, + } + ruleId := alertrule.GetAlertingRuleId(&baseRule) + + relabeledRule := monitoringv1.Rule{ + Alert: "KubePodCrashLooping", + Expr: intstr.FromString("rate(kube_pod_restart_total[5m]) > 0"), + Labels: map[string]string{ + managementlabels.AlertNameLabel: "KubePodCrashLooping", + "severity": "warning", + k8s.AlertRuleLabelId: ruleId, + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "kube-state-metrics", + }, + } + + f.mockK8s.RelabeledRulesFunc = func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{relabeledRule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == ruleId { + return relabeledRule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + } + f.mockK8s.NamespaceFunc = func() k8s.NamespaceInterface { + return &testutils.MockNamespaceInterface{ + IsClusterMonitoringNamespaceFunc: func(name string) bool { return name == "openshift-monitoring" }, + } + } + f.mockPrometheusAlerts.SetActiveAlerts([]k8s.PrometheusAlert{ + { + Labels: map[string]string{ + managementlabels.AlertNameLabel: "KubePodCrashLooping", + "severity": "warning", + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.AlertBackendLabel: "alertmanager", + }, + State: "firing", + ActiveAt: time.Now(), + }, + }) + f.rebuild() + + w := f.get(t, "/api/v1/alerting/alerts") + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + resp := decodeAlertsResp(t, w) + if len(resp.Data.Alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(resp.Data.Alerts)) + } + if resp.Data.Alerts[0].AlertRuleId != ruleId { + t.Errorf("expected ruleId %s, got %s", ruleId, resp.Data.Alerts[0].AlertRuleId) + } +} diff --git a/internal/managementrouter/query_filters.go b/internal/managementrouter/query_filters.go new file mode 100644 index 000000000..f8e3e5e9d --- /dev/null +++ b/internal/managementrouter/query_filters.go @@ -0,0 +1,35 @@ +package managementrouter + +import ( + "fmt" + "net/url" + "strings" +) + +var validStates = map[string]bool{ + "": true, + "pending": true, + "firing": true, + "silenced": true, +} + +// parseStateAndLabels returns the optional state filter and label matches. +// Any query param other than "state" is treated as a label match. +// Returns an error if the state value is not one of the known states. +func parseStateAndLabels(q url.Values) (string, map[string]string, error) { + state := strings.ToLower(strings.TrimSpace(q.Get("state"))) + if !validStates[state] { + return "", nil, fmt.Errorf("invalid state filter %q: must be one of pending, firing, silenced", q.Get("state")) + } + + labels := make(map[string]string) + for key, vals := range q { + if key == "state" { + continue + } + if len(vals) > 0 && strings.TrimSpace(vals[0]) != "" { + labels[strings.TrimSpace(key)] = strings.TrimSpace(vals[0]) + } + } + return state, labels, nil +} diff --git a/internal/managementrouter/router.go b/internal/managementrouter/router.go index fd0e0cda7..8706b7b04 100644 --- a/internal/managementrouter/router.go +++ b/internal/managementrouter/router.go @@ -43,6 +43,9 @@ func New(managementClient management.Client) *mux.Router { BaseURL: "/api/v1/alerting", BaseRouter: r, }) + // GET /alerts is not yet in the OpenAPI spec; registered manually + // until its branch adds the spec entry and generated bindings. + r.HandleFunc("/api/v1/alerting/alerts", hr.GetAlerts).Methods(http.MethodGet) return r } diff --git a/pkg/alertcomponent/matcher.go b/pkg/alertcomponent/matcher.go new file mode 100644 index 000000000..8aa6f9227 --- /dev/null +++ b/pkg/alertcomponent/matcher.go @@ -0,0 +1,381 @@ +package alertcomponent + +import ( + "regexp" + + "github.com/prometheus/common/model" + + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +const ( + labelNamespace = "namespace" + labelSeverity = "severity" +) + +func ns(values ...string) LabelsMatcher { + return NewLabelsMatcher(labelNamespace, NewStringValuesMatcher(values...)) +} + +func alertNames(values ...string) LabelsMatcher { + return NewLabelsMatcher(managementlabels.AlertNameLabel, NewStringValuesMatcher(values...)) +} + +func regexAlertNames(regexes ...*regexp.Regexp) LabelsMatcher { + return NewLabelsMatcher(managementlabels.AlertNameLabel, NewRegexValuesMatcher(regexes...)) +} + +func labelValues(key string, values ...string) LabelsMatcher { + return NewLabelsMatcher(key, NewStringValuesMatcher(values...)) +} + +func comp(component string, ms ...LabelsMatcher) componentMatcher { + return componentMatcher{component: component, matchers: ms} +} + +// LabelsMatcher represents a matcher definition for a set of labels. +// It matches if all of the label matchers match the labels. +type LabelsMatcher interface { + Matches(labels model.LabelSet) (match bool, keys []model.LabelName) + Equals(other LabelsMatcher) bool +} + +func NewLabelsMatcher(key string, matcher ValueMatcher) LabelsMatcher { + return labelMatcher{key: key, matcher: matcher} +} + +func NewStringValuesMatcher(keys ...string) ValueMatcher { + return stringMatcher(keys) +} + +func NewRegexValuesMatcher(regexes ...*regexp.Regexp) ValueMatcher { + return regexpMatcher(regexes) +} + +// labelMatcher represents a matcher definition for a label. +type labelMatcher struct { + key string + matcher ValueMatcher +} + +// Matches implements the LabelsMatcher interface. +func (l labelMatcher) Matches(labels model.LabelSet) (bool, []model.LabelName) { + if l.matcher.Matches(string(labels[model.LabelName(l.key)])) { + return true, []model.LabelName{model.LabelName(l.key)} + } + return false, nil +} + +// Equals implements the LabelsMatcher interface. +func (l labelMatcher) Equals(other LabelsMatcher) bool { + ol, ok := other.(labelMatcher) + if !ok { + return false + } + return l.key == ol.key && l.matcher.Equals(ol.matcher) +} + +// ValueMatcher represents a matcher for a specific value. +// +// Multiple implementations are provided for different types of matchers. +type ValueMatcher interface { + Matches(value string) bool + Equals(other ValueMatcher) bool +} + +// stringMatcher is a matcher for a list of strings. +// +// It matches if the value is in the list of strings. +type stringMatcher []string + +func (s stringMatcher) Matches(value string) bool { + for _, v := range s { + if v == value { + return true + } + } + return false +} + +// Equals implements the ValueMatcher interface. +func (s stringMatcher) Equals(other ValueMatcher) bool { + o, ok := other.(stringMatcher) + if !ok { + return false + } + return equalsNoOrder(s, o) +} + +// regexpMatcher is a matcher for a list of regular expressions. +// +// It matches if the value matches any of the regular expressions. +type regexpMatcher []*regexp.Regexp + +func (r regexpMatcher) Matches(value string) bool { + for _, re := range r { + if re.MatchString(value) { + return true + } + } + return false +} + +// Equals implements the ValueMatcher interface. +func (r regexpMatcher) Equals(other ValueMatcher) bool { + o, ok := other.(regexpMatcher) + if !ok { + return false + } + s1 := make([]string, 0, len(r)) + for _, re := range r { + s1 = append(s1, re.String()) + } + s2 := make([]string, 0, len(o)) + for _, re := range o { + s2 = append(s2, re.String()) + } + return equalsNoOrder(s1, s2) +} + +func equalsNoOrder(a, b []string) bool { + if len(a) != len(b) { + return false + } + + seen := make(map[string]int, len(a)) + for _, v := range a { + seen[v]++ + } + for _, v := range b { + if seen[v] == 0 { + return false + } + seen[v]-- + } + return true +} + +// componentMatcher represents a matcher definition for a component. +// +// It matches if any of the label matchers match the labels. +type componentMatcher struct { + component string + matchers []LabelsMatcher +} + +// findComponent tries to determine a component for given labels using the provided matchers. +// +// It returns the component and the keys that matched. +// If no match is found, it returns an empty component and nil keys. +func findComponent(compMatchers []componentMatcher, labels model.LabelSet) ( + component string, keys []model.LabelName) { + for _, compMatcher := range compMatchers { + for _, labelsMatcher := range compMatcher.matchers { + if matches, keys := labelsMatcher.Matches(labels); matches { + return compMatcher.component, keys + } + } + } + return "", nil +} + +// componentMatcherFn is a function that tries matching provided labels to a component. +// It returns the layer, component and the keys from the labels that were used for matching. +// If no match is found, it returns an empty layer, component and nil keys. +type componentMatcherFn func(labels model.LabelSet) (layer, comp model.LabelValue, keys []model.LabelName) + +func evalMatcherFns(fns []componentMatcherFn, labels model.LabelSet) ( + layer, comp string, labelsSubset model.LabelSet) { + for _, fn := range fns { + if layer, comp, keys := fn(labels); layer != "" { + return string(layer), string(comp), getLabelsSubset(labels, keys...) + } + } + return "Others", "Others", getLabelsSubset(labels) +} + +// getLabelsSubset returns a subset of the labels with given keys. +func getLabelsSubset(m model.LabelSet, keys ...model.LabelName) model.LabelSet { + keys = append([]model.LabelName{ + model.LabelName(labelNamespace), + model.LabelName(managementlabels.AlertNameLabel), + model.LabelName(labelSeverity), + }, keys...) + return getMapSubset(m, keys...) +} + +// getMapSubset returns a subset of the labels with given keys. +func getMapSubset(m model.LabelSet, keys ...model.LabelName) model.LabelSet { + subset := make(model.LabelSet, len(keys)) + for _, key := range keys { + if val, ok := m[key]; ok { + subset[key] = val + } + } + return subset +} + +var ( + nodeAlerts []model.LabelValue = []model.LabelValue{ + "NodeClockNotSynchronising", + "KubeNodeNotReady", + "KubeNodeUnreachable", + "NodeSystemSaturation", + "NodeFilesystemSpaceFillingUp", + "NodeFilesystemAlmostOutOfSpace", + "NodeMemoryMajorPagesFaults", + "NodeNetworkTransmitErrs", + "NodeTextFileCollectorScrapeError", + "NodeFilesystemFilesFillingUp", + "NodeNetworkReceiveErrs", + "NodeClockSkewDetected", + "NodeFilesystemAlmostOutOfFiles", + "NodeWithoutOVNKubeNodePodRunning", + "InfraNodesNeedResizingSRE", + "NodeHighNumberConntrackEntriesUsed", + "NodeMemHigh", + "NodeNetworkInterfaceFlapping", + "NodeWithoutSDNPod", + "NodeCpuHigh", + "CriticalNodeNotReady", + "NodeFileDescriptorLimit", + "MCCPoolAlert", + "MCCDrainError", + "MCDRebootError", + "MCDPivotError", + } + + coreMatchers = []componentMatcher{ + comp("etcd", ns("openshift-etcd", "openshift-etcd-operator")), + comp("kube-apiserver", ns("openshift-kube-apiserver", "openshift-kube-apiserver-operator")), + comp("kube-controller-manager", ns("openshift-kube-controller-manager", "openshift-kube-controller-manager-operator", "kube-system")), + comp("kube-scheduler", ns("openshift-kube-scheduler", "openshift-kube-scheduler-operator")), + comp("machine-approver", ns("openshift-cluster-machine-approver", "openshift-machine-approver-operator")), + comp("machine-config", + ns("openshift-machine-config-operator"), + alertNames( + "HighOverallControlPlaneMemory", + "ExtremelyHighIndividualControlPlaneMemory", + "MissingMachineConfig", + "MCCBootImageUpdateError", + "KubeletHealthState", + "SystemMemoryExceedsReservation", + ), + ), + comp("version", + ns("openshift-cluster-version", "openshift-version-operator"), + alertNames("ClusterNotUpgradeable", "UpdateAvailable"), + ), + comp("dns", ns("openshift-dns", "openshift-dns-operator")), + comp("authentication", ns("openshift-authentication", "openshift-oauth-apiserver", "openshift-authentication-operator")), + comp("cert-manager", ns("openshift-cert-manager", "openshift-cert-manager-operator")), + comp("cloud-controller-manager", ns("openshift-cloud-controller-manager", "openshift-cloud-controller-manager-operator")), + comp("cloud-credential", ns("openshift-cloud-credential-operator")), + comp("cluster-api", ns("openshift-cluster-api", "openshift-cluster-api-operator")), + comp("config-operator", ns("openshift-config-operator")), + comp("kube-storage-version-migrator", ns("openshift-kube-storage-version-migrator", "openshift-kube-storage-version-migrator-operator")), + comp("image-registry", ns("openshift-image-registry", "openshift-image-registry-operator")), + comp("ingress", ns("openshift-ingress", "openshift-route-controller-manager", "openshift-ingress-canary", "openshift-ingress-operator")), + comp("console", ns("openshift-console", "openshift-console-operator")), + comp("insights", ns("openshift-insights", "openshift-insights-operator")), + comp("machine-api", ns("openshift-machine-api", "openshift-machine-api-operator")), + comp("monitoring", ns("openshift-monitoring", "openshift-monitoring-operator")), + comp("network", ns("openshift-network-operator", "openshift-ovn-kubernetes", "openshift-multus", "openshift-network-diagnostics", "openshift-sdn")), + comp("node-tuning", ns("openshift-cluster-node-tuning-operator", "openshift-node-tuning-operator")), + comp("openshift-apiserver", ns("openshift-apiserver", "openshift-apiserver-operator")), + comp("openshift-controller-manager", ns("openshift-controller-manager", "openshift-controller-manager-operator")), + comp("openshift-samples", ns("openshift-cluster-samples-operator", "openshift-samples-operator")), + comp("operator-lifecycle-manager", ns("openshift-operator-lifecycle-manager")), + comp("service-ca", ns("openshift-service-ca", "openshift-service-ca-operator")), + comp("storage", ns("openshift-storage", "openshift-cluster-csi-drivers", "openshift-cluster-storage-operator", "openshift-storage-operator")), + comp("vertical-pod-autoscaler", ns("openshift-vertical-pod-autoscaler", "openshift-vertical-pod-autoscaler-operator")), + comp("marketplace", ns("openshift-marketplace", "openshift-marketplace-operator")), + } + + workloadMatchers = []componentMatcher{ + comp("openshift-compliance", ns("openshift-compliance")), + comp("openshift-file-integrity", ns("openshift-file-integrity")), + comp("openshift-logging", ns("openshift-logging")), + comp("openshift-user-workload-monitoring", ns("openshift-user-workload-monitoring")), + comp("openshift-gitops", ns("openshift-gitops", "openshift-gitops-operator")), + comp("openshift-operators", ns("openshift-operators")), + comp("openshift-local-storage", ns("openshift-local-storage")), + comp("quay", labelValues("container", "quay-app", "quay-mirror", "quay-app-upgrade")), + comp("Argo", regexAlertNames(regexp.MustCompile("^Argo"))), + } +) + +var cvoAlerts = []model.LabelValue{"ClusterOperatorDown", "ClusterOperatorDegraded"} + +func cvoAlertsMatcher(labels model.LabelSet) (layer, comp model.LabelValue, keys []model.LabelName) { + for _, v := range cvoAlerts { + if labels[managementlabels.AlertNameLabel] == v { + component := labels["name"] + if component == "" { + component = "version" + } + return "cluster", component, nil + } + } + return "", "", nil +} + +func kubevirtOperatorMatcher(labels model.LabelSet) (layer, comp model.LabelValue, keys []model.LabelName) { + if labels["kubernetes_operator_part_of"] != "kubevirt" { + return "", "", nil + } + if labels["kubernetes_operator_component"] == "cnv-observability" { + return "", "", nil + } + if labels["operator_health_impact"] == "none" && labels["kubernetes_operator_component"] == "kubevirt" { + return "namespace", "OpenShift Virtualization Virtual Machine", []model.LabelName{ + "kubernetes_operator_part_of", + "kubernetes_operator_component", + "operator_health_impact", + } + } + return "cluster", "OpenShift Virtualization Operator", []model.LabelName{ + "kubernetes_operator_part_of", + "kubernetes_operator_component", + "operator_health_impact", + } +} + +func computeMatcher(labels model.LabelSet) (layer, comp model.LabelValue, keys []model.LabelName) { + for _, nodeAlert := range nodeAlerts { + if labels[managementlabels.AlertNameLabel] == nodeAlert { + component := "compute" + return "cluster", model.LabelValue(component), nil + } + } + return "", "", nil +} + +func coreMatcher(labels model.LabelSet) (layer, comp model.LabelValue, keys []model.LabelName) { + // Try matching against core components. + if component, keys := findComponent(coreMatchers, labels); component != "" { + return "cluster", model.LabelValue(component), keys + } + return "", "", nil +} + +func workloadMatcher(labels model.LabelSet) (layer, comp model.LabelValue, keys []model.LabelName) { + // Try matching against workload components. + if component, keys := findComponent(workloadMatchers, labels); component != "" { + return "namespace", model.LabelValue(component), keys + } + return "", "", nil +} + +// DetermineComponent determines the component for a given set of labels. +// It returns the layer and component strings. +func DetermineComponent(labels model.LabelSet) (layer, component string) { + layer, component, _ = evalMatcherFns([]componentMatcherFn{ + cvoAlertsMatcher, + kubevirtOperatorMatcher, + computeMatcher, + coreMatcher, + workloadMatcher, + }, labels) + return layer, component +} diff --git a/pkg/k8s/alerting_health.go b/pkg/k8s/alerting_health.go new file mode 100644 index 000000000..0fdc40880 --- /dev/null +++ b/pkg/k8s/alerting_health.go @@ -0,0 +1,127 @@ +package k8s + +import ( + "context" + "fmt" + "strings" + "sync" + + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +const ( + clusterMonitoringConfigMap = "cluster-monitoring-config" + clusterMonitoringConfigKey = "config.yaml" +) + +type clusterMonitoringConfig struct { + EnableUserWorkload bool `yaml:"enableUserWorkload"` +} + +// clusterMonitoringConfigManager watches the cluster-monitoring-config ConfigMap +// via an informer and caches the parsed enableUserWorkload value so that +// AlertingHealth never needs a live API call. +type clusterMonitoringConfigManager struct { + informer cache.SharedIndexInformer + + mu sync.RWMutex + enabled bool + err error +} + +func newClusterMonitoringConfigManager(ctx context.Context, clientset *kubernetes.Clientset) (*clusterMonitoringConfigManager, error) { + informer := cache.NewSharedIndexInformer( + cache.NewListWatchFromClient( + clientset.CoreV1().RESTClient(), + "configmaps", + ClusterMonitoringNamespace, + fields.OneTermEqualSelector("metadata.name", clusterMonitoringConfigMap), + ), + &corev1.ConfigMap{}, + 0, + cache.Indexers{}, + ) + + m := &clusterMonitoringConfigManager{ + informer: informer, + } + + _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + cm, ok := obj.(*corev1.ConfigMap) + if !ok { + return + } + m.handleUpdate(cm) + }, + UpdateFunc: func(_, newObj interface{}) { + cm, ok := newObj.(*corev1.ConfigMap) + if !ok { + return + } + m.handleUpdate(cm) + }, + DeleteFunc: func(_ interface{}) { + m.mu.Lock() + defer m.mu.Unlock() + m.enabled = false + m.err = nil + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to add event handler to cluster-monitoring-config informer: %w", err) + } + + go informer.Run(ctx.Done()) + + if !cache.WaitForNamedCacheSync("ClusterMonitoringConfig informer", ctx.Done(), informer.HasSynced) { + return nil, fmt.Errorf("failed to sync ClusterMonitoringConfig informer") + } + + return m, nil +} + +func (m *clusterMonitoringConfigManager) handleUpdate(cm *corev1.ConfigMap) { + m.mu.Lock() + defer m.mu.Unlock() + + raw, ok := cm.Data[clusterMonitoringConfigKey] + if !ok || strings.TrimSpace(raw) == "" { + m.enabled = false + m.err = nil + return + } + + var cfg clusterMonitoringConfig + if err := yaml.Unmarshal([]byte(raw), &cfg); err != nil { + m.enabled = false + m.err = fmt.Errorf("parse cluster monitoring config.yaml: %w", err) + return + } + + m.enabled = cfg.EnableUserWorkload + m.err = nil +} + +func (m *clusterMonitoringConfigManager) userWorkloadEnabled() (bool, error) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.enabled, m.err +} + +// AlertingHealth returns alerting route health and UWM enablement status. +func (c *client) AlertingHealth(ctx context.Context) (AlertingHealth, error) { + health := c.prometheusAlerts.alertingHealth(ctx) + + enabled, err := c.clusterMonitoringConfig.userWorkloadEnabled() + if err != nil { + return health, fmt.Errorf("failed to determine user workload enablement: %w", err) + } + health.UserWorkloadEnabled = enabled + + return health, nil +} diff --git a/pkg/k8s/client.go b/pkg/k8s/client.go index 6370270ff..e16be6dd2 100644 --- a/pkg/k8s/client.go +++ b/pkg/k8s/client.go @@ -5,6 +5,7 @@ import ( "fmt" osmv1client "github.com/openshift/client-go/monitoring/clientset/versioned" + routeclient "github.com/openshift/client-go/route/clientset/versioned" monitoringv1client "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" @@ -21,11 +22,14 @@ type client struct { osmv1clientset *osmv1client.Clientset config *rest.Config + prometheusAlerts *prometheusAlerts + prometheusRuleManager *prometheusRuleManager alertRelabelConfigManager *alertRelabelConfigManager alertingRuleManager *alertingRuleManager namespaceManager *namespaceManager relabeledRulesManager *relabeledRulesManager + clusterMonitoringConfig *clusterMonitoringConfigManager } func NewClient(ctx context.Context, config *rest.Config) (Client, error) { @@ -44,6 +48,11 @@ func NewClient(ctx context.Context, config *rest.Config) (Client, error) { return nil, fmt.Errorf("failed to create osmv1 clientset: %w", err) } + routeClientset, err := routeclient.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create route clientset: %w", err) + } + c := &client{ clientset: clientset, monitoringv1clientset: monitoringv1clientset, @@ -56,6 +65,8 @@ func NewClient(ctx context.Context, config *rest.Config) (Client, error) { return nil, fmt.Errorf("failed to create PrometheusRule manager: %w", err) } + c.prometheusAlerts = newPrometheusAlerts(routeClientset, clientset.CoreV1(), config, c.prometheusRuleManager) + c.alertRelabelConfigManager, err = newAlertRelabelConfigManager(ctx, osmv1clientset, config) if err != nil { return nil, fmt.Errorf("failed to create alert relabel config manager: %w", err) @@ -71,6 +82,11 @@ func NewClient(ctx context.Context, config *rest.Config) (Client, error) { return nil, fmt.Errorf("failed to create namespace manager: %w", err) } + c.clusterMonitoringConfig, err = newClusterMonitoringConfigManager(ctx, clientset) + if err != nil { + return nil, fmt.Errorf("failed to create cluster monitoring config manager: %w", err) + } + c.relabeledRulesManager, err = newRelabeledRulesManager(ctx, c.namespaceManager, c.alertRelabelConfigManager, monitoringv1clientset, clientset) if err != nil { return nil, fmt.Errorf("failed to create relabeled rules config manager: %w", err) @@ -87,6 +103,10 @@ func (c *client) TestConnection(_ context.Context) error { return nil } +func (c *client) PrometheusAlerts() PrometheusAlertsInterface { + return c.prometheusAlerts +} + func (c *client) PrometheusRules() PrometheusRuleInterface { return c.prometheusRuleManager } diff --git a/pkg/k8s/const.go b/pkg/k8s/const.go index 243cea8d8..ff9eaf4c5 100644 --- a/pkg/k8s/const.go +++ b/pkg/k8s/const.go @@ -1,5 +1,31 @@ package k8s const ( - ClusterMonitoringNamespace = "openshift-monitoring" + ClusterMonitoringNamespace = "openshift-monitoring" + UserWorkloadMonitoringNamespace = "openshift-user-workload-monitoring" + + PlatformRouteName = "prometheus-k8s" + PlatformAlertmanagerRouteName = "alertmanager-main" + UserWorkloadRouteName = "prometheus-user-workload" + UserWorkloadAlertmanagerRouteName = "alertmanager-user-workload" + PrometheusAlertsPath = "/v1/alerts" + PrometheusRulesPath = "/v1/rules" + AlertmanagerAlertsPath = "/api/v2/alerts" + UserWorkloadAlertmanagerPort = 9095 + UserWorkloadPrometheusServiceName = "prometheus-user-workload-web" + UserWorkloadPrometheusPort = 9090 + + ThanosQuerierServiceName = "thanos-querier" + DefaultThanosQuerierTenancyRulesPort = 9093 + ThanosQuerierTenancyAlertsPath = "/api/v1/alerts" + ThanosQuerierTenancyRulesPath = "/api/v1/rules" + ServiceCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" + + AlertSourceLabel = "openshift_io_alert_source" + AlertSourcePlatform = "platform" + AlertSourceUser = "user" + AlertBackendLabel = "openshift_io_alert_backend" + AlertBackendAM = "alertmanager" + AlertBackendProm = "prometheus" + AlertBackendThanos = "thanos" ) diff --git a/pkg/k8s/prometheus_alerts.go b/pkg/k8s/prometheus_alerts.go new file mode 100644 index 000000000..c01303cea --- /dev/null +++ b/pkg/k8s/prometheus_alerts.go @@ -0,0 +1,949 @@ +package k8s + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" + + routev1 "github.com/openshift/api/route/v1" + routeclient "github.com/openshift/client-go/route/clientset/versioned" + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" +) + +var prometheusLog = logrus.WithField("module", "k8s-prometheus") + +const ( + namespaceCacheTTL = 30 * time.Second + serviceHealthTimeout = 5 * time.Second + serviceRequestTimeout = 10 * time.Second + maxTenancyProbeTargets = 3 +) + +type namespaceCache struct { + mu sync.Mutex + expiresAt time.Time + ttl time.Duration + value []string +} + +func newNamespaceCache(ttl time.Duration) *namespaceCache { + return &namespaceCache{ttl: ttl} +} + +func (c *namespaceCache) get() ([]string, bool) { + if c == nil { + return nil, false + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.expiresAt.IsZero() || time.Now().After(c.expiresAt) { + return nil, false + } + return copyStringSlice(c.value), true +} + +func (c *namespaceCache) set(namespaces []string) { + if c == nil { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + c.value = copyStringSlice(namespaces) + c.expiresAt = time.Now().Add(c.ttl) +} + +type prometheusAlerts struct { + routeClient routeclient.Interface + coreClient corev1client.CoreV1Interface + config *rest.Config + ruleManager PrometheusRuleInterface + nsCache *namespaceCache +} + +// GetAlertsRequest holds parameters for filtering alerts +type GetAlertsRequest struct { + // Labels filters alerts by labels + Labels map[string]string + // State filters alerts by state: "firing", "pending", "silenced", or "" for all states + State string +} + +type PrometheusAlert struct { + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` + State string `json:"state"` + ActiveAt time.Time `json:"activeAt"` + Value string `json:"value"` + + AlertRuleId string `json:"alertRuleId,omitempty"` + AlertComponent string `json:"alertComponent,omitempty"` + AlertLayer string `json:"alertLayer,omitempty"` +} + +type prometheusAlertsData struct { + Alerts []PrometheusAlert `json:"alerts"` +} + +type prometheusAlertsResponse struct { + Status string `json:"status"` + Data prometheusAlertsData `json:"data"` +} + +type prometheusRulesData struct { + Groups []PrometheusRuleGroup `json:"groups"` +} + +type prometheusRulesResponse struct { + Status string `json:"status"` + Data prometheusRulesData `json:"data"` +} + +type alertmanagerAlertStatus struct { + State string `json:"state"` +} + +type alertmanagerAlert struct { + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` + StartsAt time.Time `json:"startsAt"` + EndsAt time.Time `json:"endsAt"` + GeneratorURL string `json:"generatorURL"` + Status alertmanagerAlertStatus `json:"status"` +} + +func newPrometheusAlerts(routeClient routeclient.Interface, coreClient corev1client.CoreV1Interface, config *rest.Config, ruleManager PrometheusRuleInterface) *prometheusAlerts { + return &prometheusAlerts{ + routeClient: routeClient, + coreClient: coreClient, + config: config, + ruleManager: ruleManager, + nsCache: newNamespaceCache(namespaceCacheTTL), + } +} + +func (pa *prometheusAlerts) GetAlerts(ctx context.Context, req GetAlertsRequest) ([]PrometheusAlert, error) { + platformAlerts, err := pa.getAlertsForSource(ctx, ClusterMonitoringNamespace, PlatformRouteName, PlatformAlertmanagerRouteName, AlertSourcePlatform) + if err != nil { + // Namespace-scoped callers (Thanos tenancy) often lack platform + // Prometheus access. Soft-fail so tenancy results are still returned. + if namespaceFromLabels(req.Labels) == "" { + return nil, err + } + prometheusLog.Warnf("failed to get platform alerts (continuing with namespace filter): %v", err) + } + + userAlerts, err := pa.getUserWorkloadAlerts(ctx, req) + if err != nil { + prometheusLog.Warnf("failed to get user workload alerts: %v", err) + } + + mergedAlerts := append(platformAlerts, userAlerts...) + + out := make([]PrometheusAlert, 0, len(mergedAlerts)) + for _, a := range mergedAlerts { + // Filter alerts based on state if provided + if !matchesAlertState(req.State, a.State) { + continue + } + + // Filter alerts based on labels if provided + if !labelsMatch(&req, &a) { + continue + } + + out = append(out, a) + } + return out, nil +} + +func matchesAlertState(requestedState string, alertState string) bool { + if requestedState == "" { + return true + } + if requestedState == "firing" { + return alertState == "firing" || alertState == "silenced" + } + return alertState == requestedState +} + +func (pa *prometheusAlerts) GetRules(ctx context.Context, req GetRulesRequest) ([]PrometheusRuleGroup, error) { + platformRules, err := pa.getRulesViaProxy(ctx, ClusterMonitoringNamespace, PlatformRouteName, AlertSourcePlatform) + if err != nil { + // Namespace-scoped callers (Thanos tenancy) often lack platform + // Prometheus access. Soft-fail so tenancy results are still returned. + if namespaceFromLabels(req.Labels) == "" { + return nil, err + } + prometheusLog.Warnf("failed to get platform rules (continuing with namespace filter): %v", err) + } + + userRules, err := pa.getUserWorkloadRules(ctx, req) + if err != nil { + prometheusLog.Warnf("failed to get user workload rules: %v", err) + } + + groups := append(platformRules, userRules...) + + matchers, err := compileRuleLabelMatchers(req) + if err != nil { + return nil, err + } + if len(matchers) == 0 { + return groups, nil + } + + return filterRuleGroupsByLabelMatchers(groups, matchers), nil +} + +func (pa *prometheusAlerts) alertingHealth(ctx context.Context) AlertingHealth { + userPrometheus := pa.routeHealth(ctx, UserWorkloadMonitoringNamespace, UserWorkloadRouteName, PrometheusRulesPath) + if userPrometheus.Status != RouteReachable { + if ok := pa.thanosTenancyReachable(ctx, ThanosQuerierTenancyAlertsPath); ok { + userPrometheus.FallbackReachable = true + } + } + + userAlertmanager := pa.routeHealth(ctx, UserWorkloadMonitoringNamespace, UserWorkloadAlertmanagerRouteName, AlertmanagerAlertsPath) + if userAlertmanager.Status != RouteReachable { + if ok := pa.serviceReachable(ctx, UserWorkloadMonitoringNamespace, UserWorkloadAlertmanagerRouteName, UserWorkloadAlertmanagerPort, AlertmanagerAlertsPath); ok { + userAlertmanager.FallbackReachable = true + } + } + + platformStack := pa.stackHealth(ctx, ClusterMonitoringNamespace, PlatformRouteName, PlatformAlertmanagerRouteName) + userWorkloadStack := AlertingStackHealth{ + Prometheus: userPrometheus, + Alertmanager: userAlertmanager, + } + + return AlertingHealth{ + Platform: &platformStack, + UserWorkload: &userWorkloadStack, + } +} + +func (pa *prometheusAlerts) stackHealth(ctx context.Context, namespace string, promRouteName string, amRouteName string) AlertingStackHealth { + return AlertingStackHealth{ + Prometheus: pa.routeHealth(ctx, namespace, promRouteName, PrometheusRulesPath), + Alertmanager: pa.routeHealth(ctx, namespace, amRouteName, AlertmanagerAlertsPath), + } +} + +func (pa *prometheusAlerts) routeHealth(ctx context.Context, namespace string, routeName string, path string) AlertingRouteHealth { + health := AlertingRouteHealth{ + Name: routeName, + Namespace: namespace, + } + + if pa.routeClient == nil { + health.Error = "route client is not configured" + return health + } + + route, err := pa.routeClient.RouteV1().Routes(namespace).Get(ctx, routeName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + health.Status = RouteNotFound + health.Error = err.Error() + return health + } + health.Error = err.Error() + return health + } + + url := buildRouteURL(route.Spec.Host, route.Spec.Path, path) + client, err := pa.createHTTPClient() + if err != nil { + health.Status = RouteUnreachable + health.Error = err.Error() + return health + } + + if _, err := pa.executeRequest(ctx, client, url); err != nil { + health.Status = RouteUnreachable + health.Error = err.Error() + return health + } + + health.Status = RouteReachable + return health +} + +func (pa *prometheusAlerts) getAlertsForSource(ctx context.Context, namespace string, promRouteName string, amRouteName string, source string) ([]PrometheusAlert, error) { + amAlerts, amErr := pa.getAlertmanagerAlerts(ctx, namespace, amRouteName, source) + promAlerts, promErr := pa.getAlertsViaProxy(ctx, namespace, promRouteName, source) + + if amErr == nil { + pending := filterAlertsByState(promAlerts, "pending") + return append(amAlerts, pending...), nil + } + + if promErr != nil { + return nil, promErr + } + + return promAlerts, nil +} + +func (pa *prometheusAlerts) getUserWorkloadAlerts(ctx context.Context, req GetAlertsRequest) ([]PrometheusAlert, error) { + if shouldPreferUserAlertmanager(req.State) { + alerts, err := pa.getUserWorkloadAlertsViaAlertmanager(ctx) + if err == nil { + return alerts, nil + } + prometheusLog.Warnf("failed to get user workload alerts via alertmanager: %v", err) + } + + namespace := namespaceFromLabels(req.Labels) + if namespace != "" { + alerts, err := pa.getAlertsViaThanosTenancy(ctx, namespace, AlertSourceUser) + if err == nil { + return alerts, nil + } + prometheusLog.Warnf("failed to get user workload alerts via thanos tenancy: %v", err) + } + + userNamespaces := pa.userRuleNamespaces(ctx) + if len(userNamespaces) > 0 { + alerts, err := pa.getAlertsViaThanosTenancyNamespaces(ctx, userNamespaces, AlertSourceUser) + if err == nil { + return alerts, nil + } + prometheusLog.Warnf("failed to get user workload alerts via thanos tenancy namespaces: %v", err) + } + + return pa.getAlertsForSource(ctx, UserWorkloadMonitoringNamespace, UserWorkloadRouteName, UserWorkloadAlertmanagerRouteName, AlertSourceUser) +} + +func shouldPreferUserAlertmanager(state string) bool { + return state == "firing" || state == "silenced" +} + +func (pa *prometheusAlerts) getUserWorkloadAlertsViaAlertmanager(ctx context.Context) ([]PrometheusAlert, error) { + alerts, err := pa.getAlertmanagerAlerts(ctx, UserWorkloadMonitoringNamespace, UserWorkloadAlertmanagerRouteName, AlertSourceUser) + if err != nil { + alerts, err = pa.getAlertmanagerAlertsViaService(ctx, UserWorkloadMonitoringNamespace, UserWorkloadAlertmanagerRouteName, UserWorkloadAlertmanagerPort, AlertSourceUser) + if err != nil { + return nil, err + } + } + + pending, err := pa.getAlertsViaProxy(ctx, UserWorkloadMonitoringNamespace, UserWorkloadRouteName, AlertSourceUser) + if err != nil { + pending, err = pa.getPrometheusAlertsViaService(ctx, UserWorkloadMonitoringNamespace, UserWorkloadPrometheusServiceName, UserWorkloadPrometheusPort, AlertSourceUser) + if err != nil { + return alerts, nil + } + } + + return append(alerts, filterAlertsByState(pending, "pending")...), nil +} + +func (pa *prometheusAlerts) getPrometheusAlertsViaService(ctx context.Context, namespace string, serviceName string, port int32, source string) ([]PrometheusAlert, error) { + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + timeoutCtx, cancel := context.WithTimeout(ctx, serviceRequestTimeout) + defer cancel() + ctx = timeoutCtx + } + + raw, err := pa.getServiceResponse(ctx, namespace, serviceName, port, PrometheusAlertsPath) + if err != nil { + return nil, err + } + + var alertsResp prometheusAlertsResponse + if err := json.Unmarshal(raw, &alertsResp); err != nil { + return nil, fmt.Errorf("decode prometheus response: %w", err) + } + + if alertsResp.Status != "success" { + return nil, fmt.Errorf("prometheus API returned non-success status: %s", alertsResp.Status) + } + + applyAlertMetadata(alertsResp.Data.Alerts, source, AlertBackendProm) + return alertsResp.Data.Alerts, nil +} + +func (pa *prometheusAlerts) getAlertmanagerAlertsViaService(ctx context.Context, namespace string, serviceName string, port int32, source string) ([]PrometheusAlert, error) { + raw, err := pa.getServiceResponse(ctx, namespace, serviceName, port, AlertmanagerAlertsPath) + if err != nil { + return nil, err + } + + converted, err := parseAlertmanagerResponse(raw) + if err != nil { + return nil, err + } + + applyAlertMetadata(converted, source, AlertBackendAM) + if len(converted) == 0 { + return []PrometheusAlert{}, nil + } + return converted, nil +} + +// parseAlertmanagerResponse unmarshals a raw Alertmanager GET /api/v2/alerts +// response and converts it to PrometheusAlert structs. No routing labels are +// added — callers that need them should call applyAlertMetadata. +func parseAlertmanagerResponse(raw []byte) ([]PrometheusAlert, error) { + var amAlerts []alertmanagerAlert + if err := json.Unmarshal(raw, &amAlerts); err != nil { + return nil, fmt.Errorf("decode alertmanager response: %w", err) + } + + converted := make([]PrometheusAlert, 0, len(amAlerts)) + for _, alert := range amAlerts { + state := mapAlertmanagerState(alert.Status.State) + if state == "" { + continue + } + converted = append(converted, PrometheusAlert{ + Labels: alert.Labels, + Annotations: alert.Annotations, + State: state, + ActiveAt: alert.StartsAt, + }) + } + return converted, nil +} + +func (pa *prometheusAlerts) serviceReachable(ctx context.Context, namespace string, serviceName string, port int32, path string) bool { + healthCtx, cancel := context.WithTimeout(ctx, serviceHealthTimeout) + defer cancel() + + _, err := pa.getServiceResponse(healthCtx, namespace, serviceName, port, path) + return err == nil +} + +func (pa *prometheusAlerts) getServiceResponse(ctx context.Context, namespace string, serviceName string, port int32, path string) ([]byte, error) { + baseURL := fmt.Sprintf("https://%s.%s.svc:%d", serviceName, namespace, port) + requestURL := fmt.Sprintf("%s%s", baseURL, path) + + client, err := pa.createHTTPClient() + if err != nil { + return nil, err + } + + return pa.executeRequest(ctx, client, requestURL) +} + +func (pa *prometheusAlerts) thanosTenancyReachable(ctx context.Context, path string) bool { + namespaces := pa.userRuleNamespaces(ctx) + if len(namespaces) == 0 { + return false + } + + limit := maxTenancyProbeTargets + if limit <= 0 || limit > len(namespaces) { + limit = len(namespaces) + } + + for i := 0; i < limit; i++ { + healthCtx, cancel := context.WithTimeout(ctx, serviceHealthTimeout) + _, err := pa.getThanosTenancyResponse(healthCtx, path, namespaces[i]) + cancel() + + if err == nil { + return true + } + if isTenancyExpectedError(err) { + continue + } + return false + } + + return false +} + +// isTenancyExpectedError returns true for errors that are expected when probing +// Thanos tenancy endpoints across user namespaces — e.g. the namespace has no +// rules (404), the SA lacks access (401/403), or the namespace is not yet +// instrumented. These are skipped; only a network/server error aborts the probe. +func isTenancyExpectedError(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "status 401") || + strings.Contains(msg, "status 403") || + strings.Contains(msg, "status 404") || + strings.Contains(msg, "unauthorized") || + strings.Contains(msg, "forbidden") || + strings.Contains(msg, "not found") +} + +func (pa *prometheusAlerts) getAlertsViaProxy(ctx context.Context, namespace string, routeName string, source string) ([]PrometheusAlert, error) { + raw, err := pa.getPrometheusResponse(ctx, namespace, routeName, PrometheusAlertsPath) + if err != nil { + return nil, err + } + + var alertsResp prometheusAlertsResponse + if err := json.Unmarshal(raw, &alertsResp); err != nil { + return nil, fmt.Errorf("decode prometheus response: %w", err) + } + + if alertsResp.Status != "success" { + return nil, fmt.Errorf("prometheus API returned non-success status: %s", alertsResp.Status) + } + + applyAlertMetadata(alertsResp.Data.Alerts, source, AlertBackendProm) + return alertsResp.Data.Alerts, nil +} + +func (pa *prometheusAlerts) getAlertsViaThanosTenancy(ctx context.Context, namespace string, source string) ([]PrometheusAlert, error) { + raw, err := pa.getThanosTenancyResponse(ctx, ThanosQuerierTenancyAlertsPath, namespace) + if err != nil { + return nil, err + } + + var alertsResp prometheusAlertsResponse + if err := json.Unmarshal(raw, &alertsResp); err != nil { + return nil, fmt.Errorf("decode thanos response: %w", err) + } + + if alertsResp.Status != "success" { + return nil, fmt.Errorf("thanos API returned non-success status: %s", alertsResp.Status) + } + + applyAlertMetadata(alertsResp.Data.Alerts, source, AlertBackendThanos) + return alertsResp.Data.Alerts, nil +} + +func (pa *prometheusAlerts) getAlertmanagerAlerts(ctx context.Context, namespace string, routeName string, source string) ([]PrometheusAlert, error) { + raw, err := pa.getPrometheusResponse(ctx, namespace, routeName, AlertmanagerAlertsPath) + if err != nil { + return nil, err + } + + converted, err := parseAlertmanagerResponse(raw) + if err != nil { + return nil, err + } + + applyAlertMetadata(converted, source, AlertBackendAM) + if len(converted) == 0 { + return []PrometheusAlert{}, nil + } + return converted, nil +} + +func (pa *prometheusAlerts) getUserWorkloadRules(ctx context.Context, req GetRulesRequest) ([]PrometheusRuleGroup, error) { + namespace := namespaceFromLabels(req.Labels) + if namespace != "" { + rules, err := pa.getRulesViaThanosTenancy(ctx, namespace, AlertSourceUser) + if err == nil { + return rules, nil + } + prometheusLog.Warnf("failed to get user workload rules via thanos tenancy: %v", err) + } + + userNamespaces := pa.userRuleNamespaces(ctx) + if len(userNamespaces) > 0 { + groups, err := pa.getRulesViaThanosTenancyNamespaces(ctx, userNamespaces, AlertSourceUser) + if err == nil { + return groups, nil + } + prometheusLog.Warnf("failed to get user workload rules via thanos tenancy namespaces: %v", err) + } + + return pa.getRulesViaProxy(ctx, UserWorkloadMonitoringNamespace, UserWorkloadRouteName, AlertSourceUser) +} + +func (pa *prometheusAlerts) userRuleNamespaces(ctx context.Context) []string { + if cached, ok := pa.nsCache.get(); ok { + return cached + } + + if pa.ruleManager == nil { + namespaces := pa.allNonPlatformNamespaces(ctx) + pa.nsCache.set(namespaces) + return namespaces + } + + prometheusRules, err := pa.ruleManager.List() + if err != nil { + prometheusLog.WithError(err).Warn("failed to list PrometheusRules for user namespace discovery") + namespaces := pa.allNonPlatformNamespaces(ctx) + pa.nsCache.set(namespaces) + return namespaces + } + + namespaces := map[string]struct{}{} + for _, pr := range prometheusRules { + if pr.Namespace == "" { + continue + } + if pr.Namespace == ClusterMonitoringNamespace || pr.Namespace == UserWorkloadMonitoringNamespace { + continue + } + namespaces[pr.Namespace] = struct{}{} + } + + out := make([]string, 0, len(namespaces)) + for ns := range namespaces { + out = append(out, ns) + } + pa.nsCache.set(out) + return out +} + +func (pa *prometheusAlerts) allNonPlatformNamespaces(ctx context.Context) []string { + if pa.coreClient == nil { + return nil + } + + namespaceList, err := pa.coreClient.Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + prometheusLog.WithError(err).Warn("failed to list namespaces for user namespace discovery") + return nil + } + + out := make([]string, 0, len(namespaceList.Items)) + for _, ns := range namespaceList.Items { + if ns.Name == ClusterMonitoringNamespace || ns.Name == UserWorkloadMonitoringNamespace { + continue + } + out = append(out, ns.Name) + } + return out +} + +// fanOutThanosTenancy calls fetch for each namespace, accumulates results, and +// returns combined results (or the last error if nothing succeeded). +func fanOutThanosTenancy[T any](namespaces []string, fetch func(string) ([]T, error)) ([]T, error) { + var out []T + var lastErr error + for _, namespace := range namespaces { + results, err := fetch(namespace) + if err != nil { + lastErr = err + continue + } + out = append(out, results...) + } + if len(out) > 0 { + return out, nil + } + return out, lastErr +} + +func (pa *prometheusAlerts) getAlertsViaThanosTenancyNamespaces(ctx context.Context, namespaces []string, source string) ([]PrometheusAlert, error) { + return fanOutThanosTenancy(namespaces, func(ns string) ([]PrometheusAlert, error) { + return pa.getAlertsViaThanosTenancy(ctx, ns, source) + }) +} + +func (pa *prometheusAlerts) getRulesViaThanosTenancyNamespaces(ctx context.Context, namespaces []string, source string) ([]PrometheusRuleGroup, error) { + return fanOutThanosTenancy(namespaces, func(ns string) ([]PrometheusRuleGroup, error) { + return pa.getRulesViaThanosTenancy(ctx, ns, source) + }) +} + +func (pa *prometheusAlerts) getRulesViaProxy(ctx context.Context, namespace string, routeName string, source string) ([]PrometheusRuleGroup, error) { + raw, err := pa.getPrometheusResponse(ctx, namespace, routeName, PrometheusRulesPath) + if err != nil { + return nil, err + } + + var rulesResp prometheusRulesResponse + if err := json.Unmarshal(raw, &rulesResp); err != nil { + return nil, fmt.Errorf("decode prometheus response: %w", err) + } + + if rulesResp.Status != "success" { + return nil, fmt.Errorf("prometheus API returned non-success status: %s", rulesResp.Status) + } + + applyRuleSource(rulesResp.Data.Groups, source) + return rulesResp.Data.Groups, nil +} + +func (pa *prometheusAlerts) getRulesViaThanosTenancy(ctx context.Context, namespace string, source string) ([]PrometheusRuleGroup, error) { + raw, err := pa.getThanosTenancyResponse(ctx, ThanosQuerierTenancyRulesPath, namespace) + if err != nil { + return nil, err + } + + var rulesResp prometheusRulesResponse + if err := json.Unmarshal(raw, &rulesResp); err != nil { + return nil, fmt.Errorf("decode thanos response: %w", err) + } + + if rulesResp.Status != "success" { + return nil, fmt.Errorf("thanos API returned non-success status: %s", rulesResp.Status) + } + + applyRuleSource(rulesResp.Data.Groups, source) + return rulesResp.Data.Groups, nil +} + +func (pa *prometheusAlerts) getPrometheusResponse(ctx context.Context, namespace string, routeName string, path string) ([]byte, error) { + url, err := pa.buildPrometheusURL(ctx, namespace, routeName, path) + if err != nil { + return nil, err + } + client, err := pa.createHTTPClient() + if err != nil { + return nil, err + } + + return pa.executeRequest(ctx, client, url) +} + +func (pa *prometheusAlerts) getThanosTenancyResponse(ctx context.Context, path string, namespace string) ([]byte, error) { + if namespace == "" { + return nil, fmt.Errorf("namespace is required for thanos tenancy requests") + } + + baseURL := fmt.Sprintf("https://%s.%s.svc:%d", ThanosQuerierServiceName, ClusterMonitoringNamespace, DefaultThanosQuerierTenancyRulesPort) + requestURL := fmt.Sprintf("%s%s?namespace=%s", baseURL, path, url.QueryEscape(namespace)) + + client, err := pa.createHTTPClient() + if err != nil { + return nil, err + } + + return pa.executeRequest(ctx, client, requestURL) +} + +func (pa *prometheusAlerts) buildPrometheusURL(ctx context.Context, namespace string, routeName string, path string) (string, error) { + route, err := pa.fetchPrometheusRoute(ctx, namespace, routeName) + if err != nil { + return "", err + } + + return buildRouteURL(route.Spec.Host, route.Spec.Path, path), nil +} + +func (pa *prometheusAlerts) fetchPrometheusRoute(ctx context.Context, namespace string, routeName string) (*routev1.Route, error) { + if pa.routeClient == nil { + return nil, fmt.Errorf("route client is not configured") + } + + route, err := pa.routeClient.RouteV1().Routes(namespace).Get(ctx, routeName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get prometheus route: %w", err) + } + + return route, nil +} + +func applyAlertMetadata(alerts []PrometheusAlert, source, backend string) { + for i := range alerts { + if alerts[i].Labels == nil { + alerts[i].Labels = map[string]string{} + } + alerts[i].Labels[AlertSourceLabel] = source + alerts[i].Labels[AlertBackendLabel] = backend + } +} + +func applyRuleSource(groups []PrometheusRuleGroup, source string) { + for gi := range groups { + for ri := range groups[gi].Rules { + rule := &groups[gi].Rules[ri] + if rule.Labels == nil { + rule.Labels = map[string]string{} + } + rule.Labels[AlertSourceLabel] = source + for ai := range rule.Alerts { + if rule.Alerts[ai].Labels == nil { + rule.Alerts[ai].Labels = map[string]string{} + } + rule.Alerts[ai].Labels[AlertSourceLabel] = source + } + } + } +} + +func filterAlertsByState(alerts []PrometheusAlert, state string) []PrometheusAlert { + out := make([]PrometheusAlert, 0, len(alerts)) + for _, alert := range alerts { + if alert.State == state { + out = append(out, alert) + } + } + return out +} + +func mapAlertmanagerState(state string) string { + if state == "active" { + return "firing" + } + if state == "suppressed" { + return "silenced" + } + return "" +} + +func buildRouteURL(host string, routePath string, requestPath string) string { + basePath := strings.TrimSuffix(routePath, "/") + if basePath == "" { + return fmt.Sprintf("https://%s%s", host, requestPath) + } + if requestPath == basePath || strings.HasPrefix(requestPath, basePath+"/") { + return fmt.Sprintf("https://%s%s", host, requestPath) + } + return fmt.Sprintf("https://%s%s%s", host, basePath, requestPath) +} + +func namespaceFromLabels(labels map[string]string) string { + if labels == nil { + return "" + } + return strings.TrimSpace(labels["namespace"]) +} + +func (pa *prometheusAlerts) createHTTPClient() (*http.Client, error) { + tlsConfig, err := pa.buildTLSConfig() + if err != nil { + return nil, err + } + + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + }, nil +} + +func (pa *prometheusAlerts) buildTLSConfig() (*tls.Config, error) { + caCertPool, err := pa.loadCACertPool() + if err != nil { + return nil, err + } + + return &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: caCertPool, + }, nil +} + +func (pa *prometheusAlerts) loadCACertPool() (*x509.CertPool, error) { + caCertPool, err := x509.SystemCertPool() + if err != nil { + caCertPool = x509.NewCertPool() + } + + if len(pa.config.CAData) > 0 { + caCertPool.AppendCertsFromPEM(pa.config.CAData) + return caCertPool, nil + } + + if pa.config.CAFile != "" { + caCert, err := os.ReadFile(pa.config.CAFile) + if err != nil { + return nil, fmt.Errorf("read CA cert file: %w", err) + } + caCertPool.AppendCertsFromPEM(caCert) + } + + // OpenShift service CA bundle for in-cluster service certs. + if serviceCA, err := os.ReadFile(ServiceCAPath); err == nil { + caCertPool.AppendCertsFromPEM(serviceCA) + } + + return caCertPool, nil +} + +func copyStringSlice(in []string) []string { + if len(in) == 0 { + return []string{} + } + + out := make([]string, len(in)) + copy(out, in) + return out +} + +func (pa *prometheusAlerts) executeRequest(ctx context.Context, client *http.Client, url string) ([]byte, error) { + req, err := pa.createAuthenticatedRequest(ctx, url) + if err != nil { + return nil, err + } + + return pa.performRequest(client, req) +} + +func (pa *prometheusAlerts) createAuthenticatedRequest(ctx context.Context, url string) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + token := BearerTokenFromContext(ctx) + if token == "" { + var err error + token, err = pa.loadBearerToken() + if err != nil { + return nil, err + } + } + + req.Header.Set("Authorization", "Bearer "+token) + return req, nil +} + +func (pa *prometheusAlerts) loadBearerToken() (string, error) { + if pa.config.BearerToken != "" { + return pa.config.BearerToken, nil + } + + if pa.config.BearerTokenFile == "" { + return "", fmt.Errorf("no bearer token or token file configured") + } + + tokenBytes, err := os.ReadFile(pa.config.BearerTokenFile) + if err != nil { + return "", fmt.Errorf("load bearer token file: %w", err) + } + + return strings.TrimSpace(string(tokenBytes)), nil +} + +func (pa *prometheusAlerts) performRequest(client *http.Client, req *http.Request) ([]byte, error) { + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("execute request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) + } + + return body, nil +} + +func labelsMatch(req *GetAlertsRequest, alert *PrometheusAlert) bool { + for key, value := range req.Labels { + if alertValue, exists := alert.Labels[key]; !exists || alertValue != value { + return false + } + } + + return true +} diff --git a/pkg/k8s/relabeled_rules.go b/pkg/k8s/relabeled_rules.go index 02452c385..a853630ea 100644 --- a/pkg/k8s/relabeled_rules.go +++ b/pkg/k8s/relabeled_rules.go @@ -9,7 +9,6 @@ import ( "sync" "time" - osmv1 "github.com/openshift/api/monitoring/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" monitoringv1client "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" "github.com/prometheus/common/model" @@ -260,11 +259,6 @@ func (rrm *relabeledRulesManager) collectAlerts(ctx context.Context, relabelConf alerts := make(map[string]monitoringv1.Rule) seenIDs := make(map[string]struct{}) - // Fetch all ARCs once from the informer cache (O(1) per-rule lookup below). - // This avoids O(n) live API server calls inside the per-rule loop that would - // cause exponential rate-limit backoff and stale cache data for new rules. - arcByName := rrm.arcsByName(ctx) - for _, obj := range rrm.prometheusRulesInformer.GetStore().List() { promRule, ok := obj.(*monitoringv1.PrometheusRule) if !ok { @@ -322,7 +316,7 @@ func (rrm *relabeledRulesManager) collectAlerts(ctx context.Context, relabelConf rule.Labels[managementlabels.AlertingRuleLabelName] = arName } - ruleManagedBy, relabelConfigManagedBy := rrm.determineManagedBy(promRule, alertRuleId, arcByName) + ruleManagedBy, relabelConfigManagedBy := rrm.determineManagedBy(ctx, promRule, alertRuleId) if ruleManagedBy != "" { rule.Labels[managementlabels.RuleManagedByLabel] = ruleManagedBy } @@ -390,28 +384,8 @@ func shortHash(id string, n int) string { return full[:n] } -// arcsByName builds a namespace/name → ARC map from the informer cache. -// Called once per sync cycle so that determineManagedBy can do O(1) lookups -// instead of one live API call per rule. -func (rrm *relabeledRulesManager) arcsByName(ctx context.Context) map[string]*osmv1.AlertRelabelConfig { - if rrm.alertRelabelConfigs == nil { - return nil - } - arcs, err := rrm.alertRelabelConfigs.List(ctx, "") - if err != nil { - log.Errorf("arcsByName: failed to list ARCs from cache: %v", err) - return nil - } - m := make(map[string]*osmv1.AlertRelabelConfig, len(arcs)) - for i := range arcs { - key := arcs[i].Namespace + "/" + arcs[i].Name - m[key] = &arcs[i] - } - return m -} - // determineManagedBy determines the openshift_io_rule_managed_by and openshift_io_relabel_config_managed_by label values -func (rrm *relabeledRulesManager) determineManagedBy(promRule *monitoringv1.PrometheusRule, alertRuleId string, arcByName map[string]*osmv1.AlertRelabelConfig) (string, string) { +func (rrm *relabeledRulesManager) determineManagedBy(ctx context.Context, promRule *monitoringv1.PrometheusRule, alertRuleId string) (string, string) { // Determine ruleManagedBy from PrometheusRule var ruleManagedBy string // If generated by AlertingRule CRD, do not mark as operator-managed; treat as user-via-platform @@ -425,14 +399,13 @@ func (rrm *relabeledRulesManager) determineManagedBy(promRule *monitoringv1.Prom } } - // Determine relabelConfigManagedBy only for platform rules using the - // pre-fetched cache map; no live API call is made here. + // Determine relabelConfigManagedBy only for platform rules isPlatform := rrm.namespaceManager.IsClusterMonitoringNamespace(promRule.Namespace) var relabelConfigManagedBy string - if isPlatform && arcByName != nil { + if isPlatform && rrm.alertRelabelConfigs != nil { arcName := GetAlertRelabelConfigName(promRule.Name, alertRuleId) - key := promRule.Namespace + "/" + arcName - if arc, found := arcByName[key]; found { + arc, found, err := rrm.alertRelabelConfigs.Get(ctx, promRule.Namespace, arcName) + if err == nil && found { if IsManagedByGitOps(arc.Annotations, arc.Labels) { relabelConfigManagedBy = managementlabels.ManagedByGitOps } @@ -442,27 +415,13 @@ func (rrm *relabeledRulesManager) determineManagedBy(promRule *monitoringv1.Prom return ruleManagedBy, relabelConfigManagedBy } -// DetermineManagedBy determines the managed-by labels for a single PrometheusRule -// alert rule. Callers that have a user-scoped context (e.g. tests) can pass a -// live AlertRelabelConfigInterface; a targeted Get is performed for that one rule. +// DetermineManagedBy determines the managed-by labels for a PrometheusRule alert rule. func DetermineManagedBy(ctx context.Context, alertRelabelConfigs AlertRelabelConfigInterface, namespaceManager NamespaceInterface, promRule *monitoringv1.PrometheusRule, alertRuleId string) (string, string) { - // Single-rule path: fetch only the specific ARC with RBAC enforcement on the - // caller's context, then build a one-entry map for determineManagedBy. - var arcByName map[string]*osmv1.AlertRelabelConfig - if alertRelabelConfigs != nil && namespaceManager.IsClusterMonitoringNamespace(promRule.Namespace) { - arcName := GetAlertRelabelConfigName(promRule.Name, alertRuleId) - arc, found, err := alertRelabelConfigs.Get(ctx, promRule.Namespace, arcName) - if err == nil && found { - arcByName = map[string]*osmv1.AlertRelabelConfig{ - promRule.Namespace + "/" + arcName: arc, - } - } - } rrm := &relabeledRulesManager{ alertRelabelConfigs: alertRelabelConfigs, namespaceManager: namespaceManager, } - return rrm.determineManagedBy(promRule, alertRuleId, arcByName) + return rrm.determineManagedBy(ctx, promRule, alertRuleId) } func (rrm *relabeledRulesManager) List(ctx context.Context) []monitoringv1.Rule { diff --git a/pkg/k8s/relabeled_rules_test.go b/pkg/k8s/relabeled_rules_test.go deleted file mode 100644 index 1d10ef48c..000000000 --- a/pkg/k8s/relabeled_rules_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package k8s - -import ( - "context" - "testing" - - osmv1 "github.com/openshift/api/monitoring/v1" - monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/openshift/monitoring-plugin/pkg/managementlabels" -) - -// arcGetPanicInterface implements AlertRelabelConfigInterface and panics if -// Get is called. It is used to verify that the sync path never calls Get. -type arcGetPanicInterface struct { - arcs []osmv1.AlertRelabelConfig -} - -func (m *arcGetPanicInterface) List(_ context.Context, namespace string) ([]osmv1.AlertRelabelConfig, error) { - if namespace == "" { - return m.arcs, nil - } - var filtered []osmv1.AlertRelabelConfig - for _, a := range m.arcs { - if a.Namespace == namespace { - filtered = append(filtered, a) - } - } - return filtered, nil -} - -func (m *arcGetPanicInterface) Get(_ context.Context, _, _ string) (*osmv1.AlertRelabelConfig, bool, error) { - panic("Get must not be called during sync; use the arcByName cache map instead") -} - -func (m *arcGetPanicInterface) Create(_ context.Context, arc osmv1.AlertRelabelConfig) (*osmv1.AlertRelabelConfig, error) { - return &arc, nil -} - -func (m *arcGetPanicInterface) Update(_ context.Context, _ osmv1.AlertRelabelConfig) error { - return nil -} - -func (m *arcGetPanicInterface) Delete(_ context.Context, _, _ string) error { - return nil -} - -// stubNamespaceManager implements NamespaceInterface for tests. -type stubNamespaceManager struct { - platformNamespaces map[string]bool -} - -func (s *stubNamespaceManager) IsClusterMonitoringNamespace(name string) bool { - return s.platformNamespaces[name] -} - -// TestDetermineManagedBy_NeverCallsGet verifies that determineManagedBy -// uses the pre-fetched arcByName map and never issues a live Get call, -// even for platform-namespace rules with a matching ARC. -func TestDetermineManagedBy_NeverCallsGet(t *testing.T) { - const ( - namespace = "openshift-monitoring" - promRuleName = "test-rule" - alertRuleID = "abc123" - ) - - arcName := GetAlertRelabelConfigName(promRuleName, alertRuleID) - arc := osmv1.AlertRelabelConfig{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: arcName, - Annotations: map[string]string{ - "argocd.argoproj.io/managed-by": "some-app", - }, - }, - } - - rrm := &relabeledRulesManager{ - // arcGetPanicInterface panics if Get is called — this is the guard. - alertRelabelConfigs: &arcGetPanicInterface{arcs: []osmv1.AlertRelabelConfig{arc}}, - namespaceManager: &stubNamespaceManager{ - platformNamespaces: map[string]bool{namespace: true}, - }, - } - - promRule := &monitoringv1.PrometheusRule{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: promRuleName, - }, - } - - // Build arcByName from List (no Get call). - arcByName := rrm.arcsByName(context.Background()) - - // This must not panic (i.e. must not call Get). - ruleManagedBy, relabelConfigManagedBy := rrm.determineManagedBy(promRule, alertRuleID, arcByName) - - if ruleManagedBy != "" { - t.Errorf("expected empty ruleManagedBy, got %q", ruleManagedBy) - } - if relabelConfigManagedBy != managementlabels.ManagedByGitOps { - t.Errorf("expected relabelConfigManagedBy=%q, got %q", managementlabels.ManagedByGitOps, relabelConfigManagedBy) - } -} - -// TestDetermineManagedBy_NoARCMatch verifies that a platform rule with no -// matching ARC in the cache produces empty relabelConfigManagedBy. -func TestDetermineManagedBy_NoARCMatch(t *testing.T) { - const namespace = "openshift-monitoring" - - rrm := &relabeledRulesManager{ - alertRelabelConfigs: &arcGetPanicInterface{arcs: nil}, - namespaceManager: &stubNamespaceManager{ - platformNamespaces: map[string]bool{namespace: true}, - }, - } - - promRule := &monitoringv1.PrometheusRule{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "some-rule", - }, - } - - arcByName := rrm.arcsByName(context.Background()) - _, relabelConfigManagedBy := rrm.determineManagedBy(promRule, "no-match-id", arcByName) - - if relabelConfigManagedBy != "" { - t.Errorf("expected empty relabelConfigManagedBy for no ARC match, got %q", relabelConfigManagedBy) - } -} - -// TestDetermineManagedBy_NonPlatformRuleSkipsARCLookup verifies that a -// user-workload rule (non-platform namespace) does not consult ARCs at all. -func TestDetermineManagedBy_NonPlatformRuleSkipsARCLookup(t *testing.T) { - rrm := &relabeledRulesManager{ - // Non-nil but panics on Get — confirms no lookup occurs. - alertRelabelConfigs: &arcGetPanicInterface{arcs: nil}, - namespaceManager: &stubNamespaceManager{platformNamespaces: map[string]bool{}}, - } - - promRule := &monitoringv1.PrometheusRule{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "user-namespace", - Name: "user-rule", - }, - } - - arcByName := rrm.arcsByName(context.Background()) - _, relabelConfigManagedBy := rrm.determineManagedBy(promRule, "some-id", arcByName) - - if relabelConfigManagedBy != "" { - t.Errorf("expected empty relabelConfigManagedBy for non-platform rule, got %q", relabelConfigManagedBy) - } -} diff --git a/pkg/k8s/rule_label_matchers.go b/pkg/k8s/rule_label_matchers.go new file mode 100644 index 000000000..cf8eb1f51 --- /dev/null +++ b/pkg/k8s/rule_label_matchers.go @@ -0,0 +1,91 @@ +package k8s + +import ( + "fmt" + "strings" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/promql/parser" +) + +const namespaceLabelKey = "namespace" + +func compileRuleLabelMatchers(req GetRulesRequest) ([]*labels.Matcher, error) { + var out []*labels.Matcher + + for k, v := range req.Labels { + if strings.TrimSpace(k) == "" { + continue + } + if k == namespaceLabelKey { + continue + } + m, err := labels.NewMatcher(labels.MatchEqual, k, v) + if err != nil { + return nil, fmt.Errorf("invalid label matcher %q=%q: %w", k, v, err) + } + out = append(out, m) + } + + for _, raw := range req.Matchers { + sel := strings.TrimSpace(raw) + if sel == "" { + continue + } + if !strings.HasPrefix(sel, "{") || !strings.HasSuffix(sel, "}") { + sel = "{" + sel + "}" + } + matchers, err := parser.ParseMetricSelector(sel) + if err != nil { + return nil, fmt.Errorf("invalid matcher %q: %w", raw, err) + } + out = append(out, matchers...) + } + + return out, nil +} + +func filterRuleGroupsByLabelMatchers(groups []PrometheusRuleGroup, matchers []*labels.Matcher) []PrometheusRuleGroup { + if len(matchers) == 0 || len(groups) == 0 { + return groups + } + + out := make([]PrometheusRuleGroup, 0, len(groups)) + for _, g := range groups { + kept := make([]PrometheusRule, 0, len(g.Rules)) + for _, r := range g.Rules { + if ruleMatchesLabelMatchers(r, matchers) { + kept = append(kept, r) + } + } + if len(kept) == 0 { + continue + } + g.Rules = kept + out = append(out, g) + } + + return out +} + +func ruleMatchesLabelMatchers(rule PrometheusRule, matchers []*labels.Matcher) bool { + if len(matchers) == 0 { + return true + } + + for _, m := range matchers { + val, ok := rule.Labels[m.Name] + if !ok { + // Prometheus semantics: negative matchers match missing labels. + if m.Type == labels.MatchNotEqual || m.Type == labels.MatchNotRegexp { + continue + } + return false + } + if !m.Matches(val) { + return false + } + } + + return true +} diff --git a/pkg/k8s/rule_label_matchers_test.go b/pkg/k8s/rule_label_matchers_test.go new file mode 100644 index 000000000..34169eaa7 --- /dev/null +++ b/pkg/k8s/rule_label_matchers_test.go @@ -0,0 +1,58 @@ +package k8s + +import "testing" + +func TestCompileRuleLabelMatchers_IgnoresNamespaceLabel(t *testing.T) { + matchers, err := compileRuleLabelMatchers(GetRulesRequest{ + Labels: map[string]string{ + "namespace": "ns-a", + "severity": "critical", + }, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if len(matchers) != 1 { + t.Fatalf("expected 1 matcher (severity), got %d", len(matchers)) + } + if matchers[0].Name != "severity" { + t.Fatalf("expected matcher for severity, got %q", matchers[0].Name) + } +} + +func TestRuleMatchesLabelMatchers_PrometheusMissingLabelSemantics(t *testing.T) { + neg, err := compileRuleLabelMatchers(GetRulesRequest{ + Matchers: []string{`missing!="x"`}, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if !ruleMatchesLabelMatchers(PrometheusRule{Labels: map[string]string{}}, neg) { + t.Fatalf("expected negative matcher to match missing label") + } + + pos, err := compileRuleLabelMatchers(GetRulesRequest{ + Matchers: []string{`missing="x"`}, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if ruleMatchesLabelMatchers(PrometheusRule{Labels: map[string]string{}}, pos) { + t.Fatalf("expected positive matcher not to match missing label") + } +} + +func TestCompileRuleLabelMatchers_AcceptsSelectorBody(t *testing.T) { + matchers, err := compileRuleLabelMatchers(GetRulesRequest{ + Matchers: []string{`severity=~"warning|critical"`}, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if len(matchers) != 1 { + t.Fatalf("expected 1 matcher, got %d", len(matchers)) + } + if matchers[0].Name != "severity" { + t.Fatalf("expected severity matcher, got %q", matchers[0].Name) + } +} diff --git a/pkg/k8s/types.go b/pkg/k8s/types.go index 102d5fccf..bf7b61b5b 100644 --- a/pkg/k8s/types.go +++ b/pkg/k8s/types.go @@ -21,6 +21,12 @@ type Client interface { // TestConnection tests the connection to the Kubernetes cluster TestConnection(ctx context.Context) error + // AlertingHealth returns alerting route and stack health details + AlertingHealth(ctx context.Context) (AlertingHealth, error) + + // PrometheusAlerts retrieves active Prometheus alerts + PrometheusAlerts() PrometheusAlertsInterface + // PrometheusRules returns the PrometheusRule interface PrometheusRules() PrometheusRuleInterface @@ -37,6 +43,14 @@ type Client interface { Namespace() NamespaceInterface } +// PrometheusAlertsInterface defines operations for managing PrometheusAlerts +type PrometheusAlertsInterface interface { + // GetAlerts retrieves Prometheus alerts with optional state filtering + GetAlerts(ctx context.Context, req GetAlertsRequest) ([]PrometheusAlert, error) + // GetRules retrieves Prometheus alerting rules and active alerts + GetRules(ctx context.Context, req GetRulesRequest) ([]PrometheusRuleGroup, error) +} + // PrometheusRuleInterface defines operations for managing PrometheusRules type PrometheusRuleInterface interface { // List lists all PrometheusRules from the informer cache @@ -104,6 +118,37 @@ type RelabeledRulesInterface interface { Config() []*relabel.Config } +// RouteStatus describes the availability state of a monitoring route. +type RouteStatus string + +const ( + RouteNotFound RouteStatus = "notFound" + RouteUnreachable RouteStatus = "unreachable" + RouteReachable RouteStatus = "reachable" +) + +// AlertingRouteHealth describes route availability and reachability. +type AlertingRouteHealth struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + Status RouteStatus `json:"status"` + FallbackReachable bool `json:"fallbackReachable,omitempty"` + Error string `json:"error,omitempty"` +} + +// AlertingStackHealth describes alerting health for a monitoring stack. +type AlertingStackHealth struct { + Prometheus AlertingRouteHealth `json:"prometheus"` + Alertmanager AlertingRouteHealth `json:"alertmanager"` +} + +// AlertingHealth provides alerting health details for platform and user workload stacks. +type AlertingHealth struct { + Platform *AlertingStackHealth `json:"platform"` + UserWorkloadEnabled bool `json:"userWorkloadEnabled"` + UserWorkload *AlertingStackHealth `json:"userWorkload"` +} + // NamespaceInterface defines operations for Namespaces type NamespaceInterface interface { // IsClusterMonitoringNamespace checks if a namespace has the openshift.io/cluster-monitoring=true label diff --git a/pkg/management/get_alerting_health.go b/pkg/management/get_alerting_health.go new file mode 100644 index 000000000..001d13f15 --- /dev/null +++ b/pkg/management/get_alerting_health.go @@ -0,0 +1,21 @@ +package management + +import ( + "context" + "time" + + "github.com/openshift/monitoring-plugin/pkg/k8s" +) + +const alertingHealthTimeout = 10 * time.Second + +// GetAlertingHealth retrieves alerting health details. +func (c *client) GetAlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) { + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + timeoutCtx, cancel := context.WithTimeout(ctx, alertingHealthTimeout) + defer cancel() + ctx = timeoutCtx + } + + return c.k8sClient.AlertingHealth(ctx) +} diff --git a/pkg/management/get_alerts.go b/pkg/management/get_alerts.go new file mode 100644 index 000000000..f7495227a --- /dev/null +++ b/pkg/management/get_alerts.go @@ -0,0 +1,308 @@ +package management + +import ( + "context" + "fmt" + "strings" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/relabel" + "k8s.io/apimachinery/pkg/types" + + alertrule "github.com/openshift/monitoring-plugin/pkg/alert_rule" + "github.com/openshift/monitoring-plugin/pkg/alertcomponent" + "github.com/openshift/monitoring-plugin/pkg/classification" + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +var cvoAlertNames = map[string]struct{}{ + "ClusterOperatorDown": {}, + "ClusterOperatorDegraded": {}, +} + +func (c *client) GetAlerts(ctx context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + alerts, err := c.k8sClient.PrometheusAlerts().GetAlerts(ctx, req) + if err != nil { + return nil, fmt.Errorf("failed to get prometheus alerts: %w", err) + } + + configs := c.k8sClient.RelabeledRules().Config() + rules := c.k8sClient.RelabeledRules().List(ctx) + + result := make([]k8s.PrometheusAlert, 0, len(alerts)) + for _, alert := range alerts { + // Only apply relabel configs for platform alerts. User workload alerts + // already come from their own stack and should not be relabeled here. + if alert.Labels[k8s.AlertSourceLabel] != k8s.AlertSourceUser { + relabels, keep := relabel.Process(labels.FromMap(alert.Labels), configs...) + if !keep { + continue + } + alert.Labels = relabels.Map() + } + + // Add calculated rule ID and source when not present (labels enrichment) + c.setRuleIDAndSourceIfMissing(ctx, &alert, rules) + + // correlate alert -> base alert rule via subset matching against relabeled rules + alertRuleId := alert.Labels[k8s.AlertRuleLabelId] + component := "" + layer := "" + + bestRule, corrId := correlateAlertToRule(alert.Labels, rules) + if corrId != "" { + alertRuleId = corrId + } + if bestRule == nil && alertRuleId != "" { + if rule, ok := c.k8sClient.RelabeledRules().Get(ctx, alertRuleId); ok { + bestRule = &rule + } + } + + if bestRule != nil { + if src := c.deriveAlertSource(bestRule.Labels); src != "" { + alert.Labels[k8s.AlertSourceLabel] = src + } + component, layer = classifyFromRule(bestRule) + } else { + component, layer = classifyFromAlertLabels(alert.Labels) + } + + if cvoComponent, cvoLayer, ok := classifyCvoAlert(alert.Labels); ok { + component = cvoComponent + layer = cvoLayer + } + + // Dynamic classification: _from labels on the rule point to alert labels + // whose runtime values become the classification. Takes precedence over + // static classification labels. + if bestRule != nil { + component, layer = ApplyDynamicClassification(bestRule.Labels, alert.Labels, component, layer) + } + + // keep label and optional enriched fields consistent + if alert.Labels[k8s.AlertRuleLabelId] == "" && alertRuleId != "" { + alert.Labels[k8s.AlertRuleLabelId] = alertRuleId + } + alert.AlertRuleId = alertRuleId + + alert.AlertComponent = component + alert.AlertLayer = layer + + delete(alert.Labels, managementlabels.ClassificationManagedByKey) + + result = append(result, alert) + } + + return result, nil +} + +func (c *client) setRuleIDAndSourceIfMissing(ctx context.Context, alert *k8s.PrometheusAlert, rules []monitoringv1.Rule) { + if alert.Labels[k8s.AlertRuleLabelId] == "" { + for _, existing := range rules { + if existing.Alert != alert.Labels[managementlabels.AlertNameLabel] { + continue + } + if !ruleMatchesAlert(existing.Labels, alert.Labels) { + continue + } + rid := alertrule.GetAlertingRuleId(&existing) + alert.Labels[k8s.AlertRuleLabelId] = rid + if alert.Labels[k8s.AlertSourceLabel] == "" { + if src := c.deriveAlertSource(existing.Labels); src != "" { + alert.Labels[k8s.AlertSourceLabel] = src + } + } + break + } + } + if alert.Labels[k8s.AlertSourceLabel] != "" { + return + } + if rid := alert.Labels[k8s.AlertRuleLabelId]; rid != "" { + if existing, ok := c.k8sClient.RelabeledRules().Get(ctx, rid); ok { + if src := c.deriveAlertSource(existing.Labels); src != "" { + alert.Labels[k8s.AlertSourceLabel] = src + } + } + } +} + +func ruleMatchesAlert(existingRuleLabels, alertLabels map[string]string) bool { + existingBusiness := filterBusinessLabels(existingRuleLabels) + for k, v := range existingBusiness { + lv, ok := alertLabels[k] + if !ok || lv != v { + return false + } + } + return true +} + +// correlateAlertToRule tries to find the base alert rule for the given alert labels +// by subset-matching against relabeled rules. +func correlateAlertToRule(alertLabels map[string]string, rules []monitoringv1.Rule) (*monitoringv1.Rule, string) { + // Determine best match: prefer rules with more labels (more specific) + var ( + bestId string + bestRule *monitoringv1.Rule + bestLabelCount int + ) + for i := range rules { + rule := &rules[i] + ruleLabels := sanitizeRuleLabels(rule.Labels) + if isSubset(ruleLabels, alertLabels) { + if len(ruleLabels) > bestLabelCount { + bestLabelCount = len(ruleLabels) + bestRule = rule + bestId = rule.Labels[k8s.AlertRuleLabelId] + } + } + } + if bestRule == nil { + return nil, "" + } + return bestRule, bestId +} + +// sanitizeRuleLabels removes meta labels that will not be present on alerts +func sanitizeRuleLabels(in map[string]string) map[string]string { + out := make(map[string]string, len(in)) + for k, v := range in { + if k == k8s.PrometheusRuleLabelNamespace || k == k8s.PrometheusRuleLabelName || k == k8s.AlertRuleLabelId { + continue + } + out[k] = v + } + return out +} + +// isSubset returns true if all key/value pairs in sub are present in sup +func isSubset(sub map[string]string, sup map[string]string) bool { + for k, v := range sub { + if sv, ok := sup[k]; !ok || sv != v { + return false + } + } + return true +} + +func (c *client) deriveAlertSource(ruleLabels map[string]string) string { + ns := ruleLabels[k8s.PrometheusRuleLabelNamespace] + name := ruleLabels[k8s.PrometheusRuleLabelName] + if ns == "" || name == "" { + return "" + } + if c.isPlatformManagedPrometheusRule(types.NamespacedName{Namespace: ns, Name: name}) { + return k8s.AlertSourcePlatform + } + return k8s.AlertSourceUser +} + +func classifyFromRule(rule *monitoringv1.Rule) (string, string) { + lbls := model.LabelSet{} + for k, v := range rule.Labels { + lbls[model.LabelName(k)] = model.LabelValue(v) + } + if _, ok := lbls["namespace"]; !ok { + if ns := rule.Labels[k8s.PrometheusRuleLabelNamespace]; ns != "" { + lbls["namespace"] = model.LabelValue(ns) + } + } + if rule.Alert != "" { + lbls[model.LabelName(managementlabels.AlertNameLabel)] = model.LabelValue(rule.Alert) + } + + layer, component := alertcomponent.DetermineComponent(lbls) + if component == "" || component == "Others" { + component = "other" + layer = deriveLayerFromSource(rule.Labels) + } + + component, layer = applyRuleScopedDefaults(rule.Labels, component, layer) + return component, layer +} + +func classifyFromAlertLabels(alertLabels map[string]string) (string, string) { + lbls := model.LabelSet{} + for k, v := range alertLabels { + lbls[model.LabelName(k)] = model.LabelValue(v) + } + layer, component := alertcomponent.DetermineComponent(lbls) + if component == "" || component == "Others" { + component = "other" + layer = deriveLayerFromSource(alertLabels) + } + component, layer = applyRuleScopedDefaults(alertLabels, component, layer) + return component, layer +} + +func deriveLayerFromSource(labels map[string]string) string { + if labels[k8s.AlertSourceLabel] == k8s.AlertSourcePlatform { + return "cluster" + } + if labels[k8s.PrometheusRuleLabelNamespace] == k8s.ClusterMonitoringNamespace { + return "cluster" + } + promSrc := labels["prometheus"] + if strings.HasPrefix(promSrc, "openshift-monitoring/") { + return "cluster" + } + return "namespace" +} + +// applyRuleScopedDefaults applies static classification labels from the rule. +func applyRuleScopedDefaults(ruleLabels map[string]string, component, layer string) (string, string) { + if ruleLabels == nil { + return component, layer + } + if v := strings.TrimSpace(ruleLabels[k8s.AlertRuleClassificationComponentKey]); v != "" { + if classification.ValidateComponent(v) { + component = v + } + } + if v := strings.TrimSpace(ruleLabels[k8s.AlertRuleClassificationLayerKey]); v != "" { + if classification.ValidateLayer(v) { + layer = strings.ToLower(strings.TrimSpace(v)) + } + } + return component, layer +} + +// applyDynamicClassification handles _from labels: the rule label points to an +// alert label whose runtime value becomes the classification. _from takes +// precedence over static classification labels. +func ApplyDynamicClassification(ruleLabels, alertLabels map[string]string, component, layer string) (string, string) { + if ruleLabels == nil { + return component, layer + } + if from := strings.TrimSpace(ruleLabels[k8s.AlertRuleClassificationComponentFromKey]); from != "" { + if classification.ValidatePromLabelName(from) { + if v := strings.TrimSpace(alertLabels[from]); v != "" && classification.ValidateComponent(v) { + component = v + } + } + } + if from := strings.TrimSpace(ruleLabels[k8s.AlertRuleClassificationLayerFromKey]); from != "" { + if classification.ValidatePromLabelName(from) { + if v := strings.ToLower(strings.TrimSpace(alertLabels[from])); classification.ValidateLayer(v) { + layer = v + } + } + } + return component, layer +} + +func classifyCvoAlert(alertLabels map[string]string) (string, string, bool) { + if _, ok := cvoAlertNames[alertLabels[managementlabels.AlertNameLabel]]; !ok { + return "", "", false + } + component := alertLabels["name"] + if component == "" { + component = "version" + } + return component, "cluster", true +} diff --git a/pkg/management/get_alerts_test.go b/pkg/management/get_alerts_test.go new file mode 100644 index 000000000..66b0e1902 --- /dev/null +++ b/pkg/management/get_alerts_test.go @@ -0,0 +1,465 @@ +package management_test + +import ( + "context" + "errors" + "strings" + "testing" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus/prometheus/model/relabel" + + alertrule "github.com/openshift/monitoring-plugin/pkg/alert_rule" + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management" + "github.com/openshift/monitoring-plugin/pkg/management/testutils" + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +func TestGetAlerts_ErrorPropagated(t *testing.T) { + ctx := context.Background() + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return nil, errors.New("failed to get alerts") + }, + } + }, + } + client := management.New(ctx, mockK8s) + + _, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "failed to get prometheus alerts") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestGetAlerts_ReturnsAllWithoutRelabelConfigs(t *testing.T) { + ctx := context.Background() + alert1 := k8s.PrometheusAlert{ + Labels: map[string]string{managementlabels.AlertNameLabel: "Alert1", "severity": "warning", "namespace": "default"}, + State: "firing", + } + alert2 := k8s.PrometheusAlert{ + Labels: map[string]string{managementlabels.AlertNameLabel: "Alert2", "severity": "critical", "namespace": "kube-system"}, + State: "pending", + } + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alert1, alert2}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 2 { + t.Fatalf("expected 2 alerts, got %d", len(alerts)) + } + if alerts[0].Labels[managementlabels.AlertNameLabel] != "Alert1" { + t.Errorf("alert[0] name mismatch") + } + if alerts[1].Labels[managementlabels.AlertNameLabel] != "Alert2" { + t.Errorf("alert[1] name mismatch") + } +} + +func TestGetAlerts_AppliesStaticClassificationFromRelabeledRule(t *testing.T) { + ctx := context.Background() + alert1 := k8s.PrometheusAlert{ + Labels: map[string]string{managementlabels.AlertNameLabel: "Alert1", "severity": "warning", "namespace": "default"}, + State: "firing", + } + + rule := monitoringv1.Rule{ + Alert: "Alert1", + Labels: map[string]string{ + "severity": "warning", + "namespace": "default", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "test-rule", + k8s.AlertRuleClassificationComponentKey: "networking", + k8s.AlertRuleClassificationLayerKey: "cluster", + }, + } + rule.Labels[k8s.AlertRuleLabelId] = alertrule.GetAlertingRuleId(&rule) + + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alert1}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{rule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == rule.Labels[k8s.AlertRuleLabelId] { + return rule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + NamespaceFunc: func() k8s.NamespaceInterface { + ns := &testutils.MockNamespaceInterface{} + ns.SetMonitoringNamespaces(map[string]bool{"openshift-monitoring": true}) + return ns + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].AlertComponent != "networking" { + t.Errorf("expected component=networking, got %q", alerts[0].AlertComponent) + } + if alerts[0].AlertLayer != "cluster" { + t.Errorf("expected layer=cluster, got %q", alerts[0].AlertLayer) + } +} + +func TestGetAlerts_DerivesComponentFromAlertLabel(t *testing.T) { + ctx := context.Background() + alertWithName := k8s.PrometheusAlert{ + Labels: map[string]string{ + managementlabels.AlertNameLabel: "Alert1", + "severity": "warning", + "namespace": "default", + "name": "kube_apiserver", + }, + State: "firing", + } + + rule := monitoringv1.Rule{ + Alert: "Alert1", + Labels: map[string]string{ + "severity": "warning", + "namespace": "default", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "test-rule", + k8s.AlertRuleClassificationComponentFromKey: "name", + k8s.AlertRuleClassificationLayerKey: "namespace", + }, + } + rule.Labels[k8s.AlertRuleLabelId] = alertrule.GetAlertingRuleId(&rule) + + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alertWithName}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{rule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == rule.Labels[k8s.AlertRuleLabelId] { + return rule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + NamespaceFunc: func() k8s.NamespaceInterface { + ns := &testutils.MockNamespaceInterface{} + ns.SetMonitoringNamespaces(map[string]bool{"openshift-monitoring": true}) + return ns + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].AlertComponent != "kube_apiserver" { + t.Errorf("expected component=kube_apiserver, got %q", alerts[0].AlertComponent) + } + if alerts[0].AlertLayer != "namespace" { + t.Errorf("expected layer=namespace, got %q", alerts[0].AlertLayer) + } +} + +func TestGetAlerts_DerivesLayerFromAlertLabel(t *testing.T) { + ctx := context.Background() + alertWithLayer := k8s.PrometheusAlert{ + Labels: map[string]string{ + managementlabels.AlertNameLabel: "Alert1", + "severity": "warning", + "namespace": "default", + "tier": "Cluster", + }, + State: "firing", + } + + rule := monitoringv1.Rule{ + Alert: "Alert1", + Labels: map[string]string{ + "severity": "warning", + "namespace": "default", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "test-rule", + k8s.AlertRuleClassificationComponentKey: "networking", + k8s.AlertRuleClassificationLayerFromKey: "tier", + }, + } + rule.Labels[k8s.AlertRuleLabelId] = alertrule.GetAlertingRuleId(&rule) + + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alertWithLayer}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{rule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == rule.Labels[k8s.AlertRuleLabelId] { + return rule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + NamespaceFunc: func() k8s.NamespaceInterface { + ns := &testutils.MockNamespaceInterface{} + ns.SetMonitoringNamespaces(map[string]bool{"openshift-monitoring": true}) + return ns + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].AlertComponent != "networking" { + t.Errorf("expected component=networking, got %q", alerts[0].AlertComponent) + } + // "Cluster" from alert label lowercased to "cluster" + if alerts[0].AlertLayer != "cluster" { + t.Errorf("expected layer=cluster, got %q", alerts[0].AlertLayer) + } +} + +func TestGetAlerts_UsesRuleLabelsAsDefaults(t *testing.T) { + ctx := context.Background() + alert := k8s.PrometheusAlert{ + Labels: map[string]string{ + "alertname": "AlertRuleDefaults", + "severity": "warning", + "namespace": "default", + k8s.AlertRuleClassificationComponentKey: "team_a", + k8s.AlertRuleClassificationLayerKey: "namespace", + }, + State: "firing", + } + + rule := monitoringv1.Rule{ + Alert: "AlertRuleDefaults", + Labels: map[string]string{ + "severity": "warning", + "namespace": "default", + k8s.AlertRuleClassificationComponentKey: "team_a", + k8s.AlertRuleClassificationLayerKey: "namespace", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "defaults-rule", + }, + } + rule.Labels[k8s.AlertRuleLabelId] = alertrule.GetAlertingRuleId(&rule) + + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alert}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{rule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == rule.Labels[k8s.AlertRuleLabelId] { + return rule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].AlertComponent != "team_a" { + t.Errorf("expected component=team_a, got %q", alerts[0].AlertComponent) + } + if alerts[0].AlertLayer != "namespace" { + t.Errorf("expected layer=namespace, got %q", alerts[0].AlertLayer) + } +} + +func TestGetAlerts_FallsBackToDefaultWhenNoMatchingRule(t *testing.T) { + ctx := context.Background() + alert1 := k8s.PrometheusAlert{ + Labels: map[string]string{managementlabels.AlertNameLabel: "Alert1", "severity": "warning", "namespace": "default"}, + State: "firing", + } + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alert1}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{} }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].AlertComponent != "other" { + t.Errorf("expected component=other, got %q", alerts[0].AlertComponent) + } + if alerts[0].AlertLayer != "namespace" { + t.Errorf("expected layer=namespace, got %q", alerts[0].AlertLayer) + } +} + +func TestGetAlerts_FallsBackToDefaultWithMatchingRuleNoLabels(t *testing.T) { + ctx := context.Background() + alert1 := k8s.PrometheusAlert{ + Labels: map[string]string{managementlabels.AlertNameLabel: "Alert1", "severity": "warning", "namespace": "default"}, + State: "firing", + } + + rule := monitoringv1.Rule{ + Alert: "Alert1", + Labels: map[string]string{ + "severity": "warning", + "namespace": "default", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "default-rule", + }, + } + rule.Labels[k8s.AlertRuleLabelId] = alertrule.GetAlertingRuleId(&rule) + + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alert1}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{rule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == rule.Labels[k8s.AlertRuleLabelId] { + return rule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].AlertComponent != "other" { + t.Errorf("expected component=other, got %q", alerts[0].AlertComponent) + } + if alerts[0].AlertLayer != "cluster" { + t.Errorf("expected layer=cluster, got %q", alerts[0].AlertLayer) + } +} + +func TestGetAlerts_ReturnsEmptyList(t *testing.T) { + ctx := context.Background() + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 0 { + t.Errorf("expected empty list, got %d", len(alerts)) + } +} diff --git a/pkg/management/management_suite_test.go b/pkg/management/management_suite_test.go new file mode 100644 index 000000000..d5fa1082b --- /dev/null +++ b/pkg/management/management_suite_test.go @@ -0,0 +1,15 @@ +package management_test + +import ( + "os" + "testing" + + "github.com/prometheus/common/model" +) + +func TestMain(m *testing.M) { + // LegacyValidation is required for tests that construct relabel configs + // containing label names with special characters (e.g. slashes). + model.NameValidationScheme = model.LegacyValidation //nolint:staticcheck + os.Exit(m.Run()) +} diff --git a/pkg/management/testutils/k8s_client_mock.go b/pkg/management/testutils/k8s_client_mock.go index 0b9adbdb2..370125966 100644 --- a/pkg/management/testutils/k8s_client_mock.go +++ b/pkg/management/testutils/k8s_client_mock.go @@ -17,6 +17,8 @@ import ( // by AlertingRules().Get) hit the same store. type MockClient struct { TestConnectionFunc func(ctx context.Context) error + AlertingHealthFunc func(ctx context.Context) (k8s.AlertingHealth, error) + PrometheusAlertsFunc func() k8s.PrometheusAlertsInterface PrometheusRulesFunc func() k8s.PrometheusRuleInterface AlertRelabelConfigsFunc func() k8s.AlertRelabelConfigInterface AlertingRulesFunc func() k8s.AlertingRuleInterface @@ -38,6 +40,20 @@ func (m *MockClient) TestConnection(ctx context.Context) error { return nil } +func (m *MockClient) AlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) { + if m.AlertingHealthFunc != nil { + return m.AlertingHealthFunc(ctx) + } + return k8s.AlertingHealth{}, nil +} + +func (m *MockClient) PrometheusAlerts() k8s.PrometheusAlertsInterface { + if m.PrometheusAlertsFunc != nil { + return m.PrometheusAlertsFunc() + } + return &MockPrometheusAlertsInterface{} +} + func (m *MockClient) PrometheusRules() k8s.PrometheusRuleInterface { if m.PrometheusRulesFunc != nil { return m.PrometheusRulesFunc() @@ -88,7 +104,42 @@ func (m *MockClient) Namespace() k8s.NamespaceInterface { return m.namespace } -// MockPrometheusRuleInterface is a mock implementation of k8s.PrometheusRuleInterface +type MockPrometheusAlertsInterface struct { + GetAlertsFunc func(ctx context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) + GetRulesFunc func(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) + + ActiveAlerts []k8s.PrometheusAlert + RuleGroups []k8s.PrometheusRuleGroup +} + +func (m *MockPrometheusAlertsInterface) SetActiveAlerts(alerts []k8s.PrometheusAlert) { + m.ActiveAlerts = alerts +} + +func (m *MockPrometheusAlertsInterface) SetRuleGroups(groups []k8s.PrometheusRuleGroup) { + m.RuleGroups = groups +} + +func (m *MockPrometheusAlertsInterface) GetAlerts(ctx context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + if m.GetAlertsFunc != nil { + return m.GetAlertsFunc(ctx, req) + } + if m.ActiveAlerts != nil { + return m.ActiveAlerts, nil + } + return []k8s.PrometheusAlert{}, nil +} + +func (m *MockPrometheusAlertsInterface) GetRules(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + if m.GetRulesFunc != nil { + return m.GetRulesFunc(ctx, req) + } + if m.RuleGroups != nil { + return m.RuleGroups, nil + } + return []k8s.PrometheusRuleGroup{}, nil +} + type MockPrometheusRuleInterface struct { ListFunc func() ([]monitoringv1.PrometheusRule, error) GetFunc func(ctx context.Context, namespace string, name string) (*monitoringv1.PrometheusRule, bool, error) diff --git a/pkg/management/types.go b/pkg/management/types.go index e150832c8..a694c3ff3 100644 --- a/pkg/management/types.go +++ b/pkg/management/types.go @@ -4,6 +4,8 @@ import ( "context" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + + "github.com/openshift/monitoring-plugin/pkg/k8s" ) // Client is the interface for managing alert rules @@ -46,6 +48,12 @@ type Client interface { UpdateAlertRuleClassification(ctx context.Context, req UpdateRuleClassificationRequest) error // BulkUpdateAlertRuleClassification updates classification for multiple rule ids BulkUpdateAlertRuleClassification(ctx context.Context, items []UpdateRuleClassificationRequest) []error + + // GetAlerts retrieves Prometheus alerts + GetAlerts(ctx context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) + + // GetAlertingHealth retrieves the alerting stack health status + GetAlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) } // PrometheusRuleOptions specifies options for selecting PrometheusRule resources and groups diff --git a/pkg/management/update_classification.go b/pkg/management/update_classification.go index 777fc8846..fa76ec913 100644 --- a/pkg/management/update_classification.go +++ b/pkg/management/update_classification.go @@ -434,28 +434,3 @@ func getOriginalPlatformRuleFromPR(pr *monitoringv1.PrometheusRule, namespace st AdditionalInfo: fmt.Sprintf("in PrometheusRule %s/%s", namespace, name), } } - -// ApplyDynamicClassification resolves the effective component and layer for an -// alert by applying _from indirection. If a rule carries a component_from or -// layer_from label, the corresponding alert label value is used instead of the -// static default. Unresolvable or empty lookups fall back to the supplied -// defaults. -func ApplyDynamicClassification(ruleLabels, alertLabels map[string]string, defaultComponent, defaultLayer string) (string, string) { - component := defaultComponent - layer := defaultLayer - - if ruleLabels != nil { - if fromKey := ruleLabels[k8s.AlertRuleClassificationComponentFromKey]; fromKey != "" { - if v, ok := alertLabels[fromKey]; ok && v != "" { - component = v - } - } - if fromKey := ruleLabels[k8s.AlertRuleClassificationLayerFromKey]; fromKey != "" { - if v, ok := alertLabels[fromKey]; ok && v != "" { - layer = strings.ToLower(v) - } - } - } - - return component, layer -} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index eb181d521..9357ed5a6 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -332,6 +332,64 @@ func (f *Framework) CreateScopedUser(ctx context.Context, name, namespace, apiGr return &ScopedUser{Token: token, Cleanup: func() error { rollback(); return nil }}, nil } +// CreateUserWithClusterRole creates a ServiceAccount in the given namespace and +// binds an existing ClusterRole to it via a namespaced RoleBinding. Use this +// for OpenShift built-in roles such as monitoring-rules-view (Thanos tenancy +// on port 9093). API calls are retried to tolerate transient failures. +func (f *Framework) CreateUserWithClusterRole(ctx context.Context, name, namespace, clusterRoleName string) (*ScopedUser, error) { + rollback := func() { + _ = f.Clientset.RbacV1().RoleBindings(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + _ = f.Clientset.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + } + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + } + err := retry(3, func() error { + _, err := f.Clientset.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err + }) + if err != nil { + return nil, fmt.Errorf("creating service account %s/%s: %w", namespace, name, err) + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Subjects: []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, + Name: name, + Namespace: namespace, + }}, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: clusterRoleName, + }, + } + err = retry(3, func() error { + _, err := f.Clientset.RbacV1().RoleBindings(namespace).Create(ctx, rb, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err + }) + if err != nil { + rollback() + return nil, fmt.Errorf("creating role binding %s/%s for cluster role %s: %w", namespace, name, clusterRoleName, err) + } + + token, err := f.requestServiceAccountToken(ctx, namespace, name) + if err != nil { + rollback() + return nil, err + } + + return &ScopedUser{Token: token, Cleanup: func() error { rollback(); return nil }}, nil +} + // CreateAnonymousUser creates a ServiceAccount with no RBAC permissions. // The whole setup is retried to tolerate transient API failures. func (f *Framework) CreateAnonymousUser(ctx context.Context, name, namespace string) (*ScopedUser, error) { diff --git a/test/e2e/get_alerts_test.go b/test/e2e/get_alerts_test.go new file mode 100644 index 000000000..49fbbe334 --- /dev/null +++ b/test/e2e/get_alerts_test.go @@ -0,0 +1,297 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "testing" + "time" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/test/e2e/framework" +) + +func TestGetAlerts(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + testNamespace, cleanup, err := f.CreateUserNamespace(ctx, "test-get-alerts") + if err != nil { + t.Fatalf("Failed to create test namespace: %v", err) + } + defer cleanup() + + forDuration := monitoringv1.Duration("1s") + alertName := "E2EGetAlertsTest" + + promRule := &monitoringv1.PrometheusRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-get-alerts-rule", + Namespace: testNamespace, + }, + Spec: monitoringv1.PrometheusRuleSpec{ + Groups: []monitoringv1.RuleGroup{ + { + Name: "e2e-test-group", + Rules: []monitoringv1.Rule{ + { + Alert: alertName, + Expr: intstr.FromString("vector(1)"), + For: &forDuration, + Labels: map[string]string{ + "severity": "none", + "team": "e2e", + }, + Annotations: map[string]string{ + "summary": "E2E test alert for GET /alerts", + }, + }, + }, + }, + }, + }, + } + + _, err = f.Monitoringv1clientset.MonitoringV1().PrometheusRules(testNamespace).Create( + ctx, promRule, metav1.CreateOptions{}, + ) + if err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + httpClient := f.HTTPClient() + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + alertsURL := f.PluginURL + "/api/v1/alerting/alerts" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, alertsURL, nil) + if err != nil { + return false, err + } + if f.BearerToken != "" { + req.Header.Set("Authorization", "Bearer "+f.BearerToken) + } + + resp, err := httpClient.Do(req) + if err != nil { + t.Logf("Failed to query alerts: %v", err) + return false, nil + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Logf("GET /alerts returned status %d, retrying", resp.StatusCode) + return false, nil + } + + var alertsResp struct { + Data struct { + Alerts []k8s.PrometheusAlert `json:"alerts"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&alertsResp); err != nil { + t.Logf("Failed to decode alerts response: %v", err) + return false, nil + } + + for _, alert := range alertsResp.Data.Alerts { + if alert.Labels["alertname"] == alertName { + if alert.State != "firing" && alert.State != "pending" { + t.Logf("Found alert %s but state is %q, waiting for firing/pending", alertName, alert.State) + return false, nil + } + if alert.Labels["severity"] != "none" { + t.Errorf("Expected severity=none, got %q", alert.Labels["severity"]) + } + t.Logf("Found alert %s in state %q", alertName, alert.State) + return true, nil + } + } + + t.Logf("Alert %s not found yet (got %d alerts total)", alertName, len(alertsResp.Data.Alerts)) + return false, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for alert to appear: %v", err) + } + + t.Log("GET /alerts e2e test passed successfully") +} + +// TestRBAC_GetAlerts verifies Thanos-tenancy RBAC for GET /alerts. +// +// With ?namespace=: User A (no perms) gets HTTP 200 without the UWM alert in +// ns Y; User B (monitoring-rules-view in Y) sees Y but not Z; cluster-admin +// sees Y. +// +// Without ?namespace=: fan-out must not leak the alert to unprivileged users +// and must still return it for namespace-scoped viewers. +func TestRBAC_GetAlerts(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + nsY, cleanupY, err := f.CreateUserNamespace(ctx, "test-rbac-get-alerts-y") + if err != nil { + t.Fatalf("Failed to create namespace Y: %v", err) + } + defer func() { _ = cleanupY() }() + + nsZ, cleanupZ, err := f.CreateUserNamespace(ctx, "test-rbac-get-alerts-z") + if err != nil { + t.Fatalf("Failed to create namespace Z: %v", err) + } + defer func() { _ = cleanupZ() }() + + userA, err := f.CreateAnonymousUser(ctx, "e2e-rbac-get-a", "default") + if err != nil { + t.Fatalf("Failed to create unprivileged user A: %v", err) + } + defer func() { _ = userA.Cleanup() }() + + userB, err := f.CreateUserWithClusterRole(ctx, "e2e-rbac-get-b", nsY, "monitoring-rules-view") + if err != nil { + t.Fatalf("Failed to create scoped user B: %v", err) + } + defer func() { _ = userB.Cleanup() }() + + alertName := "E2ERBACGetAlertsTest" + if err := createFiringPrometheusRule(ctx, f, nsY, "e2e-rbac-get-alerts-rule", alertName); err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + alerts, status, err := getAlertsWithToken(f, ctx, f.BearerToken, nsY) + if err != nil { + t.Logf("Admin GET /alerts failed: %v", err) + return false, nil + } + if status != http.StatusOK { + t.Logf("Admin GET /alerts returned status %d, retrying", status) + return false, nil + } + if containsAlert(alerts, alertName) { + t.Logf("Admin sees alert %s", alertName) + return true, nil + } + t.Logf("Waiting for alert %s (admin sees %d alerts)", alertName, len(alerts)) + return false, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for admin to see alert: %v", err) + } + + cases := []struct { + name string + token string + namespace string + wantAlert bool + }{ + {"UserA_NoPerms_NamespaceY", userA.Token, nsY, false}, + {"UserA_NoPerms_NoNamespace", userA.Token, "", false}, + {"UserB_RulesView_NamespaceY", userB.Token, nsY, true}, + {"UserB_RulesView_NamespaceZ", userB.Token, nsZ, false}, + {"UserB_RulesView_NoNamespace", userB.Token, "", true}, + {"UserC_ClusterAdmin_NamespaceY", f.BearerToken, nsY, true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + alerts, status, err := getAlertsWithToken(f, ctx, tc.token, tc.namespace) + if err != nil { + t.Fatalf("GET /alerts request failed: %v", err) + } + if status != http.StatusOK { + t.Fatalf("Expected status %d, got %d", http.StatusOK, status) + } + got := containsAlert(alerts, alertName) + if got != tc.wantAlert { + t.Fatalf("Alert %s visibility: want %v, got %v (%d alerts returned)", alertName, tc.wantAlert, got, len(alerts)) + } + }) + } +} + +func createFiringPrometheusRule(ctx context.Context, f *framework.Framework, namespace, name, alertName string) error { + forDuration := monitoringv1.Duration("1s") + promRule := &monitoringv1.PrometheusRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: monitoringv1.PrometheusRuleSpec{ + Groups: []monitoringv1.RuleGroup{{ + Name: "e2e-rbac-group", + Rules: []monitoringv1.Rule{{ + Alert: alertName, + Expr: intstr.FromString("vector(1)"), + For: &forDuration, + Labels: map[string]string{ + "severity": "none", + "e2e_test": "rbac_get", + }, + }}, + }}, + }, + } + _, err := f.Monitoringv1clientset.MonitoringV1().PrometheusRules(namespace).Create(ctx, promRule, metav1.CreateOptions{}) + return err +} + +func containsAlert(alerts []k8s.PrometheusAlert, alertName string) bool { + for _, a := range alerts { + if a.Labels["alertname"] == alertName { + return true + } + } + return false +} + +// getAlertsWithToken calls GET /alerts with an optional namespace query param. +// It returns the decoded alerts and HTTP status. A non-OK status is not an +// error — callers must assert on status explicitly. +func getAlertsWithToken(f *framework.Framework, ctx context.Context, token, namespace string) ([]k8s.PrometheusAlert, int, error) { + alertsURL := f.PluginURL + "/api/v1/alerting/alerts" + if namespace != "" { + alertsURL += "?" + url.Values{"namespace": {namespace}}.Encode() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, alertsURL, nil) + if err != nil { + return nil, 0, fmt.Errorf("creating request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+token) + + resp, err := f.HTTPClient().Do(req) + if err != nil { + return nil, 0, fmt.Errorf("executing request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, resp.StatusCode, nil + } + + var alertsResp struct { + Data struct { + Alerts []k8s.PrometheusAlert `json:"alerts"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&alertsResp); err != nil { + return nil, resp.StatusCode, fmt.Errorf("decoding response: %w", err) + } + return alertsResp.Data.Alerts, resp.StatusCode, nil +} From 0cf208805afcfdbd461b9091059a7e5105149126 Mon Sep 17 00:00:00 2001 From: Shirly Radco Date: Thu, 12 Mar 2026 20:34:26 +0200 Subject: [PATCH 2/5] router: add GET /rules endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add GET /api/v1/alerting/rules endpoint with Prometheus rule group retrieval, list filtering, and label matching. Signed-off-by: Shirly Radco Signed-off-by: João Vilaça Signed-off-by: Aviv Litman Co-authored-by: AI Assistant --- internal/managementrouter/query_filters.go | 46 +- .../managementrouter/query_filters_test.go | 166 +++++++ internal/managementrouter/router.go | 5 +- internal/managementrouter/rules_get.go | 48 ++ pkg/k8s/prometheus_rules_types.go | 5 + pkg/management/get_rules.go | 391 +++++++++++++++ pkg/management/get_rules_test.go | 442 +++++++++++++++++ pkg/management/list_rules.go | 83 ++++ pkg/management/list_rules_test.go | 444 ++++++++++++++++++ pkg/management/types.go | 33 ++ test/e2e/relabeled_rules_test.go | 443 +++++++++++++++++ 11 files changed, 2097 insertions(+), 9 deletions(-) create mode 100644 internal/managementrouter/query_filters_test.go create mode 100644 internal/managementrouter/rules_get.go create mode 100644 pkg/management/get_rules.go create mode 100644 pkg/management/get_rules_test.go create mode 100644 pkg/management/list_rules.go create mode 100644 pkg/management/list_rules_test.go create mode 100644 test/e2e/relabeled_rules_test.go diff --git a/internal/managementrouter/query_filters.go b/internal/managementrouter/query_filters.go index f8e3e5e9d..5f1d58498 100644 --- a/internal/managementrouter/query_filters.go +++ b/internal/managementrouter/query_filters.go @@ -13,23 +13,55 @@ var validStates = map[string]bool{ "silenced": true, } -// parseStateAndLabels returns the optional state filter and label matches. -// Any query param other than "state" is treated as a label match. -// Returns an error if the state value is not one of the known states. -func parseStateAndLabels(q url.Values) (string, map[string]string, error) { +// reservedQueryKeys lists query parameter names that have special meaning +// and must not be treated as label equality filters. +var reservedQueryKeys = map[string]bool{ + "state": true, + "match[]": true, + "limit": true, + "next_token": true, +} + +// parseStateLabelsAndMatchers returns the optional state filter, label equality +// matches, and Prometheus-style label matchers from the query string. +// +// Reserved keys ("state", "match[]") are handled specially. Every other key is +// treated as a label equality filter (e.g. ?severity=critical). +// +// match[] values follow upstream Prometheus API conventions and may contain +// equality, inequality, regex, or negative-regex matchers: +// +// ?match[]=severity="critical"&match[]=alertname=~"Kube.*" +func parseStateLabelsAndMatchers(q url.Values) (string, map[string]string, []string, error) { state := strings.ToLower(strings.TrimSpace(q.Get("state"))) if !validStates[state] { - return "", nil, fmt.Errorf("invalid state filter %q: must be one of pending, firing, silenced", q.Get("state")) + return "", nil, nil, fmt.Errorf("invalid state filter %q: must be one of pending, firing, silenced", q.Get("state")) } labels := make(map[string]string) for key, vals := range q { - if key == "state" { + if reservedQueryKeys[key] { continue } if len(vals) > 0 && strings.TrimSpace(vals[0]) != "" { labels[strings.TrimSpace(key)] = strings.TrimSpace(vals[0]) } } - return state, labels, nil + + var matchers []string + for _, raw := range q["match[]"] { + v := strings.TrimSpace(raw) + if v != "" { + matchers = append(matchers, v) + } + } + + return state, labels, matchers, nil +} + +// parseStateAndLabels returns the optional state filter and label matches. +// Any query param other than reserved keys is treated as a label match. +func parseStateAndLabels(q url.Values) (string, map[string]string, error) { + state, labels, _, err := parseStateLabelsAndMatchers(q) + return state, labels, err } diff --git a/internal/managementrouter/query_filters_test.go b/internal/managementrouter/query_filters_test.go new file mode 100644 index 000000000..e417245e1 --- /dev/null +++ b/internal/managementrouter/query_filters_test.go @@ -0,0 +1,166 @@ +package managementrouter + +import ( + "net/url" + "testing" +) + +func TestParseStateLabelsAndMatchers(t *testing.T) { + tests := []struct { + name string + query string + wantState string + wantLabels map[string]string + wantMatchers []string + wantMatchersLen int + wantErr bool + }{ + { + name: "empty query", + query: "", + wantState: "", + wantLabels: map[string]string{}, + wantMatchers: nil, + }, + { + name: "state only", + query: "state=firing", + wantState: "firing", + wantLabels: map[string]string{}, + }, + { + name: "flat labels only", + query: "severity=critical&namespace=openshift-monitoring", + wantState: "", + wantLabels: map[string]string{ + "severity": "critical", + "namespace": "openshift-monitoring", + }, + }, + { + name: "match[] only with equality", + query: `match[]=severity="critical"`, + wantState: "", + wantLabels: map[string]string{}, + wantMatchers: []string{ + `severity="critical"`, + }, + }, + { + name: "match[] with regex", + query: `match[]=alertname=~"Kube.*CPU.*"`, + wantState: "", + wantLabels: map[string]string{}, + wantMatchers: []string{ + `alertname=~"Kube.*CPU.*"`, + }, + }, + { + name: "multiple match[] values", + query: `match[]=severity="critical"&match[]=namespace="openshift-monitoring"`, + wantState: "", + wantLabels: map[string]string{}, + wantMatchersLen: 2, + }, + { + name: "mixed flat labels and match[]", + query: `state=firing&team=sre&match[]=severity=~"critical|warning"`, + wantState: "firing", + wantLabels: map[string]string{ + "team": "sre", + }, + wantMatchers: []string{ + `severity=~"critical|warning"`, + }, + }, + { + name: "match[] is not treated as a label", + query: `match[]=severity="critical"`, + wantState: "", + wantLabels: map[string]string{}, + }, + { + name: "invalid state", + query: "state=invalid", + wantErr: true, + }, + { + name: "empty match[] values are skipped", + query: `match[]=&match[]=%20&match[]=severity="warning"`, + wantState: "", + wantLabels: map[string]string{}, + wantMatchersLen: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + q, err := url.ParseQuery(tt.query) + if err != nil { + t.Fatalf("invalid test query: %v", err) + } + + state, labels, matchers, err := parseStateLabelsAndMatchers(q) + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if state != tt.wantState { + t.Errorf("state = %q, want %q", state, tt.wantState) + } + + if tt.wantLabels != nil { + if len(labels) != len(tt.wantLabels) { + t.Errorf("labels length = %d, want %d", len(labels), len(tt.wantLabels)) + } + for k, v := range tt.wantLabels { + if labels[k] != v { + t.Errorf("labels[%q] = %q, want %q", k, labels[k], v) + } + } + if _, found := labels["match[]"]; found { + t.Error("match[] should not appear in labels map") + } + } + + if tt.wantMatchers != nil { + if len(matchers) != len(tt.wantMatchers) { + t.Errorf("matchers length = %d, want %d", len(matchers), len(tt.wantMatchers)) + } + for i, want := range tt.wantMatchers { + if i < len(matchers) && matchers[i] != want { + t.Errorf("matchers[%d] = %q, want %q", i, matchers[i], want) + } + } + } + + if tt.wantMatchersLen > 0 && len(matchers) != tt.wantMatchersLen { + t.Errorf("matchers length = %d, want %d", len(matchers), tt.wantMatchersLen) + } + }) + } +} + +func TestParseStateAndLabelsBackcompat(t *testing.T) { + q, _ := url.ParseQuery(`state=firing&severity=critical&match[]=alertname=~"Foo.*"`) + + state, labels, err := parseStateAndLabels(q) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != "firing" { + t.Errorf("state = %q, want %q", state, "firing") + } + if labels["severity"] != "critical" { + t.Errorf("severity = %q, want %q", labels["severity"], "critical") + } + if _, found := labels["match[]"]; found { + t.Error("match[] should not appear in labels map") + } +} diff --git a/internal/managementrouter/router.go b/internal/managementrouter/router.go index 8706b7b04..f0ac5cfb8 100644 --- a/internal/managementrouter/router.go +++ b/internal/managementrouter/router.go @@ -43,9 +43,10 @@ func New(managementClient management.Client) *mux.Router { BaseURL: "/api/v1/alerting", BaseRouter: r, }) - // GET /alerts is not yet in the OpenAPI spec; registered manually - // until its branch adds the spec entry and generated bindings. + // GET /alerts and GET /rules are not yet in the OpenAPI spec; registered + // manually until their respective branches add the spec entries. r.HandleFunc("/api/v1/alerting/alerts", hr.GetAlerts).Methods(http.MethodGet) + r.HandleFunc("/api/v1/alerting/rules", hr.GetRules).Methods(http.MethodGet) return r } diff --git a/internal/managementrouter/rules_get.go b/internal/managementrouter/rules_get.go new file mode 100644 index 000000000..fe9a59409 --- /dev/null +++ b/internal/managementrouter/rules_get.go @@ -0,0 +1,48 @@ +package managementrouter + +import ( + "encoding/json" + "net/http" + + "github.com/openshift/monitoring-plugin/pkg/k8s" +) + +type GetRulesResponse struct { + Data GetRulesResponseData `json:"data"` + Warnings []string `json:"warnings,omitempty"` +} + +type GetRulesResponseData struct { + Groups []k8s.PrometheusRuleGroup `json:"groups"` +} + +func (hr *httpRouter) GetRules(w http.ResponseWriter, req *http.Request) { + state, labels, matchers, err := parseStateLabelsAndMatchers(req.URL.Query()) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + ctx := req.Context() + + groups, err := hr.managementClient.GetRules(ctx, k8s.GetRulesRequest{ + Labels: labels, + Matchers: matchers, + State: state, + }) + if err != nil { + handleError(w, err) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(GetRulesResponse{ + Data: GetRulesResponseData{ + Groups: groups, + }, + Warnings: hr.rulesWarnings(ctx), + }); err != nil { + log.WithError(err).Warn("failed to encode rules response") + } +} diff --git a/pkg/k8s/prometheus_rules_types.go b/pkg/k8s/prometheus_rules_types.go index 3f5c289fb..c41ea4e89 100644 --- a/pkg/k8s/prometheus_rules_types.go +++ b/pkg/k8s/prometheus_rules_types.go @@ -5,6 +5,11 @@ import ( "time" ) +const ( + RuleTypeAlerting = "alerting" + RuleTypeRecording = "recording" +) + // GetRulesRequest holds parameters for filtering rules alerts. type GetRulesRequest struct { // Labels filters rules by exact label equality. The special key "namespace" diff --git a/pkg/management/get_rules.go b/pkg/management/get_rules.go new file mode 100644 index 000000000..f30822d35 --- /dev/null +++ b/pkg/management/get_rules.go @@ -0,0 +1,391 @@ +package management + +import ( + "context" + "fmt" + "math" + "sort" + "strings" + "time" + "unicode" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/relabel" + "github.com/prometheus/prometheus/promql/parser" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +func (c *client) GetRules(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + groups, err := c.k8sClient.PrometheusAlerts().GetRules(ctx, req) + if err != nil { + return nil, fmt.Errorf("failed to get prometheus rules: %w", err) + } + + configs := c.k8sClient.RelabeledRules().Config() + relabeledByAlert := indexRelabeledRules(c.k8sClient.RelabeledRules().List(ctx)) + applyFilters := req.State != "" || len(req.Labels) > 0 + + // Deduplicate rules that carry the same openshift_io_alert_rule_id across + // groups. This occurs when the same PrometheusRule group name is defined in + // multiple CRDs — Prometheus returns separate groups with identical rules + // that hash to the same ID after enrichment. + seenIDs := make(map[string]struct{}) + + filteredGroups := make([]k8s.PrometheusRuleGroup, 0, len(groups)) + for groupIdx := range groups { + group := groups[groupIdx] + filteredRules := make([]k8s.PrometheusRule, 0, len(group.Rules)) + + for ruleIdx := range group.Rules { + rule := group.Rules[ruleIdx] + if applyFilters && rule.Type != k8s.RuleTypeAlerting { + continue + } + applyRelabeledRuleLabels(&rule, relabeledByAlert) + + if ruleID := rule.Labels[k8s.AlertRuleLabelId]; ruleID != "" { + if _, seen := seenIDs[ruleID]; seen { + continue + } + seenIDs[ruleID] = struct{}{} + } + + if len(rule.Alerts) == 0 { + if applyFilters && rule.Type == k8s.RuleTypeAlerting { + continue + } + filteredRules = append(filteredRules, rule) + continue + } + + relabeledAlerts := make([]k8s.PrometheusRuleAlert, 0, len(rule.Alerts)) + for _, alert := range rule.Alerts { + if alert.State == "pending" || alert.State == "firing" { + if alert.Labels[k8s.AlertSourceLabel] != k8s.AlertSourceUser { + // Apply relabeling to the "real" alert labels only; preserve plugin meta labels. + src := alert.Labels[k8s.AlertSourceLabel] + in := make(map[string]string, len(alert.Labels)) + for k, v := range alert.Labels { + in[k] = v + } + delete(in, k8s.AlertSourceLabel) + + relabeledLabels, keep := relabel.Process(labels.FromMap(in), configs...) + if !keep { + continue + } + alert.Labels = relabeledLabels.Map() + if src != "" { + alert.Labels[k8s.AlertSourceLabel] = src + } + } + } + + if req.State != "" && alert.State != req.State { + continue + } + if !ruleAlertLabelsMatch(&req, &alert) { + continue + } + relabeledAlerts = append(relabeledAlerts, alert) + } + rule.Alerts = relabeledAlerts + + if applyFilters && rule.Type == k8s.RuleTypeAlerting && len(rule.Alerts) == 0 { + continue + } + + filteredRules = append(filteredRules, rule) + } + + group.Rules = filteredRules + if applyFilters && len(group.Rules) == 0 { + continue + } + filteredGroups = append(filteredGroups, group) + } + + return filteredGroups, nil +} + +func indexRelabeledRules(rules []monitoringv1.Rule) map[string][]monitoringv1.Rule { + byAlert := make(map[string][]monitoringv1.Rule, len(rules)) + for _, rule := range rules { + alertName := rule.Alert + if alertName == "" && rule.Labels != nil { + alertName = rule.Labels[managementlabels.AlertNameLabel] + } + if alertName == "" { + continue + } + byAlert[alertName] = append(byAlert[alertName], rule) + } + return byAlert +} + +func relabeledAlertName(rule *monitoringv1.Rule) string { + if rule == nil { + return "" + } + if rule.Alert != "" { + return rule.Alert + } + if rule.Labels != nil { + return rule.Labels[managementlabels.AlertNameLabel] + } + return "" +} + +func applyRelabeledRuleLabels(rule *k8s.PrometheusRule, relabeledByAlert map[string][]monitoringv1.Rule) { + if rule == nil || rule.Name == "" || rule.Type == k8s.RuleTypeRecording { + return + } + + // Preserve plugin meta labels added during API fetch. + source := "" + if rule.Labels != nil { + source = rule.Labels[k8s.AlertSourceLabel] + } + + match := findRelabeledMatch(rule, relabeledByAlert[rule.Name]) + if match == nil || match.Labels == nil { + return + } + + // Replace rule labels with the relabeled cache version so that actions which + // remove/rename labels (e.g. LabelDrop/LabelKeep/LabelMap) are faithfully reflected. + labelsOut := make(map[string]string, len(match.Labels)+1) + for k, v := range match.Labels { + labelsOut[k] = v + } + if source != "" { + labelsOut[k8s.AlertSourceLabel] = source + } + rule.Labels = labelsOut +} + +func findRelabeledMatch(rule *k8s.PrometheusRule, candidates []monitoringv1.Rule) *monitoringv1.Rule { + // Strict match first (preserves correctness when multiple rules share alertname). + for i := range candidates { + candidate := &candidates[i] + if promRuleMatchesRelabeled(rule, candidate) { + return candidate + } + } + + // If relabeling modified rule labels (e.g. severity), strict label matching may fail. + // Retry on a best-effort basis using (alertname, expr, for) only. If this is ambiguous, + // do not guess. + var relaxed *monitoringv1.Rule + for i := range candidates { + candidate := &candidates[i] + if rule == nil || candidate == nil { + continue + } + candidateName := relabeledAlertName(candidate) + if rule.Name == "" || candidateName == "" || rule.Name != candidateName { + continue + } + if canonicalizePromQL(rule.Query) != canonicalizePromQL(candidate.Expr.String()) { + continue + } + if !durationMatches(rule.Duration, candidate.For) { + continue + } + if relaxed != nil { + // ambiguous + relaxed = nil + break + } + relaxed = candidate + } + if relaxed != nil { + return relaxed + } + + // Fallback: if alertname is globally unique, avoid brittle PromQL/metadata matching. + // This helps when Prometheus stringifies PromQL differently than PrometheusRule YAML + // (e.g. label matcher ordering). + if len(candidates) == 1 { + return &candidates[0] + } + return nil +} + +func promRuleMatchesRelabeled(rule *k8s.PrometheusRule, candidate *monitoringv1.Rule) bool { + if rule == nil || candidate == nil { + return false + } + candidateName := relabeledAlertName(candidate) + if rule.Name == "" || candidateName == "" || rule.Name != candidateName { + return false + } + if canonicalizePromQL(rule.Query) != canonicalizePromQL(candidate.Expr.String()) { + return false + } + if !durationMatches(rule.Duration, candidate.For) { + return false + } + if !stringMapEqual(filterBusinessLabels(rule.Labels), filterBusinessLabels(candidate.Labels)) { + return false + } + return true +} + +func canonicalizePromQL(in string) string { + s := strings.TrimSpace(in) + if s == "" { + return "" + } + expr, err := parser.ParseExpr(s) + if err == nil && expr != nil { + parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error { + switch n := node.(type) { + case *parser.VectorSelector: + sort.Slice(n.LabelMatchers, func(i, j int) bool { + mi, mj := n.LabelMatchers[i], n.LabelMatchers[j] + if mi == nil || mj == nil { + return mi != nil + } + if mi.Name != mj.Name { + return mi.Name < mj.Name + } + if mi.Type != mj.Type { + return mi.Type < mj.Type + } + return mi.Value < mj.Value + }) + case *parser.AggregateExpr: + sort.Strings(n.Grouping) + case *parser.BinaryExpr: + if n.VectorMatching != nil { + sort.Strings(n.VectorMatching.MatchingLabels) + sort.Strings(n.VectorMatching.Include) + } + } + return nil + }) + + return expr.String() + } + return normalizeSpaceOutsideQuotes(s) +} + +func normalizeSpaceOutsideQuotes(in string) string { + if in == "" { + return "" + } + in = strings.TrimSpace(in) + + var b strings.Builder + b.Grow(len(in)) + + inQuote := false + escaped := false + pendingSpace := false + lastNoSpaceToken := false + + isNoSpaceToken := func(r rune) bool { + switch r { + case '(', ')', '{', '}', ',', '+', '-', '*', '/', '%', '^', '=', '!', '<', '>': + return true + default: + return false + } + } + + for _, r := range in { + if escaped { + if pendingSpace { + if !lastNoSpaceToken { + b.WriteByte(' ') + } + pendingSpace = false + } + b.WriteRune(r) + escaped = false + lastNoSpaceToken = false + continue + } + + if inQuote && r == '\\' { + if pendingSpace { + if !lastNoSpaceToken { + b.WriteByte(' ') + } + pendingSpace = false + } + b.WriteRune(r) + escaped = true + lastNoSpaceToken = false + continue + } + + if r == '"' { + if pendingSpace { + if !lastNoSpaceToken { + b.WriteByte(' ') + } + pendingSpace = false + } + inQuote = !inQuote + b.WriteRune(r) + lastNoSpaceToken = false + continue + } + + if !inQuote && unicode.IsSpace(r) { + pendingSpace = true + continue + } + + if pendingSpace && !lastNoSpaceToken && !isNoSpaceToken(r) { + b.WriteByte(' ') + } + pendingSpace = false + + b.WriteRune(r) + lastNoSpaceToken = !inQuote && isNoSpaceToken(r) + } + + return strings.TrimSpace(b.String()) +} + +func durationMatches(seconds float64, duration *monitoringv1.Duration) bool { + if duration == nil { + return seconds == 0 + } + parsed, err := time.ParseDuration(string(*duration)) + if err != nil { + return false + } + return math.Abs(parsed.Seconds()-seconds) < 0.001 +} + +func stringMapEqual(a, b map[string]string) bool { + if len(a) == 0 && len(b) == 0 { + return true + } + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +func ruleAlertLabelsMatch(req *k8s.GetRulesRequest, alert *k8s.PrometheusRuleAlert) bool { + for key, value := range req.Labels { + if alertValue, exists := alert.Labels[key]; !exists || alertValue != value { + return false + } + } + + return true +} diff --git a/pkg/management/get_rules_test.go b/pkg/management/get_rules_test.go new file mode 100644 index 000000000..56a5844fe --- /dev/null +++ b/pkg/management/get_rules_test.go @@ -0,0 +1,442 @@ +package management_test + +import ( + "context" + "testing" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/relabel" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management" + "github.com/openshift/monitoring-plugin/pkg/management/testutils" + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +// grFixture builds a management client with a PrometheusAlerts mock returning +// the given groups and a RelabeledRules mock returning the given configs/rules. +type grFixture struct { + groups []k8s.PrometheusRuleGroup + relabelRules []monitoringv1.Rule + relabelConfig []*relabel.Config +} + +func (f grFixture) client(t *testing.T) management.Client { + t.Helper() + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetRulesFunc: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return f.groups, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return f.relabelRules }, + ConfigFunc: func() []*relabel.Config { return f.relabelConfig }, + } + }, + } + return management.New(context.Background(), mockK8s) +} + +// threeAlertGroup returns a rule group containing one alerting rule with +// firing Alert1, pending Alert2, and inactive Alert3. +func threeAlertGroup() []k8s.PrometheusRuleGroup { + return []k8s.PrometheusRuleGroup{ + { + Name: "group-a", + Rules: []k8s.PrometheusRule{ + { + Name: "rule-a", + Type: k8s.RuleTypeAlerting, + Alerts: []k8s.PrometheusRuleAlert{ + {State: "firing", Labels: map[string]string{"alertname": "Alert1", "severity": "warning"}}, + {State: "pending", Labels: map[string]string{"alertname": "Alert2", "severity": "critical"}}, + {State: "inactive", Labels: map[string]string{"alertname": "Alert3", "severity": "warning"}}, + }, + }, + }, + }, + } +} + +func dropAlert2ReplaceAlert1Severity() []*relabel.Config { + return []*relabel.Config{ + { + SourceLabels: model.LabelNames{"alertname"}, + Regex: relabel.MustNewRegexp("Alert2"), + Action: relabel.Drop, + NameValidationScheme: model.UTF8Validation, + }, + { + SourceLabels: model.LabelNames{"alertname"}, + Regex: relabel.MustNewRegexp("Alert1"), + TargetLabel: "severity", + Replacement: "critical", + Action: relabel.Replace, + NameValidationScheme: model.UTF8Validation, + }, + } +} + +func TestGetRules_AppliesRelabelConfigsToPendingFiringOnly(t *testing.T) { + f := grFixture{ + groups: threeAlertGroup(), + relabelRules: []monitoringv1.Rule{}, + relabelConfig: dropAlert2ReplaceAlert1Severity(), + } + client := f.client(t) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 1 { + t.Fatalf("expected 1 group, got %d", len(groups)) + } + rules := groups[0].Rules + if len(rules) != 1 { + t.Fatalf("expected 1 rule, got %d", len(rules)) + } + alerts := rules[0].Alerts + if len(alerts) != 2 { + t.Fatalf("expected 2 alerts after drop, got %d", len(alerts)) + } + if alerts[0].Labels["alertname"] != "Alert1" || alerts[0].Labels["severity"] != "critical" { + t.Errorf("alert[0]: got alertname=%s severity=%s", alerts[0].Labels["alertname"], alerts[0].Labels["severity"]) + } + if alerts[1].Labels["alertname"] != "Alert3" || alerts[1].Labels["severity"] != "warning" { + t.Errorf("alert[1]: got alertname=%s severity=%s", alerts[1].Labels["alertname"], alerts[1].Labels["severity"]) + } +} + +func TestGetRules_FiltersByStateAndLabels(t *testing.T) { + f := grFixture{ + groups: threeAlertGroup(), + relabelRules: []monitoringv1.Rule{}, + relabelConfig: dropAlert2ReplaceAlert1Severity(), + } + client := f.client(t) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{ + State: "firing", + Labels: map[string]string{"severity": "critical"}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 1 { + t.Fatalf("expected 1 group, got %d", len(groups)) + } + alerts := groups[0].Rules[0].Alerts + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].Labels["alertname"] != "Alert1" || alerts[0].Labels["severity"] != "critical" { + t.Errorf("unexpected alert: %v", alerts[0].Labels) + } +} + +func TestGetRules_DropsNonMatchingRulesWhenFiltered(t *testing.T) { + f := grFixture{ + groups: threeAlertGroup(), + relabelRules: []monitoringv1.Rule{}, + relabelConfig: dropAlert2ReplaceAlert1Severity(), + } + client := f.client(t) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{ + State: "firing", + Labels: map[string]string{"severity": "does-not-exist"}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 0 { + t.Errorf("expected 0 groups, got %d", len(groups)) + } +} + +func TestGetRules_AddsManagedByLabelsFromRelabeledRules(t *testing.T) { + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetRulesFunc: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return []k8s.PrometheusRuleGroup{ + { + Name: "group-a", + Rules: []k8s.PrometheusRule{ + { + Name: "AlertWithManagedBy", + Type: "alerting", + Query: "up == 0", + Labels: map[string]string{"severity": "critical"}, + Annotations: map[string]string{"summary": "test alert"}, + }, + }, + }, + }, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { + return []monitoringv1.Rule{ + { + Alert: "AlertWithManagedBy", + Expr: intstr.FromString("up ==\n 0"), + Labels: map[string]string{ + "severity": "critical", + k8s.AlertRuleLabelId: "alert-id-1", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + managementlabels.RuleManagedByLabel: "operator", + managementlabels.RelabelConfigManagedByLabel: "gitops", + }, + Annotations: map[string]string{"summary": "test alert"}, + }, + } + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(context.Background(), mockK8s) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 1 || len(groups[0].Rules) != 1 { + t.Fatalf("expected 1 group with 1 rule") + } + rule := groups[0].Rules[0] + checks := map[string]string{ + k8s.AlertRuleLabelId: "alert-id-1", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + managementlabels.RuleManagedByLabel: "operator", + managementlabels.RelabelConfigManagedByLabel: "gitops", + } + for k, want := range checks { + if got := rule.Labels[k]; got != want { + t.Errorf("label[%s]: want %q, got %q", k, want, got) + } + } +} + +func TestGetRules_EnrichesWithAllLabelTypes(t *testing.T) { + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetRulesFunc: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return []k8s.PrometheusRuleGroup{ + { + Name: "group-a", + Rules: []k8s.PrometheusRule{ + { + Name: "ARCUpdatedRule", + Type: "alerting", + Query: "up == 0", + Labels: map[string]string{"severity": "warning", k8s.AlertSourceLabel: k8s.AlertSourcePlatform}, + }, + }, + }, + }, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { + return []monitoringv1.Rule{ + { + Alert: "ARCUpdatedRule", + Expr: intstr.FromString("up ==\n 0"), + Labels: map[string]string{ + "severity": "critical", + "team": "sre", + k8s.AlertRuleLabelId: "rid-arc-1", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + k8s.AlertRuleClassificationComponentKey: "compute", + k8s.AlertRuleClassificationLayerKey: "cluster", + managementlabels.RuleManagedByLabel: "operator", + managementlabels.RelabelConfigManagedByLabel: "gitops", + }, + }, + } + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(context.Background(), mockK8s) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 1 || len(groups[0].Rules) != 1 { + t.Fatalf("expected 1 group with 1 rule") + } + rule := groups[0].Rules[0] + checks := map[string]string{ + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.AlertRuleLabelId: "rid-arc-1", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + k8s.AlertRuleClassificationComponentKey: "compute", + k8s.AlertRuleClassificationLayerKey: "cluster", + "severity": "critical", + "team": "sre", + managementlabels.RuleManagedByLabel: "operator", + managementlabels.RelabelConfigManagedByLabel: "gitops", + } + for k, want := range checks { + if got := rule.Labels[k]; got != want { + t.Errorf("label[%s]: want %q, got %q", k, want, got) + } + } +} + +func TestGetRules_EnrichesWhenAlertFieldEmpty(t *testing.T) { + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetRulesFunc: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return []k8s.PrometheusRuleGroup{ + { + Name: "group-a", + Rules: []k8s.PrometheusRule{ + { + Name: "EmptyAlertFieldRule", + Type: "alerting", + Query: "up == 0", + Labels: map[string]string{"severity": "warning", k8s.AlertSourceLabel: k8s.AlertSourcePlatform}, + }, + }, + }, + }, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { + return []monitoringv1.Rule{ + { + Alert: "", + Expr: intstr.FromString("up ==\n 0"), + Labels: map[string]string{ + managementlabels.AlertNameLabel: "EmptyAlertFieldRule", + "severity": "critical", + k8s.AlertRuleLabelId: "rid-empty-alert-1", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + }, + }, + } + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(context.Background(), mockK8s) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 1 || len(groups[0].Rules) != 1 { + t.Fatalf("expected 1 group with 1 rule") + } + rule := groups[0].Rules[0] + checks := map[string]string{ + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.AlertRuleLabelId: "rid-empty-alert-1", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + "severity": "critical", + } + for k, want := range checks { + if got := rule.Labels[k]; got != want { + t.Errorf("label[%s]: want %q, got %q", k, want, got) + } + } +} + +func TestGetRules_NoEnrichmentWhenMultipleCandidatesMatch(t *testing.T) { + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetRulesFunc: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return []k8s.PrometheusRuleGroup{ + { + Name: "group-a", + Rules: []k8s.PrometheusRule{ + { + Name: "AmbiguousRule", + Type: "alerting", + Query: "up == 0", + Labels: map[string]string{"severity": "warning", k8s.AlertSourceLabel: k8s.AlertSourcePlatform}, + }, + }, + }, + }, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { + return []monitoringv1.Rule{ + { + Alert: "", + Expr: intstr.FromString("up ==\n 0"), + Labels: map[string]string{ + managementlabels.AlertNameLabel: "AmbiguousRule", + "severity": "critical", + k8s.AlertRuleLabelId: "rid-amb-1", + }, + }, + { + Alert: "", + Expr: intstr.FromString("up==0"), + Labels: map[string]string{ + managementlabels.AlertNameLabel: "AmbiguousRule", + "severity": "critical", + k8s.AlertRuleLabelId: "rid-amb-2", + }, + }, + } + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(context.Background(), mockK8s) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 1 || len(groups[0].Rules) != 1 { + t.Fatalf("expected 1 group with 1 rule") + } + rule := groups[0].Rules[0] + if rule.Labels[k8s.AlertSourceLabel] != k8s.AlertSourcePlatform { + t.Errorf("expected source=%s, got %s", k8s.AlertSourcePlatform, rule.Labels[k8s.AlertSourceLabel]) + } + if _, hasId := rule.Labels[k8s.AlertRuleLabelId]; hasId { + t.Errorf("expected no AlertRuleLabelId on ambiguous rule, but found: %s", rule.Labels[k8s.AlertRuleLabelId]) + } + if rule.Labels["severity"] != "warning" { + t.Errorf("expected severity=warning (from original), got %s", rule.Labels["severity"]) + } +} diff --git a/pkg/management/list_rules.go b/pkg/management/list_rules.go new file mode 100644 index 000000000..1b3d354eb --- /dev/null +++ b/pkg/management/list_rules.go @@ -0,0 +1,83 @@ +package management + +import ( + "context" + "sort" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + + "github.com/openshift/monitoring-plugin/pkg/k8s" +) + +func (c *client) ListRules(ctx context.Context, prOptions PrometheusRuleOptions, arOptions AlertRuleOptions, pgOptions PaginationOptions) (ListRulesResult, error) { + if prOptions.Name != "" && prOptions.Namespace == "" { + return ListRulesResult{}, &ValidationError{Message: "namespace is required when prometheusRuleName is specified"} + } + + allRules := c.k8sClient.RelabeledRules().List(ctx) + var filteredRules []monitoringv1.Rule + + for _, rule := range allRules { + if prOptions.Name != "" && prOptions.Namespace != "" { + namespace := rule.Labels[k8s.PrometheusRuleLabelNamespace] + name := rule.Labels[k8s.PrometheusRuleLabelName] + if namespace != prOptions.Namespace || name != prOptions.Name { + continue + } + } + + if !c.matchesAlertRuleFilters(rule, arOptions) { + continue + } + + filteredRules = append(filteredRules, rule) + } + + sort.Slice(filteredRules, func(i, j int) bool { + return filteredRules[i].Labels[k8s.AlertRuleLabelId] < filteredRules[j].Labels[k8s.AlertRuleLabelId] + }) + + if pgOptions.NextToken != "" { + idx := sort.Search(len(filteredRules), func(i int) bool { + return filteredRules[i].Labels[k8s.AlertRuleLabelId] > pgOptions.NextToken + }) + filteredRules = filteredRules[idx:] + } + + var nextToken string + if pgOptions.Limit > 0 && len(filteredRules) > pgOptions.Limit { + nextToken = filteredRules[pgOptions.Limit-1].Labels[k8s.AlertRuleLabelId] + filteredRules = filteredRules[:pgOptions.Limit] + } + + return ListRulesResult{Rules: filteredRules, NextToken: nextToken}, nil +} + +func (c *client) matchesAlertRuleFilters(rule monitoringv1.Rule, arOptions AlertRuleOptions) bool { + // Filter by alert name + if arOptions.Name != "" && string(rule.Alert) != arOptions.Name { + return false + } + + // Filter by source (platform) + if arOptions.Source == k8s.AlertSourcePlatform { + source, exists := rule.Labels[k8s.AlertSourceLabel] + if !exists { + return false + } + + return source == k8s.AlertSourcePlatform + } + + // Filter by labels + if len(arOptions.Labels) > 0 { + for key, value := range arOptions.Labels { + ruleValue, exists := rule.Labels[key] + if !exists || ruleValue != value { + return false + } + } + } + + return true +} diff --git a/pkg/management/list_rules_test.go b/pkg/management/list_rules_test.go new file mode 100644 index 000000000..d3564d382 --- /dev/null +++ b/pkg/management/list_rules_test.go @@ -0,0 +1,444 @@ +package management_test + +import ( + "context" + "errors" + "testing" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management" + "github.com/openshift/monitoring-plugin/pkg/management/testutils" +) + +var ( + lrRule1 = monitoringv1.Rule{ + Alert: "Alert1", + Expr: intstr.FromString("up == 0"), + Labels: map[string]string{ + "severity": "warning", + k8s.PrometheusRuleLabelNamespace: "namespace1", + k8s.PrometheusRuleLabelName: "rule1", + k8s.AlertRuleLabelId: "rid_aaa", + }, + } + + lrRule2 = monitoringv1.Rule{ + Alert: "Alert2", + Expr: intstr.FromString("up == 0"), + Labels: map[string]string{ + "severity": "critical", + k8s.PrometheusRuleLabelNamespace: "namespace1", + k8s.PrometheusRuleLabelName: "rule2", + k8s.AlertRuleLabelId: "rid_bbb", + }, + } + + lrRule3 = monitoringv1.Rule{ + Alert: "Alert3", + Expr: intstr.FromString("down == 1"), + Labels: map[string]string{ + "severity": "warning", + k8s.PrometheusRuleLabelNamespace: "namespace2", + k8s.PrometheusRuleLabelName: "rule3", + k8s.AlertRuleLabelId: "rid_ccc", + }, + } + + lrPlatformRule = monitoringv1.Rule{ + Alert: "PlatformAlert", + Expr: intstr.FromString("node_down == 1"), + Labels: map[string]string{ + "severity": "critical", + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + k8s.AlertRuleLabelId: "rid_ddd", + }, + } + + lrCustomLabelRule = monitoringv1.Rule{ + Alert: "CustomLabelAlert", + Expr: intstr.FromString("custom == 1"), + Labels: map[string]string{ + "severity": "info", + "team": "backend", + "env": "production", + k8s.PrometheusRuleLabelNamespace: "namespace1", + k8s.PrometheusRuleLabelName: "rule1", + k8s.AlertRuleLabelId: "rid_eee", + }, + } +) + +func newListRulesClient(t *testing.T, rules []monitoringv1.Rule) management.Client { + t.Helper() + mockK8s := &testutils.MockClient{ + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return rules }, + } + }, + } + return management.New(context.Background(), mockK8s) +} + +var allLRRules = []monitoringv1.Rule{lrRule1, lrRule2, lrRule3, lrPlatformRule, lrCustomLabelRule} +var noPagination = management.PaginationOptions{} + +func TestListRules_MissingNamespaceReturnsValidationError(t *testing.T) { + client := newListRulesClient(t, allLRRules) + _, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{Name: "rule1"}, + management.AlertRuleOptions{}, + noPagination, + ) + if err == nil { + t.Fatal("expected error, got nil") + } + var ve *management.ValidationError + if !errors.As(err, &ve) { + t.Fatalf("expected ValidationError, got %T: %v", err, err) + } + if !containsSubstring(err.Error(), "namespace is required when prometheusRuleName is specified") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestListRules_NoFiltersReturnsAll(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 5 { + t.Errorf("expected 5 rules, got %d", len(result.Rules)) + } + if result.NextToken != "" { + t.Errorf("expected no next token, got %q", result.NextToken) + } +} + +func TestListRules_FilterByNameAndNamespace(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{Name: "rule1", Namespace: "namespace1"}, + management.AlertRuleOptions{}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 2 { + t.Fatalf("expected 2 rules, got %d", len(result.Rules)) + } + for _, r := range result.Rules { + if r.Alert != "Alert1" && r.Alert != "CustomLabelAlert" { + t.Errorf("unexpected rule: %s", r.Alert) + } + } +} + +func TestListRules_FilterByNameAndNamespace_NoMatch(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{Name: "nonexistent", Namespace: "namespace1"}, + management.AlertRuleOptions{}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 0 { + t.Errorf("expected 0 rules, got %d", len(result.Rules)) + } +} + +func TestListRules_FilterByAlertName(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{Name: "Alert1"}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 1 || result.Rules[0].Alert != "Alert1" { + t.Errorf("expected 1 rule Alert1, got %v", result.Rules) + } +} + +func TestListRules_FilterByAlertName_NoMatch(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{Name: "NonexistentAlert"}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 0 { + t.Errorf("expected 0 rules, got %d", len(result.Rules)) + } +} + +func TestListRules_FilterBySourcePlatform(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{Source: k8s.AlertSourcePlatform}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 1 { + t.Fatalf("expected 1 rule, got %d", len(result.Rules)) + } + if result.Rules[0].Alert != "PlatformAlert" { + t.Errorf("expected PlatformAlert, got %s", result.Rules[0].Alert) + } + if result.Rules[0].Labels[k8s.AlertSourceLabel] != k8s.AlertSourcePlatform { + t.Errorf("expected source=platform, got %s", result.Rules[0].Labels[k8s.AlertSourceLabel]) + } +} + +func TestListRules_FilterBySingleLabel(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{Labels: map[string]string{"severity": "warning"}}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 2 { + t.Errorf("expected 2 warning rules, got %d", len(result.Rules)) + } +} + +func TestListRules_FilterByMultipleLabels(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{Labels: map[string]string{"team": "backend", "env": "production"}}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 1 || result.Rules[0].Alert != "CustomLabelAlert" { + t.Errorf("expected 1 CustomLabelAlert, got %v", result.Rules) + } +} + +func TestListRules_FilterByLabels_NoMatch(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{Labels: map[string]string{"nonexistent": "value"}}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 0 { + t.Errorf("expected 0 rules, got %d", len(result.Rules)) + } +} + +func TestListRules_CombinedFilters(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{Name: "rule1", Namespace: "namespace1"}, + management.AlertRuleOptions{Labels: map[string]string{"severity": "warning"}}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 1 || result.Rules[0].Alert != "Alert1" { + t.Errorf("expected 1 Alert1, got %v", result.Rules) + } +} + +func TestListRules_CombinedFilters_NoMatch(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{Name: "rule1", Namespace: "namespace1"}, + management.AlertRuleOptions{Labels: map[string]string{"severity": "critical"}}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 0 { + t.Errorf("expected 0 rules, got %d", len(result.Rules)) + } +} + +func TestListRules_EmptyRelabeledRules(t *testing.T) { + client := newListRulesClient(t, []monitoringv1.Rule{}) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 0 { + t.Errorf("expected 0 rules, got %d", len(result.Rules)) + } +} + +func TestListRules_Pagination_FirstPage(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 2}, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 2 { + t.Fatalf("expected 2 rules, got %d", len(result.Rules)) + } + if result.NextToken == "" { + t.Error("expected next token, got empty") + } + if result.Rules[0].Labels[k8s.AlertRuleLabelId] != "rid_aaa" { + t.Errorf("expected rid_aaa, got %s", result.Rules[0].Labels[k8s.AlertRuleLabelId]) + } + if result.Rules[1].Labels[k8s.AlertRuleLabelId] != "rid_bbb" { + t.Errorf("expected rid_bbb, got %s", result.Rules[1].Labels[k8s.AlertRuleLabelId]) + } +} + +func TestListRules_Pagination_SecondPage(t *testing.T) { + client := newListRulesClient(t, allLRRules) + first, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 2}, + ) + if err != nil { + t.Fatalf("unexpected error on first page: %v", err) + } + + second, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 2, NextToken: first.NextToken}, + ) + if err != nil { + t.Fatalf("unexpected error on second page: %v", err) + } + if len(second.Rules) != 2 { + t.Fatalf("expected 2 rules on second page, got %d", len(second.Rules)) + } + if second.Rules[0].Labels[k8s.AlertRuleLabelId] != "rid_ccc" { + t.Errorf("expected rid_ccc, got %s", second.Rules[0].Labels[k8s.AlertRuleLabelId]) + } + if second.Rules[1].Labels[k8s.AlertRuleLabelId] != "rid_ddd" { + t.Errorf("expected rid_ddd, got %s", second.Rules[1].Labels[k8s.AlertRuleLabelId]) + } + if second.NextToken == "" { + t.Error("expected next token for third page") + } +} + +func TestListRules_Pagination_LastPageNoNextToken(t *testing.T) { + client := newListRulesClient(t, allLRRules) + first, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 2}, + ) + if err != nil { + t.Fatalf("page 1 error: %v", err) + } + second, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 2, NextToken: first.NextToken}, + ) + if err != nil { + t.Fatalf("page 2 error: %v", err) + } + third, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 2, NextToken: second.NextToken}, + ) + if err != nil { + t.Fatalf("page 3 error: %v", err) + } + if len(third.Rules) != 1 { + t.Errorf("expected 1 rule on last page, got %d", len(third.Rules)) + } + if third.NextToken != "" { + t.Errorf("expected no next token on last page, got %q", third.NextToken) + } +} + +func TestListRules_Pagination_LimitExceedsTotal(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 100}, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 5 { + t.Errorf("expected 5 rules, got %d", len(result.Rules)) + } + if result.NextToken != "" { + t.Errorf("expected no next token, got %q", result.NextToken) + } +} + +func TestListRules_Pagination_SortedByRuleId(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for i := 1; i < len(result.Rules); i++ { + prev := result.Rules[i-1].Labels[k8s.AlertRuleLabelId] + curr := result.Rules[i].Labels[k8s.AlertRuleLabelId] + if prev >= curr { + t.Errorf("rules not sorted: %s >= %s at index %d", prev, curr, i) + } + } +} + +// containsSubstring is a local helper to avoid importing strings in test files +// that don't otherwise need it. +func containsSubstring(s, sub string) bool { + if len(sub) == 0 { + return true + } + for i := 0; i <= len(s)-len(sub); i++ { + if s[i:i+len(sub)] == sub { + return true + } + } + return false +} diff --git a/pkg/management/types.go b/pkg/management/types.go index a694c3ff3..dfa31a32e 100644 --- a/pkg/management/types.go +++ b/pkg/management/types.go @@ -49,8 +49,13 @@ type Client interface { // BulkUpdateAlertRuleClassification updates classification for multiple rule ids BulkUpdateAlertRuleClassification(ctx context.Context, items []UpdateRuleClassificationRequest) []error + // ListRules lists alert rules, optionally paginated via cursor-based pagination + ListRules(ctx context.Context, prOptions PrometheusRuleOptions, arOptions AlertRuleOptions, pgOptions PaginationOptions) (ListRulesResult, error) + // GetAlerts retrieves Prometheus alerts GetAlerts(ctx context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) + // GetRules retrieves Prometheus alerting rules and active alerts + GetRules(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) // GetAlertingHealth retrieves the alerting stack health status GetAlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) @@ -67,3 +72,31 @@ type PrometheusRuleOptions struct { // GroupName of the RuleGroup within the PrometheusRule resource GroupName string `json:"groupName"` } + +// AlertRuleOptions specifies additional filtering options for alert rules +type AlertRuleOptions struct { + // Name filters alert rules by alert name + Name string `json:"name,omitempty"` + + // Source filters alert rules by source type (platform or user-defined) + Source string `json:"source,omitempty"` + + // Labels filters alert rules by arbitrary label key-value pairs + Labels map[string]string `json:"labels,omitempty"` +} + +// PaginationOptions controls cursor-based pagination for list endpoints. +type PaginationOptions struct { + // Limit is the maximum number of results to return. Zero means no limit. + Limit int + + // NextToken is an opaque cursor returned by a previous call; results will + // start after the rule identified by this token. + NextToken string +} + +// ListRulesResult holds a page of rules and an optional cursor for the next page. +type ListRulesResult struct { + Rules []monitoringv1.Rule `json:"rules"` + NextToken string `json:"nextToken,omitempty"` +} diff --git a/test/e2e/relabeled_rules_test.go b/test/e2e/relabeled_rules_test.go new file mode 100644 index 000000000..d5087ed38 --- /dev/null +++ b/test/e2e/relabeled_rules_test.go @@ -0,0 +1,443 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "testing" + "time" + + osmv1 "github.com/openshift/api/monitoring/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/test/e2e/framework" +) + +type listRulesRuleGroup struct { + Name string `json:"name"` + Rules []monitoringv1.Rule `json:"rules"` +} + +type listRulesResponse struct { + Data struct { + Groups []listRulesRuleGroup `json:"groups"` + } `json:"data"` +} + +func listRules(ctx context.Context, f *framework.Framework) ([]monitoringv1.Rule, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, f.PluginURL+"/api/v1/alerting/rules", nil) + if err != nil { + return nil, err + } + if f.BearerToken != "" { + req.Header.Set("Authorization", "Bearer "+f.BearerToken) + } + + resp, err := f.HTTPClient().Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + var listResp listRulesResponse + if err := json.NewDecoder(resp.Body).Decode(&listResp); err != nil { + return nil, err + } + + var allRules []monitoringv1.Rule + for _, group := range listResp.Data.Groups { + allRules = append(allRules, group.Rules...) + } + return allRules, nil +} + +func TestPrometheusRuleAppearsInMemory(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + testNamespace, cleanup, err := f.CreateUserNamespace(ctx, "test-prometheus-rule") + if err != nil { + t.Fatalf("Failed to create test namespace: %v", err) + } + defer cleanup() + + testAlertName := "TestAlert" + forDuration := monitoringv1.Duration("5m") + testRule := monitoringv1.Rule{ + Alert: testAlertName, + Expr: intstr.FromString("up == 0"), + For: &forDuration, + Labels: map[string]string{ + "severity": "warning", + }, + Annotations: map[string]string{ + "description": "Test alert for e2e testing", + "summary": "Test alert", + }, + } + + _, err = createPrometheusRule(ctx, f, testNamespace, testRule) + if err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + rules, err := listRules(ctx, f) + if err != nil { + t.Logf("Failed to list rules: %v", err) + return false, nil + } + + for _, rule := range rules { + if rule.Alert == testAlertName { + expectedLabels := map[string]string{ + k8s.PrometheusRuleLabelNamespace: testNamespace, + k8s.PrometheusRuleLabelName: "test-prometheus-rule", + } + + if err := compareRuleLabels(t, testAlertName, rule.Labels, expectedLabels); err != nil { + return false, err + } + + if _, ok := rule.Labels[k8s.AlertRuleLabelId]; !ok { + t.Errorf("Alert %s missing openshift_io_alert_rule_id label", testAlertName) + return false, fmt.Errorf("alert missing openshift_io_alert_rule_id label") + } + + t.Logf("Found alert %s in memory with all expected labels", testAlertName) + return true, nil + } + } + + t.Logf("Alert %s not found in memory yet (found %d rules)", testAlertName, len(rules)) + return false, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for alert to appear in memory: %v", err) + } +} + +func TestRelabelAlert(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + testNamespace, cleanup, err := f.CreatePlatformNamespace(ctx, "test-relabel-alert") + if err != nil { + t.Fatalf("Failed to create test namespace: %v", err) + } + defer cleanup() + + forDuration := monitoringv1.Duration("5m") + + criticalRule := monitoringv1.Rule{ + Alert: "TestRelabelAlert", + Expr: intstr.FromString("up == 0"), + For: &forDuration, + Labels: map[string]string{ + "severity": "critical", + "team": "web", + }, + Annotations: map[string]string{ + "description": "Critical alert for relabel testing", + "summary": "Critical test alert", + }, + } + + warningRule := monitoringv1.Rule{ + Alert: "TestRelabelAlert", + Expr: intstr.FromString("up == 1"), + For: &forDuration, + Labels: map[string]string{ + "severity": "warning", + "team": "web", + }, + Annotations: map[string]string{ + "description": "Warning alert for relabel testing", + "summary": "Warning test alert", + }, + } + + _, err = createPrometheusRule(ctx, f, testNamespace, criticalRule, warningRule) + if err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + relabelConfigName := "change-critical-team" + arc := &osmv1.AlertRelabelConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: relabelConfigName, + Namespace: k8s.ClusterMonitoringNamespace, + }, + Spec: osmv1.AlertRelabelConfigSpec{ + Configs: []osmv1.RelabelConfig{ + { + SourceLabels: []osmv1.LabelName{"alertname", "severity"}, + Regex: "TestRelabelAlert;critical", + Separator: ";", + TargetLabel: "team", + Replacement: "ops", + Action: "Replace", + }, + }, + }, + } + + _, err = f.Osmv1clientset.MonitoringV1().AlertRelabelConfigs(k8s.ClusterMonitoringNamespace).Create( + ctx, arc, metav1.CreateOptions{}, + ) + if err != nil { + t.Fatalf("Failed to create AlertRelabelConfig: %v", err) + } + defer func() { + err = f.Osmv1clientset.MonitoringV1().AlertRelabelConfigs(k8s.ClusterMonitoringNamespace).Delete(ctx, relabelConfigName, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Failed to delete AlertRelabelConfig: %v", err) + } + }() + + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + rules, err := listRules(ctx, f) + if err != nil { + t.Logf("Failed to list rules: %v", err) + return false, nil + } + + foundCriticalWithOps := false + + for _, rule := range rules { + if rule.Alert == "TestRelabelAlert" { + if rule.Labels["team"] == "ops" && rule.Labels["severity"] == "critical" { + t.Logf("Found critical alert with team=ops (relabeling successful)") + foundCriticalWithOps = true + } + } + } + + if foundCriticalWithOps { + t.Logf("Relabeling verified: critical alert has team=ops") + return true, nil + } + + t.Logf("Waiting for relabeling to take effect") + return false, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for relabeling to take effect: %v", err) + } +} + +func createPrometheusRule(ctx context.Context, f *framework.Framework, namespace string, rules ...monitoringv1.Rule) (*monitoringv1.PrometheusRule, error) { + interval := monitoringv1.Duration("30s") + prometheusRule := &monitoringv1.PrometheusRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-prometheus-rule", + Namespace: namespace, + }, + Spec: monitoringv1.PrometheusRuleSpec{ + Groups: []monitoringv1.RuleGroup{ + { + Name: "test-group", + Interval: &interval, + Rules: rules, + }, + }, + }, + } + + return f.Monitoringv1clientset.MonitoringV1().PrometheusRules(namespace).Create( + ctx, prometheusRule, metav1.CreateOptions{}, + ) +} + +func compareRuleLabels(t *testing.T, alertName string, foundLabels map[string]string, wantedLabels map[string]string) error { + t.Helper() + if foundLabels == nil { + t.Errorf("Alert %s has no labels", alertName) + return fmt.Errorf("alert has no labels") + } + + for key, wantValue := range wantedLabels { + if gotValue, ok := foundLabels[key]; !ok { + t.Errorf("Alert %s missing %s label", alertName, key) + return fmt.Errorf("alert missing %s label", key) + } else if gotValue != wantValue { + t.Errorf("Alert %s has wrong %s label. Expected %s, got %s", + alertName, key, wantValue, gotValue) + return fmt.Errorf("alert has wrong %s label", key) + } + } + + return nil +} + +// TestRBAC_GetRules verifies Thanos-tenancy RBAC for GET /rules. +// +// With ?namespace=: User A (no perms) gets HTTP 200 without the UWM rule in +// ns Y; User B (monitoring-rules-view in Y) sees Y but not Z; cluster-admin +// sees Y. +// +// Without ?namespace=: fan-out must not leak the rule to unprivileged users +// and must still return it for namespace-scoped viewers. +func TestRBAC_GetRules(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + nsY, cleanupY, err := f.CreateUserNamespace(ctx, "test-rbac-get-rules-y") + if err != nil { + t.Fatalf("Failed to create namespace Y: %v", err) + } + defer func() { _ = cleanupY() }() + + nsZ, cleanupZ, err := f.CreateUserNamespace(ctx, "test-rbac-get-rules-z") + if err != nil { + t.Fatalf("Failed to create namespace Z: %v", err) + } + defer func() { _ = cleanupZ() }() + + userA, err := f.CreateAnonymousUser(ctx, "e2e-rbac-rules-a", "default") + if err != nil { + t.Fatalf("Failed to create unprivileged user A: %v", err) + } + defer func() { _ = userA.Cleanup() }() + + userB, err := f.CreateUserWithClusterRole(ctx, "e2e-rbac-rules-b", nsY, "monitoring-rules-view") + if err != nil { + t.Fatalf("Failed to create scoped user B: %v", err) + } + defer func() { _ = userB.Cleanup() }() + + alertName := "E2ERBACGetRulesTest" + forDuration := monitoringv1.Duration("5m") + testRule := monitoringv1.Rule{ + Alert: alertName, + Expr: intstr.FromString("vector(1)"), + For: &forDuration, + Labels: map[string]string{ + "severity": "none", + "e2e_test": "rbac_get_rules", + }, + } + + _, err = createPrometheusRule(ctx, f, nsY, testRule) + if err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + rules, status, err := listRulesWithToken(ctx, f, f.BearerToken, nsY) + if err != nil { + t.Logf("Admin GET /rules failed: %v", err) + return false, nil + } + if status != http.StatusOK { + t.Logf("Admin GET /rules returned status %d, retrying", status) + return false, nil + } + if containsRule(rules, alertName) { + return true, nil + } + t.Logf("Waiting for rule %s (admin sees %d rules)", alertName, len(rules)) + return false, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for admin to see rule: %v", err) + } + + cases := []struct { + name string + token string + namespace string + wantRule bool + }{ + {"UserA_NoPerms_NamespaceY", userA.Token, nsY, false}, + {"UserA_NoPerms_NoNamespace", userA.Token, "", false}, + {"UserB_RulesView_NamespaceY", userB.Token, nsY, true}, + {"UserB_RulesView_NamespaceZ", userB.Token, nsZ, false}, + {"UserB_RulesView_NoNamespace", userB.Token, "", true}, + {"UserC_ClusterAdmin_NamespaceY", f.BearerToken, nsY, true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rules, status, err := listRulesWithToken(ctx, f, tc.token, tc.namespace) + if err != nil { + t.Fatalf("GET /rules request failed: %v", err) + } + if status != http.StatusOK { + t.Fatalf("Expected status %d, got %d", http.StatusOK, status) + } + got := containsRule(rules, alertName) + if got != tc.wantRule { + t.Fatalf("Rule %s visibility: want %v, got %v (%d rules returned)", alertName, tc.wantRule, got, len(rules)) + } + }) + } +} + +func containsRule(rules []monitoringv1.Rule, alertName string) bool { + for _, r := range rules { + if r.Alert == alertName { + return true + } + } + return false +} + +// listRulesWithToken calls GET /rules with an optional namespace query param. +// A non-OK status is not an error — callers must assert on status explicitly. +func listRulesWithToken(ctx context.Context, f *framework.Framework, token, namespace string) ([]monitoringv1.Rule, int, error) { + rulesURL := f.PluginURL + "/api/v1/alerting/rules" + if namespace != "" { + rulesURL += "?" + url.Values{"namespace": {namespace}}.Encode() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rulesURL, nil) + if err != nil { + return nil, 0, err + } + req.Header.Set("Authorization", "Bearer "+token) + + resp, err := f.HTTPClient().Do(req) + if err != nil { + return nil, 0, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, resp.StatusCode, nil + } + + var listResp listRulesResponse + if err := json.NewDecoder(resp.Body).Decode(&listResp); err != nil { + return nil, resp.StatusCode, err + } + + var allRules []monitoringv1.Rule + for _, group := range listResp.Data.Groups { + allRules = append(allRules, group.Rules...) + } + return allRules, resp.StatusCode, nil +} From 1fbff5c5cbef021bebb011ffa6d3353cef0e33e8 Mon Sep 17 00:00:00 2001 From: Shirly Radco Date: Thu, 12 Mar 2026 20:34:26 +0200 Subject: [PATCH 3/5] router: add GET /health endpoint and tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add GET /api/v1/alerting/health endpoint with handler tests and rules GET tests. Signed-off-by: Shirly Radco Signed-off-by: João Vilaça Signed-off-by: Aviv Litman Co-authored-by: AI Assistant --- internal/managementrouter/health_get.go | 32 +++ internal/managementrouter/health_get_test.go | 264 +++++++++++++++++++ internal/managementrouter/router.go | 6 +- pkg/management/types.go | 3 +- test/e2e/health_test.go | 57 ++++ 5 files changed, 358 insertions(+), 4 deletions(-) create mode 100644 internal/managementrouter/health_get.go create mode 100644 internal/managementrouter/health_get_test.go create mode 100644 test/e2e/health_test.go diff --git a/internal/managementrouter/health_get.go b/internal/managementrouter/health_get.go new file mode 100644 index 000000000..2db846c85 --- /dev/null +++ b/internal/managementrouter/health_get.go @@ -0,0 +1,32 @@ +package managementrouter + +import ( + "encoding/json" + "net/http" + + "github.com/openshift/monitoring-plugin/pkg/k8s" +) + +type GetHealthResponse struct { + Alerting *k8s.AlertingHealth `json:"alerting,omitempty"` +} + +func (hr *httpRouter) GetHealth(w http.ResponseWriter, r *http.Request) { + resp := GetHealthResponse{} + + if hr.managementClient != nil { + health, err := hr.managementClient.GetAlertingHealth(r.Context()) + if err != nil { + handleError(w, err) + return + } + resp.Alerting = &health + } + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + log.WithError(err).Warn("failed to encode health response") + } +} diff --git a/internal/managementrouter/health_get_test.go b/internal/managementrouter/health_get_test.go new file mode 100644 index 000000000..a6be1046f --- /dev/null +++ b/internal/managementrouter/health_get_test.go @@ -0,0 +1,264 @@ +package managementrouter_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + + "github.com/openshift/monitoring-plugin/internal/managementrouter" + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management" +) + +// stubClient is a configurable stub implementing management.Client. +// Fields are set per-test; all methods default to no-op returns. +type stubClient struct { + getRules func(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) + alertingHealth func(ctx context.Context) (k8s.AlertingHealth, error) +} + +func (s *stubClient) ListRules(_ context.Context, _ management.PrometheusRuleOptions, _ management.AlertRuleOptions, _ management.PaginationOptions) (management.ListRulesResult, error) { + return management.ListRulesResult{}, nil +} +func (s *stubClient) GetRuleById(_ context.Context, _ string) (monitoringv1.Rule, error) { + return monitoringv1.Rule{}, nil +} +func (s *stubClient) CreateUserDefinedAlertRule(_ context.Context, _ monitoringv1.Rule, _ management.PrometheusRuleOptions) (string, error) { + return "", nil +} +func (s *stubClient) CreatePlatformAlertRule(_ context.Context, _ monitoringv1.Rule) (string, error) { + return "", nil +} +func (s *stubClient) UpdateUserDefinedAlertRule(_ context.Context, _ string, _ monitoringv1.Rule) (string, error) { + return "", nil +} +func (s *stubClient) DeleteAlertRuleById(_ context.Context, _ string) error { return nil } +func (s *stubClient) UpdatePlatformAlertRule(_ context.Context, _ string, _ monitoringv1.Rule) error { + return nil +} +func (s *stubClient) DropAlertRule(_ context.Context, _ string) error { return nil } +func (s *stubClient) RestoreAlertRule(_ context.Context, _ string) error { return nil } +func (s *stubClient) UpdateAlertRuleLabels(_ context.Context, _ string, _ map[string]*string) (string, error) { + return "", nil +} +func (s *stubClient) GetAlerts(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return nil, nil +} +func (s *stubClient) GetRules(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + if s.getRules != nil { + return s.getRules(ctx, req) + } + return []k8s.PrometheusRuleGroup{}, nil +} +func (s *stubClient) GetAlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) { + if s.alertingHealth != nil { + return s.alertingHealth(ctx) + } + return k8s.AlertingHealth{}, nil +} +func (s *stubClient) UpdateAlertRuleClassification(_ context.Context, _ management.UpdateRuleClassificationRequest) error { + return nil +} +func (s *stubClient) BulkUpdateAlertRuleClassification(_ context.Context, _ []management.UpdateRuleClassificationRequest) []error { + return nil +} + +// newStubRouter builds a router backed by stub and adds a Bearer token header +// to requests via the helper get/getNoAuth methods. +func newStubRouter(stub *stubClient) http.Handler { + return managementrouter.New(stub) +} + +func stubGet(router http.Handler, url string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, url, nil) + req.Header.Set("Authorization", "Bearer test-token") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + return w +} + +// --- health_get tests --- + +func healthStub() *stubClient { + return &stubClient{ + alertingHealth: func(_ context.Context) (k8s.AlertingHealth, error) { + return k8s.AlertingHealth{ + Platform: &k8s.AlertingStackHealth{ + Prometheus: k8s.AlertingRouteHealth{Name: "prometheus-k8s", Namespace: "openshift-monitoring", Status: k8s.RouteReachable}, + Alertmanager: k8s.AlertingRouteHealth{Name: "alertmanager-main", Namespace: "openshift-monitoring", Status: k8s.RouteReachable}, + }, + UserWorkloadEnabled: true, + UserWorkload: &k8s.AlertingStackHealth{ + Prometheus: k8s.AlertingRouteHealth{Name: "prometheus-user-workload", Namespace: "openshift-user-workload-monitoring", Status: k8s.RouteReachable}, + Alertmanager: k8s.AlertingRouteHealth{Name: "alertmanager-user-workload", Namespace: "openshift-user-workload-monitoring", Status: k8s.RouteReachable}, + }, + }, nil + }, + } +} + +func TestGetHealth_Returns200(t *testing.T) { + router := newStubRouter(healthStub()) + w := stubGet(router, "/api/v1/alerting/health") + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } +} + +func TestGetHealth_ReturnsAlertingStructure(t *testing.T) { + router := newStubRouter(healthStub()) + w := stubGet(router, "/api/v1/alerting/health") + + var response managementrouter.GetHealthResponse + if err := json.NewDecoder(w.Body).Decode(&response); err != nil { + t.Fatalf("decode error: %v", err) + } + if response.Alerting == nil { + t.Error("expected non-nil Alerting in response") + } +} + +func TestGetHealth_Returns500OnError(t *testing.T) { + stub := &stubClient{ + alertingHealth: func(_ context.Context) (k8s.AlertingHealth, error) { + return k8s.AlertingHealth{}, fmt.Errorf("connection refused") + }, + } + router := newStubRouter(stub) + w := stubGet(router, "/api/v1/alerting/health") + + if w.Code != http.StatusInternalServerError { + t.Fatalf("expected 500, got %d: %s", w.Code, w.Body) + } + var errResp map[string]string + if err := json.NewDecoder(w.Body).Decode(&errResp); err != nil { + t.Fatalf("decode error: %v", err) + } + if errResp["error"] != "An unexpected error occurred" { + t.Errorf("unexpected error message: %q", errResp["error"]) + } +} + +// --- rules_get tests --- + +func TestGetRules_ParsesFlatQueryParams(t *testing.T) { + stub := &stubClient{} + var captured k8s.GetRulesRequest + stub.getRules = func(_ context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + captured = req + return []k8s.PrometheusRuleGroup{}, nil + } + router := newStubRouter(stub) + w := stubGet(router, "/api/v1/alerting/rules?namespace=ns1&severity=critical&state=firing&team=sre") + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + if captured.State != "firing" { + t.Errorf("expected state=firing, got %q", captured.State) + } + for k, want := range map[string]string{"namespace": "ns1", "severity": "critical", "team": "sre"} { + if got := captured.Labels[k]; got != want { + t.Errorf("label[%s]: want %q, got %q", k, want, got) + } + } +} + +func TestGetRules_ReturnsGroupsInResponse(t *testing.T) { + stub := &stubClient{ + getRules: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return []k8s.PrometheusRuleGroup{{Name: "group-a"}}, nil + }, + } + router := newStubRouter(stub) + w := stubGet(router, "/api/v1/alerting/rules") + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + if ct := w.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("expected application/json, got %q", ct) + } + var response managementrouter.GetRulesResponse + if err := json.NewDecoder(w.Body).Decode(&response); err != nil { + t.Fatalf("decode error: %v", err) + } + if len(response.Data.Groups) != 1 || response.Data.Groups[0].Name != "group-a" { + t.Errorf("unexpected groups: %v", response.Data.Groups) + } +} + +func TestGetRules_WarningWhenUserWorkloadPromRouteMissing(t *testing.T) { + stub := &stubClient{ + alertingHealth: func(_ context.Context) (k8s.AlertingHealth, error) { + return k8s.AlertingHealth{ + UserWorkloadEnabled: true, + UserWorkload: &k8s.AlertingStackHealth{ + Prometheus: k8s.AlertingRouteHealth{Status: k8s.RouteNotFound}, + }, + }, nil + }, + } + router := newStubRouter(stub) + w := stubGet(router, "/api/v1/alerting/rules") + + var response managementrouter.GetRulesResponse + if err := json.NewDecoder(w.Body).Decode(&response); err != nil { + t.Fatalf("decode error: %v", err) + } + found := false + for _, warn := range response.Warnings { + if warn == "user workload Prometheus route is missing" { + found = true + break + } + } + if !found { + t.Errorf("expected Prometheus route warning, got: %v", response.Warnings) + } +} + +func TestGetRules_SuppressesWarningWhenFallbackHealthy(t *testing.T) { + stub := &stubClient{ + alertingHealth: func(_ context.Context) (k8s.AlertingHealth, error) { + return k8s.AlertingHealth{ + UserWorkloadEnabled: true, + UserWorkload: &k8s.AlertingStackHealth{ + Prometheus: k8s.AlertingRouteHealth{Status: k8s.RouteUnreachable, FallbackReachable: true}, + }, + }, nil + }, + } + router := newStubRouter(stub) + w := stubGet(router, "/api/v1/alerting/rules") + + var response managementrouter.GetRulesResponse + if err := json.NewDecoder(w.Body).Decode(&response); err != nil { + t.Fatalf("decode error: %v", err) + } + if len(response.Warnings) != 0 { + t.Errorf("expected no warnings, got: %v", response.Warnings) + } +} + +func TestGetRules_Returns500OnError(t *testing.T) { + stub := &stubClient{ + getRules: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return nil, fmt.Errorf("connection error") + }, + } + router := newStubRouter(stub) + w := stubGet(router, "/api/v1/alerting/rules") + + if w.Code != http.StatusInternalServerError { + t.Fatalf("expected 500, got %d: %s", w.Code, w.Body) + } + if body := w.Body.String(); !containsStr(body, "An unexpected error occurred") { + t.Errorf("expected error message in body, got: %s", body) + } +} diff --git a/internal/managementrouter/router.go b/internal/managementrouter/router.go index f0ac5cfb8..915b62f8c 100644 --- a/internal/managementrouter/router.go +++ b/internal/managementrouter/router.go @@ -43,10 +43,12 @@ func New(managementClient management.Client) *mux.Router { BaseURL: "/api/v1/alerting", BaseRouter: r, }) - // GET /alerts and GET /rules are not yet in the OpenAPI spec; registered - // manually until their respective branches add the spec entries. + // GET /alerts, GET /rules, and GET /health are not yet in the OpenAPI + // spec; registered manually until their respective branches add the spec + // entries. r.HandleFunc("/api/v1/alerting/alerts", hr.GetAlerts).Methods(http.MethodGet) r.HandleFunc("/api/v1/alerting/rules", hr.GetRules).Methods(http.MethodGet) + r.HandleFunc("/api/v1/alerting/health", hr.GetHealth).Methods(http.MethodGet) return r } diff --git a/pkg/management/types.go b/pkg/management/types.go index dfa31a32e..8c431f2bd 100644 --- a/pkg/management/types.go +++ b/pkg/management/types.go @@ -57,7 +57,7 @@ type Client interface { // GetRules retrieves Prometheus alerting rules and active alerts GetRules(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) - // GetAlertingHealth retrieves the alerting stack health status + // GetAlertingHealth retrieves alerting health details GetAlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) } @@ -73,7 +73,6 @@ type PrometheusRuleOptions struct { GroupName string `json:"groupName"` } -// AlertRuleOptions specifies additional filtering options for alert rules type AlertRuleOptions struct { // Name filters alert rules by alert name Name string `json:"name,omitempty"` diff --git a/test/e2e/health_test.go b/test/e2e/health_test.go new file mode 100644 index 000000000..d0485293e --- /dev/null +++ b/test/e2e/health_test.go @@ -0,0 +1,57 @@ +package e2e + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/test/e2e/framework" +) + +func TestGetHealth(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + healthURL := f.PluginURL + "/api/v1/alerting/health" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthURL, nil) + if err != nil { + t.Fatalf("Failed to create HTTP request: %v", err) + } + if f.BearerToken != "" { + req.Header.Set("Authorization", "Bearer "+f.BearerToken) + } + + resp, err := f.HTTPClient().Do(req) + if err != nil { + t.Fatalf("Failed to make health request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("Expected status 200, got %d", resp.StatusCode) + } + + var healthResp struct { + Alerting *k8s.AlertingHealth `json:"alerting"` + } + if err := json.NewDecoder(resp.Body).Decode(&healthResp); err != nil { + t.Fatalf("Failed to decode health response: %v", err) + } + + if healthResp.Alerting == nil { + t.Fatal("Expected 'alerting' field in health response") + } + + if healthResp.Alerting.Platform == nil { + t.Error("Expected 'platform' field in alerting health") + } + + t.Logf("Health response: userWorkloadEnabled=%v", healthResp.Alerting.UserWorkloadEnabled) + t.Log("GET /health e2e test passed successfully") +} From 0d89ed96b4b6c8dc665220d5ea9e21a670d41385 Mon Sep 17 00:00:00 2001 From: Shirly Radco Date: Thu, 12 Mar 2026 20:42:50 +0200 Subject: [PATCH 4/5] k8s: add orphan AlertRelabelConfig GC Detect and remove orphan AlertRelabelConfig resources that no longer have a matching PrometheusRule, preventing stale relabel configs from accumulating. Signed-off-by: Shirly Radco Co-authored-by: AI Assistant --- go.mod | 4 +- pkg/k8s/alert_relabel_config_gc.go | 52 ++++ pkg/k8s/alert_relabel_config_gc_test.go | 168 +++++++++++ pkg/k8s/relabeled_rules.go | 19 +- pkg/metrics/alerts_collector.go | 223 ++++++++++++++ pkg/metrics/alerts_collector_test.go | 354 +++++++++++++++++++++++ pkg/metrics/leader_election.go | 87 ++++++ test/e2e/alerts_effective_metric_test.go | 315 ++++++++++++++++++++ 8 files changed, 1214 insertions(+), 8 deletions(-) create mode 100644 pkg/k8s/alert_relabel_config_gc.go create mode 100644 pkg/k8s/alert_relabel_config_gc_test.go create mode 100644 pkg/metrics/alerts_collector.go create mode 100644 pkg/metrics/alerts_collector_test.go create mode 100644 pkg/metrics/leader_election.go create mode 100644 test/e2e/alerts_effective_metric_test.go diff --git a/go.mod b/go.mod index bcc7afc8a..e57ab90e6 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,8 @@ require ( github.com/openshift/library-go v0.0.0-20240905123346-5bdbfe35a6f5 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.87.0 github.com/prometheus-operator/prometheus-operator/pkg/client v0.87.0 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.67.4 github.com/prometheus/prometheus v0.308.0 github.com/sirupsen/logrus v1.9.3 @@ -57,8 +59,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect - github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/pkg/k8s/alert_relabel_config_gc.go b/pkg/k8s/alert_relabel_config_gc.go new file mode 100644 index 000000000..7c9e92a2e --- /dev/null +++ b/pkg/k8s/alert_relabel_config_gc.go @@ -0,0 +1,52 @@ +package k8s + +import ( + "context" + + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +// gcOrphanedARCs deletes AlertRelabelConfigs whose associated alert rule no +// longer exists. This handles the case where an operator (or manual action) +// removes rules from a PrometheusRule or deletes the CR entirely — the ARCs +// that were created by the plugin for classification/drop/stamp become orphans. +// +// Only ARCs carrying the plugin's alertRuleId annotation are considered. +// GitOps-managed ARCs are never deleted automatically; a warning is logged +// so that operators can clean them up manually. +func (rrm *relabeledRulesManager) gcOrphanedARCs(ctx context.Context, liveRuleIDs map[string]struct{}) { + if rrm.alertRelabelConfigs == nil { + return + } + + arcs, err := rrm.alertRelabelConfigs.List(ctx, "") + if err != nil { + log.Errorf("orphan ARC GC: failed to list ARCs: %v", err) + return + } + + for i := range arcs { + arc := &arcs[i] + + ruleID, ok := arc.Annotations[managementlabels.ARCAnnotationAlertRuleIDKey] + if !ok || ruleID == "" { + continue + } + + if _, alive := liveRuleIDs[ruleID]; alive { + continue + } + + if IsManagedByGitOps(arc.Annotations, arc.Labels) { + log.Warnf("orphan ARC GC: ARC %s/%s (ruleId=%s) is orphaned but GitOps-managed — skipping deletion, manual cleanup required", arc.Namespace, arc.Name, ruleID) + continue + } + + if err := rrm.alertRelabelConfigs.Delete(ctx, arc.Namespace, arc.Name); err != nil { + log.Errorf("orphan ARC GC: failed to delete ARC %s/%s: %v", arc.Namespace, arc.Name, err) + continue + } + + log.Infof("orphan ARC GC: deleted orphaned ARC %s/%s (ruleId=%s)", arc.Namespace, arc.Name, ruleID) + } +} diff --git a/pkg/k8s/alert_relabel_config_gc_test.go b/pkg/k8s/alert_relabel_config_gc_test.go new file mode 100644 index 000000000..e139ad965 --- /dev/null +++ b/pkg/k8s/alert_relabel_config_gc_test.go @@ -0,0 +1,168 @@ +package k8s + +import ( + "context" + "testing" + + osmv1 "github.com/openshift/api/monitoring/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +type mockARCInterface struct { + arcs map[string]*osmv1.AlertRelabelConfig + deleted []string +} + +func (m *mockARCInterface) List(_ context.Context, _ string) ([]osmv1.AlertRelabelConfig, error) { + var result []osmv1.AlertRelabelConfig + for _, arc := range m.arcs { + result = append(result, *arc) + } + return result, nil +} + +func (m *mockARCInterface) Get(_ context.Context, ns, name string) (*osmv1.AlertRelabelConfig, bool, error) { + if arc, ok := m.arcs[ns+"/"+name]; ok { + return arc, true, nil + } + return nil, false, nil +} + +func (m *mockARCInterface) Create(_ context.Context, arc osmv1.AlertRelabelConfig) (*osmv1.AlertRelabelConfig, error) { + return &arc, nil +} + +func (m *mockARCInterface) Update(_ context.Context, _ osmv1.AlertRelabelConfig) error { return nil } + +func (m *mockARCInterface) Delete(_ context.Context, ns, name string) error { + m.deleted = append(m.deleted, ns+"/"+name) + delete(m.arcs, ns+"/"+name) + return nil +} + +func newARC(ns, name, ruleID string, annotations, labels map[string]string) *osmv1.AlertRelabelConfig { + if annotations == nil { + annotations = map[string]string{} + } + if ruleID != "" { + annotations[managementlabels.ARCAnnotationAlertRuleIDKey] = ruleID + } + return &osmv1.AlertRelabelConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Annotations: annotations, + Labels: labels, + }, + } +} + +func TestGCOrphanedARCs_DeletesOrphan(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-orphan": newARC("openshift-monitoring", "arc-orphan", "rule-gone", nil, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) + + if len(mock.deleted) != 1 || mock.deleted[0] != "openshift-monitoring/arc-orphan" { + t.Fatalf("expected orphan ARC to be deleted, got deleted=%v", mock.deleted) + } +} + +func TestGCOrphanedARCs_KeepsLiveRule(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-live": newARC("openshift-monitoring", "arc-live", "rule-alive", nil, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{"rule-alive": {}}) + + if len(mock.deleted) != 0 { + t.Fatalf("expected no deletions, got deleted=%v", mock.deleted) + } +} + +func TestGCOrphanedARCs_SkipsGitOpsManaged(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-gitops": newARC("openshift-monitoring", "arc-gitops", "rule-gone", + map[string]string{"argocd.argoproj.io/tracking-id": "some-id"}, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) + + if len(mock.deleted) != 0 { + t.Fatalf("expected GitOps-managed ARC to be preserved, got deleted=%v", mock.deleted) + } +} + +func TestGCOrphanedARCs_SkipsARCWithoutAnnotation(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-manual": newARC("openshift-monitoring", "arc-manual", "", nil, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) + + if len(mock.deleted) != 0 { + t.Fatalf("expected ARC without annotation to be preserved, got deleted=%v", mock.deleted) + } +} + +func TestGCOrphanedARCs_MixedScenario(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-live": newARC("openshift-monitoring", "arc-live", "rule-1", nil, nil), + "openshift-monitoring/arc-orphan1": newARC("openshift-monitoring", "arc-orphan1", "rule-deleted-1", nil, nil), + "openshift-monitoring/arc-orphan2": newARC("openshift-monitoring", "arc-orphan2", "rule-deleted-2", nil, nil), + "openshift-monitoring/arc-gitops": newARC("openshift-monitoring", "arc-gitops", "rule-deleted-3", + map[string]string{"argocd.argoproj.io/tracking-id": "t"}, nil), + "openshift-monitoring/arc-manual": newARC("openshift-monitoring", "arc-manual", "", nil, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + liveIDs := map[string]struct{}{"rule-1": {}} + rrm.gcOrphanedARCs(context.Background(), liveIDs) + + deletedSet := map[string]bool{} + for _, d := range mock.deleted { + deletedSet[d] = true + } + + if len(mock.deleted) != 2 { + t.Fatalf("expected 2 deletions, got %d: %v", len(mock.deleted), mock.deleted) + } + if !deletedSet["openshift-monitoring/arc-orphan1"] { + t.Error("expected arc-orphan1 to be deleted") + } + if !deletedSet["openshift-monitoring/arc-orphan2"] { + t.Error("expected arc-orphan2 to be deleted") + } + if deletedSet["openshift-monitoring/arc-live"] { + t.Error("arc-live should not have been deleted") + } + if deletedSet["openshift-monitoring/arc-gitops"] { + t.Error("arc-gitops should not have been deleted (GitOps-managed)") + } + if deletedSet["openshift-monitoring/arc-manual"] { + t.Error("arc-manual should not have been deleted (no annotation)") + } +} + +func TestGCOrphanedARCs_NilInterface(t *testing.T) { + rrm := &relabeledRulesManager{alertRelabelConfigs: nil} + // Should not panic + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) +} diff --git a/pkg/k8s/relabeled_rules.go b/pkg/k8s/relabeled_rules.go index a853630ea..9ecec5325 100644 --- a/pkg/k8s/relabeled_rules.go +++ b/pkg/k8s/relabeled_rules.go @@ -148,7 +148,7 @@ func newRelabeledRulesManager(ctx context.Context, namespaceManager NamespaceInt return nil, fmt.Errorf("failed to sync RelabeledRulesConfig informer") } - if err := rrm.sync(ctx); err != nil { + if err := rrm.sync(ctx, "initial-sync"); err != nil { return nil, fmt.Errorf("initial relabeled rules sync failed: %w", err) } @@ -179,7 +179,7 @@ func (rrm *relabeledRulesManager) processNextWorkItem(ctx context.Context) bool defer rrm.queue.Done(key) - if err := rrm.sync(ctx); err != nil { + if err := rrm.sync(ctx, key); err != nil { log.Errorf("error syncing relabeled rules: %v", err) rrm.queue.AddRateLimited(key) return true @@ -190,7 +190,7 @@ func (rrm *relabeledRulesManager) processNextWorkItem(ctx context.Context) bool return true } -func (rrm *relabeledRulesManager) sync(ctx context.Context) error { +func (rrm *relabeledRulesManager) sync(ctx context.Context, key string) error { relabelConfigs, err := rrm.loadRelabelConfigs() if err != nil { return fmt.Errorf("failed to load relabel configs: %w", err) @@ -200,13 +200,20 @@ func (rrm *relabeledRulesManager) sync(ctx context.Context) error { rrm.relabelConfigs = relabelConfigs rrm.mu.Unlock() - alerts := rrm.collectAlerts(ctx, relabelConfigs) + alerts, allRuleIDs := rrm.collectAlerts(ctx, relabelConfigs) rrm.mu.Lock() rrm.relabeledRules = alerts rrm.mu.Unlock() log.Infof("Synced %d relabeled rules in memory", len(alerts)) + + // GC orphaned ARCs only when triggered by PrometheusRule events or + // initial sync — secret-only changes cannot create orphans. + if key == "prometheus-rule-sync" || key == "initial-sync" { + rrm.gcOrphanedARCs(ctx, allRuleIDs) + } + return nil } @@ -255,7 +262,7 @@ func (rrm *relabeledRulesManager) loadRelabelConfigs() ([]*relabel.Config, error return configs, nil } -func (rrm *relabeledRulesManager) collectAlerts(ctx context.Context, relabelConfigs []*relabel.Config) map[string]monitoringv1.Rule { +func (rrm *relabeledRulesManager) collectAlerts(ctx context.Context, relabelConfigs []*relabel.Config) (map[string]monitoringv1.Rule, map[string]struct{}) { alerts := make(map[string]monitoringv1.Rule) seenIDs := make(map[string]struct{}) @@ -330,7 +337,7 @@ func (rrm *relabeledRulesManager) collectAlerts(ctx context.Context, relabelConf } log.Debugf("Collected %d alerts", len(alerts)) - return alerts + return alerts, seenIDs } // alertingRuleOwner returns the name of the AlertingRule CR that generated diff --git a/pkg/metrics/alerts_collector.go b/pkg/metrics/alerts_collector.go new file mode 100644 index 000000000..63aa18e83 --- /dev/null +++ b/pkg/metrics/alerts_collector.go @@ -0,0 +1,223 @@ +package metrics + +import ( + "context" + "fmt" + "net/http" + "sort" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "k8s.io/client-go/rest" +) + +var metricsLog = logrus.WithField("module", "metrics") + +const ( + MetricName = "alerts_effective_active_at_timestamp_seconds" + metricHelp = "The activeAt timestamp of effective (post-ARC) alerts. " + + "Value is the Unix timestamp when the alert became active." + + DefaultSyncInterval = 30 * time.Second + + labelAlertState = "alertstate" +) + +// AlertsFetcher retrieves enriched alerts for the metric. The management.Client +// satisfies this interface — it applies ARC relabeling and computes +// classification (AlertComponent / AlertLayer) on every alert. +type AlertsFetcher interface { + GetAlerts(ctx context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) +} + +// alertMetric holds a single alert's pre-built metric data. +// The prometheus.Desc is created once during sync, not on every scrape. +type alertMetric struct { + desc *prometheus.Desc + labelValues []string + activeAtSec float64 +} + +// AlertsCollector implements prometheus.Collector. It periodically fetches +// alerts via the management client's GetAlerts (which applies ARC relabeling +// and computes classification) and exposes them as the +// alerts_effective_active_at_timestamp_seconds gauge. +// +// Only the leader pod (determined via Lease-based leader election) runs the +// sync loop and exposes metrics. Follower pods return nothing on Collect, +// ensuring each alert appears exactly once in Prometheus. +// +// Each alert produces one time series whose value is the alert's activeAt +// Unix timestamp. Labels are the alert's enriched labels (post-ARC, source, +// backend, component, layer) plus "alertstate". Thanos-sourced alerts are +// filtered out to avoid duplicates. Annotations are excluded because they +// are available from the alert rule definition. +type AlertsCollector struct { + fetcher AlertsFetcher + syncInterval time.Duration + isLeader func() bool + + mu sync.RWMutex + metrics []alertMetric + + sentinelDesc *prometheus.Desc +} + +// NewHandler creates a metrics HTTP handler that exposes the alerts effective +// metric. It sets up Lease-based leader election internally so that only one +// replica produces metrics, then wires the collector, registry and promhttp +// handler. Callers receive a ready-to-use http.Handler. +func NewHandler(ctx context.Context, fetcher AlertsFetcher, kubeConfig *rest.Config) (http.Handler, error) { + isLeader, err := startLeaderElection(ctx, kubeConfig, k8s.ClusterMonitoringNamespace) + if err != nil { + return nil, fmt.Errorf("start metrics leader election: %w", err) + } + + collector := NewAlertsCollector(ctx, fetcher, DefaultSyncInterval, isLeader) + registry := prometheus.NewRegistry() + registry.MustRegister(collector) + return promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), nil +} + +// NewAlertsCollector creates a collector that periodically syncs alerts and +// exposes them as Prometheus metrics. The isLeader callback controls whether +// this replica actively syncs and exposes metrics (follower pods return nothing). +func NewAlertsCollector(ctx context.Context, fetcher AlertsFetcher, syncInterval time.Duration, isLeader func() bool) *AlertsCollector { + c := &AlertsCollector{ + fetcher: fetcher, + syncInterval: syncInterval, + isLeader: isLeader, + sentinelDesc: prometheus.NewDesc(MetricName, metricHelp, nil, nil), + } + go c.syncLoop(ctx) + return c +} + +// Describe sends a sentinel descriptor to satisfy the Collector contract. +func (c *AlertsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.sentinelDesc +} + +// Collect emits the current set of alert metrics using pre-built Descs. +// Returns nothing if this replica is not the leader. +func (c *AlertsCollector) Collect(ch chan<- prometheus.Metric) { + if !c.isLeader() { + return + } + + c.mu.RLock() + defer c.mu.RUnlock() + + for i := range c.metrics { + m := &c.metrics[i] + metric, err := prometheus.NewConstMetric(m.desc, prometheus.GaugeValue, m.activeAtSec, m.labelValues...) + if err != nil { + metricsLog.WithError(err).Warn("failed to create metric") + continue + } + ch <- metric + } +} + +func (c *AlertsCollector) syncLoop(ctx context.Context) { + c.sync(ctx) + + ticker := time.NewTicker(c.syncInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.sync(ctx) + } + } +} + +func (c *AlertsCollector) sync(ctx context.Context) { + if !c.isLeader() { + return + } + + alerts, err := c.fetcher.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + metricsLog.WithError(err).Warn("failed to fetch alerts for effective metric") + return + } + + built := make([]alertMetric, 0, len(alerts)) + for i := range alerts { + alert := &alerts[i] + + // Drop Thanos-sourced alerts: they duplicate what Alertmanager and + // Prometheus already provide and would inflate the metric cardinality. + if alert.Labels[k8s.AlertBackendLabel] == k8s.AlertBackendThanos { + continue + } + + enrichClassificationLabels(alert) + + m := buildAlertMetric(alert) + if m != nil { + built = append(built, *m) + } + } + + c.mu.Lock() + c.metrics = built + c.mu.Unlock() + + metricsLog.Debugf("synced %d alerts for effective metric", len(built)) +} + +// enrichClassificationLabels copies the management-computed AlertComponent and +// AlertLayer into the alert's Labels map so they appear on the metric. Labels +// already set (e.g. via ARC) take precedence. +func enrichClassificationLabels(alert *k8s.PrometheusAlert) { + if alert.AlertComponent != "" { + if _, exists := alert.Labels[k8s.AlertRuleClassificationComponentKey]; !exists { + alert.Labels[k8s.AlertRuleClassificationComponentKey] = alert.AlertComponent + } + } + if alert.AlertLayer != "" { + if _, exists := alert.Labels[k8s.AlertRuleClassificationLayerKey]; !exists { + alert.Labels[k8s.AlertRuleClassificationLayerKey] = alert.AlertLayer + } + } +} + +// buildAlertMetric converts a PrometheusAlert into an alertMetric with a +// pre-built prometheus.Desc. Uses the alert's labels plus the alertstate label. +func buildAlertMetric(alert *k8s.PrometheusAlert) *alertMetric { + if alert.ActiveAt.IsZero() { + return nil + } + + labelNames := make([]string, 0, len(alert.Labels)+1) + for k := range alert.Labels { + labelNames = append(labelNames, k) + } + sort.Strings(labelNames) + labelNames = append(labelNames, labelAlertState) + + labelValues := make([]string, 0, len(labelNames)) + for _, name := range labelNames { + if name == labelAlertState { + labelValues = append(labelValues, alert.State) + } else { + labelValues = append(labelValues, alert.Labels[name]) + } + } + + return &alertMetric{ + desc: prometheus.NewDesc(MetricName, metricHelp, labelNames, nil), + labelValues: labelValues, + activeAtSec: float64(alert.ActiveAt.Unix()), + } +} diff --git a/pkg/metrics/alerts_collector_test.go b/pkg/metrics/alerts_collector_test.go new file mode 100644 index 000000000..6cc30efbe --- /dev/null +++ b/pkg/metrics/alerts_collector_test.go @@ -0,0 +1,354 @@ +package metrics_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/metrics" +) + +type mockAlertsFetcher struct { + alerts []k8s.PrometheusAlert + err error +} + +func (m *mockAlertsFetcher) GetAlerts(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return m.alerts, m.err +} + +func collectMetrics(t *testing.T, collector prometheus.Collector) []*dto.MetricFamily { + t.Helper() + reg := prometheus.NewRegistry() + reg.MustRegister(collector) + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather metrics: %v", err) + } + return families +} + +func findFamily(families []*dto.MetricFamily, name string) *dto.MetricFamily { + for _, f := range families { + if f.GetName() == name { + return f + } + } + return nil +} + +func labelValue(m *dto.Metric, name string) string { + for _, lp := range m.GetLabel() { + if lp.GetName() == name { + return lp.GetValue() + } + } + return "" +} + +func newCollector(t *testing.T, mock *mockAlertsFetcher) (prometheus.Collector, context.CancelFunc) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + collector := metrics.NewAlertsCollector(ctx, mock, 1*time.Hour, func() bool { return true }) + time.Sleep(100 * time.Millisecond) + t.Cleanup(cancel) + return collector, cancel +} + +func TestAlertsCollector_FiringAndSilenced(t *testing.T) { + activeAt := time.Date(2026, 3, 5, 10, 0, 0, 0, time.UTC) + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{"alertname": "HighCPU", "severity": "critical", "namespace": "production"}, + State: "firing", + ActiveAt: activeAt, + }, + { + Labels: map[string]string{"alertname": "DiskFull", "severity": "warning", "namespace": "storage"}, + State: "silenced", + ActiveAt: activeAt.Add(-1 * time.Hour), + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil { + t.Fatal("expected metric family, got nil") + } + if len(family.GetMetric()) != 2 { + t.Fatalf("expected 2 metrics, got %d", len(family.GetMetric())) + } + + var firing, silenced *dto.Metric + for _, m := range family.GetMetric() { + switch labelValue(m, "alertname") { + case "HighCPU": + firing = m + case "DiskFull": + silenced = m + } + } + + if firing == nil { + t.Fatal("expected HighCPU metric") + } + if labelValue(firing, "alertstate") != "firing" { + t.Errorf("alertstate: want firing, got %q", labelValue(firing, "alertstate")) + } + if labelValue(firing, "severity") != "critical" { + t.Errorf("severity: want critical, got %q", labelValue(firing, "severity")) + } + if labelValue(firing, "namespace") != "production" { + t.Errorf("namespace: want production, got %q", labelValue(firing, "namespace")) + } + if firing.GetGauge().GetValue() != float64(activeAt.Unix()) { + t.Errorf("gauge value: want %v, got %v", float64(activeAt.Unix()), firing.GetGauge().GetValue()) + } + + if silenced == nil { + t.Fatal("expected DiskFull metric") + } + if labelValue(silenced, "alertstate") != "silenced" { + t.Errorf("alertstate: want silenced, got %q", labelValue(silenced, "alertstate")) + } + if silenced.GetGauge().GetValue() != float64(activeAt.Add(-1*time.Hour).Unix()) { + t.Errorf("silenced gauge value mismatch") + } +} + +func TestAlertsCollector_NoAnnotationLabels(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "TestAlert"}, State: "firing", ActiveAt: time.Now()}, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil { + t.Fatal("expected metric family") + } + if len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric, got %d", len(family.GetMetric())) + } + for _, lp := range family.GetMetric()[0].GetLabel() { + switch lp.GetName() { + case "summary", "description", "runbook_url": + t.Errorf("unexpected annotation label: %s", lp.GetName()) + } + } +} + +func TestAlertsCollector_SkipsZeroActiveAt(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "NoActiveAt"}, State: "firing", ActiveAt: time.Time{}}, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family != nil && len(family.GetMetric()) != 0 { + t.Errorf("expected no metrics for zero ActiveAt, got %d", len(family.GetMetric())) + } +} + +func TestAlertsCollector_EmptyAlerts(t *testing.T) { + mock := &mockAlertsFetcher{alerts: []k8s.PrometheusAlert{}} + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family != nil && len(family.GetMetric()) != 0 { + t.Errorf("expected no metrics for empty alerts, got %d", len(family.GetMetric())) + } +} + +func TestAlertsCollector_FetcherErrorProducesNoMetrics(t *testing.T) { + mock := &mockAlertsFetcher{err: errors.New("connection refused")} + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family != nil && len(family.GetMetric()) != 0 { + t.Errorf("expected no metrics on initial failure, got %d", len(family.GetMetric())) + } +} + +func TestAlertsCollector_ClassificationLabels(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{ + "alertname": "KubePodCrashLooping", + "severity": "warning", + "namespace": "kube-system", + k8s.AlertRuleLabelId: "abc123", + k8s.AlertRuleClassificationComponentKey: "kube-controller-manager", + k8s.AlertRuleClassificationLayerKey: "cluster", + }, + State: "firing", + ActiveAt: time.Now(), + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric, got family=%v", family) + } + m := family.GetMetric()[0] + checks := map[string]string{ + k8s.AlertRuleLabelId: "abc123", + k8s.AlertRuleClassificationComponentKey: "kube-controller-manager", + k8s.AlertRuleClassificationLayerKey: "cluster", + "alertstate": "firing", + } + for k, want := range checks { + if got := labelValue(m, k); got != want { + t.Errorf("label[%s]: want %q, got %q", k, want, got) + } + } +} + +func TestAlertsCollector_IncludesPendingAlerts(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "Firing"}, State: "firing", ActiveAt: time.Now()}, + {Labels: map[string]string{"alertname": "Silenced"}, State: "silenced", ActiveAt: time.Now()}, + {Labels: map[string]string{"alertname": "Pending"}, State: "pending", ActiveAt: time.Now()}, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 3 { + t.Fatalf("expected 3 metrics, got %v", family) + } + states := map[string]bool{} + for _, m := range family.GetMetric() { + states[labelValue(m, "alertstate")] = true + } + for _, s := range []string{"firing", "silenced", "pending"} { + if !states[s] { + t.Errorf("expected state %q in metrics", s) + } + } +} + +func TestAlertsCollector_SourceAndBackendLabels(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{ + "alertname": "HighCPU", + "severity": "critical", + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.AlertBackendLabel: k8s.AlertBackendAM, + }, + State: "firing", + ActiveAt: time.Now(), + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric") + } + m := family.GetMetric()[0] + if got := labelValue(m, k8s.AlertSourceLabel); got != k8s.AlertSourcePlatform { + t.Errorf("source: want %q, got %q", k8s.AlertSourcePlatform, got) + } + if got := labelValue(m, k8s.AlertBackendLabel); got != k8s.AlertBackendAM { + t.Errorf("backend: want %q, got %q", k8s.AlertBackendAM, got) + } +} + +func TestAlertsCollector_FiltersThanosBackend(t *testing.T) { + now := time.Now() + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "HighCPU", k8s.AlertBackendLabel: k8s.AlertBackendAM, k8s.AlertSourceLabel: k8s.AlertSourcePlatform}, State: "firing", ActiveAt: now}, + {Labels: map[string]string{"alertname": "HighCPU", k8s.AlertBackendLabel: k8s.AlertBackendThanos, k8s.AlertSourceLabel: k8s.AlertSourceUser}, State: "firing", ActiveAt: now}, + {Labels: map[string]string{"alertname": "PendingAlert", k8s.AlertBackendLabel: k8s.AlertBackendProm, k8s.AlertSourceLabel: k8s.AlertSourcePlatform}, State: "pending", ActiveAt: now}, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 2 { + t.Fatalf("expected 2 metrics (thanos filtered), got %v", family) + } + for _, m := range family.GetMetric() { + if labelValue(m, k8s.AlertBackendLabel) == k8s.AlertBackendThanos { + t.Error("thanos duplicate should be filtered out") + } + } +} + +func TestAlertsCollector_InjectsClassificationFromFields(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{"alertname": "TestAlert", k8s.AlertBackendLabel: k8s.AlertBackendAM}, + State: "firing", + ActiveAt: time.Now(), + AlertComponent: "networking", + AlertLayer: "cluster", + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric") + } + m := family.GetMetric()[0] + if got := labelValue(m, k8s.AlertRuleClassificationComponentKey); got != "networking" { + t.Errorf("component: want networking, got %q", got) + } + if got := labelValue(m, k8s.AlertRuleClassificationLayerKey); got != "cluster" { + t.Errorf("layer: want cluster, got %q", got) + } +} + +func TestAlertsCollector_DoesNotOverwriteARCLabels(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{ + "alertname": "TestAlert", + k8s.AlertBackendLabel: k8s.AlertBackendAM, + k8s.AlertRuleClassificationComponentKey: "arc-component", + k8s.AlertRuleClassificationLayerKey: "namespace", + }, + State: "firing", + ActiveAt: time.Now(), + AlertComponent: "default-component", + AlertLayer: "cluster", + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric") + } + m := family.GetMetric()[0] + if got := labelValue(m, k8s.AlertRuleClassificationComponentKey); got != "arc-component" { + t.Errorf("component: want arc-component, got %q", got) + } + if got := labelValue(m, k8s.AlertRuleClassificationLayerKey); got != "namespace" { + t.Errorf("layer: want namespace, got %q", got) + } +} diff --git a/pkg/metrics/leader_election.go b/pkg/metrics/leader_election.go new file mode 100644 index 000000000..2a71f9231 --- /dev/null +++ b/pkg/metrics/leader_election.go @@ -0,0 +1,87 @@ +package metrics + +import ( + "context" + "fmt" + "os" + "sync" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" +) + +const ( + leaseName = "monitoring-plugin-metrics" + leaseDuration = 15 * time.Second + leaseRenew = 10 * time.Second + leaseRetry = 2 * time.Second +) + +// startLeaderElection sets up Lease-based leader election for the alerts +// effective metric. Returns a thread-safe isLeader callback. +func startLeaderElection(ctx context.Context, kubeConfig *rest.Config, namespace string) (func() bool, error) { + coordClient, err := coordinationv1client.NewForConfig(kubeConfig) + if err != nil { + return nil, fmt.Errorf("create coordination client: %w", err) + } + + identity, err := os.Hostname() + if err != nil { + return nil, fmt.Errorf("get hostname: %w", err) + } + + lock := &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{ + Name: leaseName, + Namespace: namespace, + }, + Client: coordClient, + LockConfig: resourcelock.ResourceLockConfig{ + Identity: identity, + }, + } + + var mu sync.Mutex + isLeading := false + + isLeader := func() bool { + mu.Lock() + defer mu.Unlock() + return isLeading + } + + le, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ + Lock: lock, + LeaseDuration: leaseDuration, + RenewDeadline: leaseRenew, + RetryPeriod: leaseRetry, + ReleaseOnCancel: true, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(_ context.Context) { + mu.Lock() + isLeading = true + mu.Unlock() + metricsLog.Info("became leader for alert management metrics") + }, + OnStoppedLeading: func() { + mu.Lock() + isLeading = false + mu.Unlock() + metricsLog.Info("lost leadership for alert management metrics") + }, + OnNewLeader: func(identity string) { + metricsLog.Infof("new leader for alert management metrics: %s", identity) + }, + }, + }) + if err != nil { + return nil, fmt.Errorf("create leader elector: %w", err) + } + + go le.Run(ctx) + return isLeader, nil +} diff --git a/test/e2e/alerts_effective_metric_test.go b/test/e2e/alerts_effective_metric_test.go new file mode 100644 index 000000000..eb16739f1 --- /dev/null +++ b/test/e2e/alerts_effective_metric_test.go @@ -0,0 +1,315 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/metrics" + "github.com/openshift/monitoring-plugin/test/e2e/framework" +) + +func fetchMetrics(f *framework.Framework) (string, error) { + resp, err := f.HTTPClient().Get(f.PluginURL + "/metrics") + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(body), nil +} + +func parseMetricLines(body string) []string { + var lines []string + for _, line := range strings.Split(body, "\n") { + if strings.HasPrefix(line, metrics.MetricName+"{") { + lines = append(lines, line) + } + } + return lines +} + +func extractLabel(metricLine, labelName string) string { + key := labelName + `="` + idx := strings.Index(metricLine, key) + if idx < 0 { + return "" + } + start := idx + len(key) + end := strings.Index(metricLine[start:], `"`) + if end < 0 { + return "" + } + return metricLine[start : start+end] +} + +// TestMetricEndpointExposesEffectiveMetric +// Verifies that the /metrics endpoint exposes alerts_effective_active_at_timestamp_seconds. +func TestMetricEndpointExposesEffectiveMetric(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + var metricBody string + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + body, err := fetchMetrics(f) + if err != nil { + t.Logf("Failed to fetch metrics: %v", err) + return false, nil + } + + if !strings.Contains(body, metrics.MetricName) { + t.Logf("Metric %s not found yet (leader election may be in progress)", metrics.MetricName) + return false, nil + } + + metricBody = body + return true, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for metric to appear: %v", err) + } + + if !strings.Contains(metricBody, "# HELP "+metrics.MetricName) { + t.Error("Missing HELP line for metric") + } + if !strings.Contains(metricBody, "# TYPE "+metrics.MetricName+" gauge") { + t.Error("Missing or incorrect TYPE line for metric (expected gauge)") + } + + lines := parseMetricLines(metricBody) + if len(lines) == 0 { + t.Fatal("Expected at least one metric series, got none") + } + + t.Logf("Found %d metric series for %s", len(lines), metrics.MetricName) +} + +// TestMetricSeriesHaveRequiredLabels +// Verifies every metric series has alertname, alertstate, openshift_io_alert_source, +// openshift_io_alert_backend, and a valid timestamp value. +func TestMetricSeriesHaveRequiredLabels(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + var lines []string + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + body, err := fetchMetrics(f) + if err != nil { + t.Logf("Failed to fetch metrics: %v", err) + return false, nil + } + lines = parseMetricLines(body) + return len(lines) > 0, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for metric series: %v", err) + } + + requiredLabels := []string{ + "alertname", + "alertstate", + k8s.AlertSourceLabel, + k8s.AlertBackendLabel, + } + + for i, line := range lines { + for _, label := range requiredLabels { + val := extractLabel(line, label) + if val == "" { + t.Errorf("Series %d missing required label %q: %s", i, label, line) + } + } + + state := extractLabel(line, "alertstate") + validStates := map[string]bool{"firing": true, "pending": true, "silenced": true, "suppressed": true} + if !validStates[state] { + t.Errorf("Series %d has unexpected alertstate=%q: %s", i, state, line) + } + + parts := strings.Split(line, " ") + if len(parts) < 2 { + t.Errorf("Series %d has no value: %s", i, line) + continue + } + var ts float64 + if _, err := fmt.Sscanf(parts[len(parts)-1], "%g", &ts); err != nil { + t.Errorf("Series %d has unparseable value %q: %v", i, parts[len(parts)-1], err) + continue + } + if ts < 9.46e+08 { + t.Errorf("Series %d has suspiciously low timestamp value: %g (before year 2000)", i, ts) + } + } + + t.Logf("All %d series have required labels and valid values", len(lines)) +} + +// TestMetricIncludesClassificationLabels +// Verifies that all metric series have classification labels +// (openshift_io_alert_rule_component and openshift_io_alert_rule_layer). +func TestMetricIncludesClassificationLabels(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + var lines []string + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + body, err := fetchMetrics(f) + if err != nil { + return false, nil + } + lines = parseMetricLines(body) + return len(lines) > 0, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for metric series: %v", err) + } + + for i, line := range lines { + if extractLabel(line, k8s.AlertRuleClassificationComponentKey) == "" { + t.Errorf("Series %d missing %s label: %s", i, k8s.AlertRuleClassificationComponentKey, line) + } + if extractLabel(line, k8s.AlertRuleClassificationLayerKey) == "" { + t.Errorf("Series %d missing %s label: %s", i, k8s.AlertRuleClassificationLayerKey, line) + } + } + + t.Logf("All %d series have classification labels (component + layer)", len(lines)) +} + +// TestMetricExcludesAnnotations +// Verifies that annotations (summary, description, runbook_url) are not +// included as metric labels. +func TestMetricExcludesAnnotations(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + var lines []string + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + body, err := fetchMetrics(f) + if err != nil { + return false, nil + } + lines = parseMetricLines(body) + return len(lines) > 0, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for metric series: %v", err) + } + + annotationLabels := []string{"summary", "description", "runbook_url"} + + for i, line := range lines { + for _, annLabel := range annotationLabels { + if extractLabel(line, annLabel) != "" { + t.Errorf("Series %d contains annotation label %q (annotations should be excluded): %s", + i, annLabel, line) + } + } + } + + t.Logf("Verified %d series - none contain annotation labels", len(lines)) +} + +// TestMetricActiveAtTimestampsAreReasonable +// Verifies that activeAt timestamps are not too recent. +func TestMetricActiveAtTimestampsAreReasonable(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + var lines []string + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + body, err := fetchMetrics(f) + if err != nil { + return false, nil + } + lines = parseMetricLines(body) + return len(lines) > 0, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for metric series: %v", err) + } + + now := float64(time.Now().Unix()) + fiveMinutesAgo := now - 300 + + recentCount := 0 + for _, line := range lines { + alertname := extractLabel(line, "alertname") + if alertname == "Watchdog" { + continue + } + + parts := strings.Split(line, " ") + if len(parts) < 2 { + continue + } + valueStr := parts[len(parts)-1] + + var ts float64 + if _, err := fmt.Sscanf(valueStr, "%e", &ts); err != nil { + if _, err := fmt.Sscanf(valueStr, "%f", &ts); err != nil { + continue + } + } + + if ts > fiveMinutesAgo { + recentCount++ + t.Logf("WARN: %s has activeAt within last 5 minutes (ts=%.0f, now=%.0f)", alertname, ts, now) + } + } + + totalNonWatchdog := 0 + for _, line := range lines { + if extractLabel(line, "alertname") != "Watchdog" { + totalNonWatchdog++ + } + } + + if totalNonWatchdog > 0 { + recentPct := float64(recentCount) / float64(totalNonWatchdog) * 100 + if recentPct > 80 { + t.Errorf("%.0f%% of alerts (%d/%d) have activeAt within last 5 minutes — "+ + "likely using Alertmanager startsAt instead of Prometheus activeAt", + recentPct, recentCount, totalNonWatchdog) + } + } + + t.Logf("Timestamp check: %d/%d non-Watchdog alerts have recent activeAt", recentCount, totalNonWatchdog) +} From 56a6e154f268881087411eaac055a8a1a6976b93 Mon Sep 17 00:00:00 2001 From: Shirly Radco Date: Thu, 12 Mar 2026 20:34:33 +0200 Subject: [PATCH 5/5] add alerts_effective_active_at metric Expose a Prometheus gauge metric whose value is the activeAt Unix timestamp for every effective alert (firing, pending, silenced). Labels include all alerts labels after relabeling plus enrichment labels and alertstate. Annotations are excluded since they are available from the alert rule definition. Signed-off-by: Shirly Radco Co-authored-by: AI Assistant --- internal/managementrouter/health_get_test.go | 14 +++ pkg/k8s/enrich_active_at_test.go | 106 ++++++++++++++++++ pkg/k8s/prometheus_alerts.go | 72 +++++++++++- pkg/management/management.go | 9 ++ .../metrics/alerts_collector.go | 2 +- .../metrics/alerts_collector_test.go | 2 +- .../metrics/leader_election.go | 0 pkg/management/types.go | 6 + pkg/server/server.go | 20 ++-- test/e2e/alerts_effective_metric_test.go | 2 +- 10 files changed, 220 insertions(+), 13 deletions(-) create mode 100644 pkg/k8s/enrich_active_at_test.go rename pkg/{ => management}/metrics/alerts_collector.go (100%) rename pkg/{ => management}/metrics/alerts_collector_test.go (99%) rename pkg/{ => management}/metrics/leader_election.go (100%) diff --git a/internal/managementrouter/health_get_test.go b/internal/managementrouter/health_get_test.go index a6be1046f..0edd58c18 100644 --- a/internal/managementrouter/health_get_test.go +++ b/internal/managementrouter/health_get_test.go @@ -9,6 +9,7 @@ import ( "testing" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/client-go/rest" "github.com/openshift/monitoring-plugin/internal/managementrouter" "github.com/openshift/monitoring-plugin/pkg/k8s" @@ -25,15 +26,19 @@ type stubClient struct { func (s *stubClient) ListRules(_ context.Context, _ management.PrometheusRuleOptions, _ management.AlertRuleOptions, _ management.PaginationOptions) (management.ListRulesResult, error) { return management.ListRulesResult{}, nil } + func (s *stubClient) GetRuleById(_ context.Context, _ string) (monitoringv1.Rule, error) { return monitoringv1.Rule{}, nil } + func (s *stubClient) CreateUserDefinedAlertRule(_ context.Context, _ monitoringv1.Rule, _ management.PrometheusRuleOptions) (string, error) { return "", nil } + func (s *stubClient) CreatePlatformAlertRule(_ context.Context, _ monitoringv1.Rule) (string, error) { return "", nil } + func (s *stubClient) UpdateUserDefinedAlertRule(_ context.Context, _ string, _ monitoringv1.Rule) (string, error) { return "", nil } @@ -46,28 +51,37 @@ func (s *stubClient) RestoreAlertRule(_ context.Context, _ string) error { retur func (s *stubClient) UpdateAlertRuleLabels(_ context.Context, _ string, _ map[string]*string) (string, error) { return "", nil } + func (s *stubClient) GetAlerts(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { return nil, nil } + func (s *stubClient) GetRules(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { if s.getRules != nil { return s.getRules(ctx, req) } return []k8s.PrometheusRuleGroup{}, nil } + func (s *stubClient) GetAlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) { if s.alertingHealth != nil { return s.alertingHealth(ctx) } return k8s.AlertingHealth{}, nil } + func (s *stubClient) UpdateAlertRuleClassification(_ context.Context, _ management.UpdateRuleClassificationRequest) error { return nil } + func (s *stubClient) BulkUpdateAlertRuleClassification(_ context.Context, _ []management.UpdateRuleClassificationRequest) []error { return nil } +func (s *stubClient) MetricsHandler(_ context.Context, _ *rest.Config) (http.Handler, error) { + return nil, nil +} + // newStubRouter builds a router backed by stub and adds a Bearer token header // to requests via the helper get/getNoAuth methods. func newStubRouter(stub *stubClient) http.Handler { diff --git a/pkg/k8s/enrich_active_at_test.go b/pkg/k8s/enrich_active_at_test.go new file mode 100644 index 000000000..95e205591 --- /dev/null +++ b/pkg/k8s/enrich_active_at_test.go @@ -0,0 +1,106 @@ +package k8s + +import ( + "testing" + "time" +) + +func TestEnrichActiveAt_ReplacesAlertmanagerTimestamp(t *testing.T) { + amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + promTime := time.Date(2026, 3, 9, 8, 0, 0, 0, time.UTC) + + amAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU", "severity": "critical", AlertSourceLabel: "platform", AlertBackendLabel: "am"}, + ActiveAt: amTime, + }} + promAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU", "severity": "critical", AlertSourceLabel: "platform", AlertBackendLabel: "prom"}, + ActiveAt: promTime, + }} + + enrichActiveAt(amAlerts, promAlerts) + + if !amAlerts[0].ActiveAt.Equal(promTime) { + t.Errorf("expected ActiveAt=%v, got %v", promTime, amAlerts[0].ActiveAt) + } +} + +func TestEnrichActiveAt_NoMatchKeepsOriginal(t *testing.T) { + amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + + amAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU", "severity": "critical"}, + ActiveAt: amTime, + }} + promAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "DiskFull", "severity": "warning"}, + ActiveAt: time.Date(2026, 3, 9, 8, 0, 0, 0, time.UTC), + }} + + enrichActiveAt(amAlerts, promAlerts) + + if !amAlerts[0].ActiveAt.Equal(amTime) { + t.Errorf("expected ActiveAt to stay %v, got %v", amTime, amAlerts[0].ActiveAt) + } +} + +func TestEnrichActiveAt_EmptyPromAlerts(t *testing.T) { + amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + + amAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU"}, + ActiveAt: amTime, + }} + + enrichActiveAt(amAlerts, nil) + + if !amAlerts[0].ActiveAt.Equal(amTime) { + t.Errorf("expected ActiveAt to stay %v, got %v", amTime, amAlerts[0].ActiveAt) + } +} + +func TestEnrichActiveAt_SkipsZeroPromActiveAt(t *testing.T) { + amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + + amAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU"}, + ActiveAt: amTime, + }} + promAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU"}, + }} + + enrichActiveAt(amAlerts, promAlerts) + + if !amAlerts[0].ActiveAt.Equal(amTime) { + t.Errorf("expected ActiveAt to stay %v when prom has zero time, got %v", amTime, amAlerts[0].ActiveAt) + } +} + +func TestAlertFingerprint_IgnoresMetadataLabels(t *testing.T) { + fp1 := alertFingerprint(map[string]string{ + "alertname": "HighCPU", + "severity": "critical", + AlertSourceLabel: "platform", + AlertBackendLabel: "am", + }) + fp2 := alertFingerprint(map[string]string{ + "alertname": "HighCPU", + "severity": "critical", + AlertSourceLabel: "platform", + AlertBackendLabel: "prom", + }) + + if fp1 != fp2 { + t.Errorf("fingerprints should match when only metadata labels differ:\n fp1=%q\n fp2=%q", fp1, fp2) + } +} + +func TestAlertFingerprint_DifferentLabelsProduceDifferentKeys(t *testing.T) { + fp1 := alertFingerprint(map[string]string{"alertname": "HighCPU", "severity": "critical"}) + fp2 := alertFingerprint(map[string]string{"alertname": "HighCPU", "severity": "warning"}) + + if fp1 == fp2 { + t.Error("fingerprints should differ when label values differ") + } +} diff --git a/pkg/k8s/prometheus_alerts.go b/pkg/k8s/prometheus_alerts.go index c01303cea..3077799c9 100644 --- a/pkg/k8s/prometheus_alerts.go +++ b/pkg/k8s/prometheus_alerts.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "os" + "sort" "strings" "sync" "time" @@ -286,11 +287,21 @@ func (pa *prometheusAlerts) routeHealth(ctx context.Context, namespace string, r return health } +// getAlertsForSource fetches alerts from both Alertmanager and Prometheus in +// parallel and merges the results. The fallback strategy is: +// - Both succeed: AM (firing+silenced) + Prom pending, with AM timestamps +// enriched from Prometheus activeAt. +// - AM only: AM alerts returned as-is (no Prom data to enrich from). +// - Prom only: all Prom alerts returned (AM was unreachable). +// - Both fail: error propagated from Prometheus. func (pa *prometheusAlerts) getAlertsForSource(ctx context.Context, namespace string, promRouteName string, amRouteName string, source string) ([]PrometheusAlert, error) { amAlerts, amErr := pa.getAlertmanagerAlerts(ctx, namespace, amRouteName, source) promAlerts, promErr := pa.getAlertsViaProxy(ctx, namespace, promRouteName, source) if amErr == nil { + if promErr == nil { + enrichActiveAt(amAlerts, promAlerts) + } pending := filterAlertsByState(promAlerts, "pending") return append(amAlerts, pending...), nil } @@ -345,15 +356,17 @@ func (pa *prometheusAlerts) getUserWorkloadAlertsViaAlertmanager(ctx context.Con } } - pending, err := pa.getAlertsViaProxy(ctx, UserWorkloadMonitoringNamespace, UserWorkloadRouteName, AlertSourceUser) + promAlerts, err := pa.getAlertsViaProxy(ctx, UserWorkloadMonitoringNamespace, UserWorkloadRouteName, AlertSourceUser) if err != nil { - pending, err = pa.getPrometheusAlertsViaService(ctx, UserWorkloadMonitoringNamespace, UserWorkloadPrometheusServiceName, UserWorkloadPrometheusPort, AlertSourceUser) + promAlerts, err = pa.getPrometheusAlertsViaService(ctx, UserWorkloadMonitoringNamespace, UserWorkloadPrometheusServiceName, UserWorkloadPrometheusPort, AlertSourceUser) if err != nil { return alerts, nil } } - return append(alerts, filterAlertsByState(pending, "pending")...), nil + // Enrich before filtering: AM alerts need activeAt from all Prom states. + enrichActiveAt(alerts, promAlerts) + return append(alerts, filterAlertsByState(promAlerts, "pending")...), nil } func (pa *prometheusAlerts) getPrometheusAlertsViaService(ctx context.Context, namespace string, serviceName string, port int32, source string) ([]PrometheusAlert, error) { @@ -784,6 +797,59 @@ func filterAlertsByState(alerts []PrometheusAlert, state string) []PrometheusAle return out } +// enrichActiveAt replaces ActiveAt in Alertmanager-sourced alerts with the +// authoritative value from Prometheus. Alertmanager only exposes startsAt +// (when it received the alert), while Prometheus tracks the true activeAt +// (when the alert condition first became true). +func enrichActiveAt(amAlerts, promAlerts []PrometheusAlert) { + if len(promAlerts) == 0 { + return + } + + lookup := make(map[string]time.Time, len(promAlerts)) + for _, alert := range promAlerts { + fp := alertFingerprint(alert.Labels) + if !alert.ActiveAt.IsZero() { + lookup[fp] = alert.ActiveAt + } + } + + for i := range amAlerts { + fp := alertFingerprint(amAlerts[i].Labels) + if activeAt, ok := lookup[fp]; ok { + amAlerts[i].ActiveAt = activeAt + } + } +} + +// alertFingerprint builds a stable identity key from an alert's labels, +// excluding metadata labels injected by this plugin (source, backend). +// This matches the same alert *instance* across Alertmanager and Prometheus +// (which may differ only in injected metadata). It is distinct from the +// alert rule ID (GetAlertingRuleId) which identifies the *rule definition* +// and is computed from the rule spec (name, expr, duration, static labels). +func alertFingerprint(labels map[string]string) string { + keys := make([]string, 0, len(labels)) + for k := range labels { + if k == AlertSourceLabel || k == AlertBackendLabel { + continue + } + keys = append(keys, k) + } + sort.Strings(keys) + + var b strings.Builder + for i, k := range keys { + if i > 0 { + b.WriteByte('\xff') + } + b.WriteString(k) + b.WriteByte('\xfe') + b.WriteString(labels[k]) + } + return b.String() +} + func mapAlertmanagerState(state string) string { if state == "active" { return "firing" diff --git a/pkg/management/management.go b/pkg/management/management.go index 652ac14de..124a98785 100644 --- a/pkg/management/management.go +++ b/pkg/management/management.go @@ -1,9 +1,14 @@ package management import ( + "context" + "net/http" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management/metrics" ) type client struct { @@ -20,3 +25,7 @@ type client struct { func (c *client) isPlatformManagedPrometheusRule(nn types.NamespacedName) bool { return c.k8sClient.Namespace().IsClusterMonitoringNamespace(nn.Namespace) } + +func (c *client) MetricsHandler(ctx context.Context, kubeConfig *rest.Config) (http.Handler, error) { + return metrics.NewHandler(ctx, c, kubeConfig) +} diff --git a/pkg/metrics/alerts_collector.go b/pkg/management/metrics/alerts_collector.go similarity index 100% rename from pkg/metrics/alerts_collector.go rename to pkg/management/metrics/alerts_collector.go index 63aa18e83..fd9dd9bed 100644 --- a/pkg/metrics/alerts_collector.go +++ b/pkg/management/metrics/alerts_collector.go @@ -11,9 +11,9 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" + "k8s.io/client-go/rest" "github.com/openshift/monitoring-plugin/pkg/k8s" - "k8s.io/client-go/rest" ) var metricsLog = logrus.WithField("module", "metrics") diff --git a/pkg/metrics/alerts_collector_test.go b/pkg/management/metrics/alerts_collector_test.go similarity index 99% rename from pkg/metrics/alerts_collector_test.go rename to pkg/management/metrics/alerts_collector_test.go index 6cc30efbe..5ce053834 100644 --- a/pkg/metrics/alerts_collector_test.go +++ b/pkg/management/metrics/alerts_collector_test.go @@ -10,7 +10,7 @@ import ( dto "github.com/prometheus/client_model/go" "github.com/openshift/monitoring-plugin/pkg/k8s" - "github.com/openshift/monitoring-plugin/pkg/metrics" + "github.com/openshift/monitoring-plugin/pkg/management/metrics" ) type mockAlertsFetcher struct { diff --git a/pkg/metrics/leader_election.go b/pkg/management/metrics/leader_election.go similarity index 100% rename from pkg/metrics/leader_election.go rename to pkg/management/metrics/leader_election.go diff --git a/pkg/management/types.go b/pkg/management/types.go index 8c431f2bd..64edd0810 100644 --- a/pkg/management/types.go +++ b/pkg/management/types.go @@ -2,8 +2,10 @@ package management import ( "context" + "net/http" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/client-go/rest" "github.com/openshift/monitoring-plugin/pkg/k8s" ) @@ -59,6 +61,10 @@ type Client interface { // GetAlertingHealth retrieves alerting health details GetAlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) + + // MetricsHandler returns an HTTP handler that exposes alert management metrics. + // It handles leader election internally using the provided kubeConfig. + MetricsHandler(ctx context.Context, kubeConfig *rest.Config) (http.Handler, error) } // PrometheusRuleOptions specifies options for selecting PrometheusRule resources and groups diff --git a/pkg/server/server.go b/pkg/server/server.go index 5c5c49800..c048d455d 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -184,7 +184,10 @@ func createHTTPServer(ctx context.Context, cfg *Config) (*http.Server, error) { log.Info("alert management API enabled") } - router, pluginConfig := setupRoutes(cfg, managementClient) + router, pluginConfig, err := setupRoutes(ctx, cfg, managementClient, k8sconfig) + if err != nil { + return nil, fmt.Errorf("failed to set up routes: %w", err) + } router.Use(corsHeaderMiddleware()) tlsConfig := &tls.Config{} @@ -275,7 +278,7 @@ func createHTTPServer(ctx context.Context, cfg *Config) (*http.Server, error) { return httpServer, nil } -func setupRoutes(cfg *Config, managementClient management.Client) (*mux.Router, *PluginConfig) { +func setupRoutes(ctx context.Context, cfg *Config, managementClient management.Client, k8sconfig *rest.Config) (*mux.Router, *PluginConfig, error) { configHandlerFunc, pluginConfig := configHandler(cfg) router := mux.NewRouter() @@ -290,11 +293,18 @@ func setupRoutes(cfg *Config, managementClient management.Client) (*mux.Router, if managementClient != nil { managementRouter := managementrouter.New(managementClient) router.PathPrefix("/api/v1/alerting").Handler(managementRouter) + + metricsHandler, err := managementClient.MetricsHandler(ctx, k8sconfig) + if err != nil { + return nil, nil, fmt.Errorf("failed to start alert management metrics: %w", err) + } + router.Path("/metrics").Handler(metricsHandler) + log.Info("alert management metrics started") } router.PathPrefix("/").Handler(filesHandler(http.Dir(cfg.StaticPath))) - return router, pluginConfig + return router, pluginConfig, nil } func setupProxyRoutes(cfg *Config, k8sclient *dynamic.DynamicClient, kind monitoring.KindType) *mux.Router { @@ -366,7 +376,6 @@ func corsHeaderMiddleware() func(next http.Handler) http.Handler { func featuresHandler(cfg *Config) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { jsonFeatures, err := json.Marshal(cfg.Features) - if err != nil { log.WithError(err).Errorf("cannot marshall, features were: %v", string(jsonFeatures)) http.Error(w, err.Error(), http.StatusInternalServerError) @@ -380,7 +389,6 @@ func featuresHandler(cfg *Config) http.HandlerFunc { func configHandler(cfg *Config) (http.HandlerFunc, *PluginConfig) { pluginConfData, err := os.ReadFile(cfg.PluginConfigPath) - if err != nil { log.WithError(err).Warnf("cannot read config file, serving plugin with default configuration, tried %s", cfg.PluginConfigPath) @@ -392,7 +400,6 @@ func configHandler(cfg *Config) (http.HandlerFunc, *PluginConfig) { var pluginConfig PluginConfig err = yaml.Unmarshal(pluginConfData, &pluginConfig) - if err != nil { log.WithError(err).Error("unable to unmarshall config data") return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -401,7 +408,6 @@ func configHandler(cfg *Config) (http.HandlerFunc, *PluginConfig) { } jsonPluginConfig, err := pluginConfig.MarshalJSON() - if err != nil { log.WithError(err).Errorf("unable to marshall, config data: %v", pluginConfig) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/test/e2e/alerts_effective_metric_test.go b/test/e2e/alerts_effective_metric_test.go index eb16739f1..614655f8c 100644 --- a/test/e2e/alerts_effective_metric_test.go +++ b/test/e2e/alerts_effective_metric_test.go @@ -14,7 +14,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "github.com/openshift/monitoring-plugin/pkg/k8s" - "github.com/openshift/monitoring-plugin/pkg/metrics" + "github.com/openshift/monitoring-plugin/pkg/management/metrics" "github.com/openshift/monitoring-plugin/test/e2e/framework" )