diff --git a/go.mod b/go.mod index bcc7afc8a..e57ab90e6 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,8 @@ require ( github.com/openshift/library-go v0.0.0-20240905123346-5bdbfe35a6f5 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.87.0 github.com/prometheus-operator/prometheus-operator/pkg/client v0.87.0 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.67.4 github.com/prometheus/prometheus v0.308.0 github.com/sirupsen/logrus v1.9.3 @@ -57,8 +59,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect - github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/internal/managementrouter/alerts_get.go b/internal/managementrouter/alerts_get.go new file mode 100644 index 000000000..abb0ab462 --- /dev/null +++ b/internal/managementrouter/alerts_get.go @@ -0,0 +1,109 @@ +package managementrouter + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/openshift/monitoring-plugin/pkg/k8s" +) + +type GetAlertsResponse struct { + Data GetAlertsResponseData `json:"data"` + Warnings []string `json:"warnings,omitempty"` +} + +type GetAlertsResponseData struct { + Alerts []k8s.PrometheusAlert `json:"alerts"` +} + +func (hr *httpRouter) GetAlerts(w http.ResponseWriter, req *http.Request) { + state, labels, err := parseStateAndLabels(req.URL.Query()) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + ctx := req.Context() + + alerts, err := hr.managementClient.GetAlerts(ctx, k8s.GetAlertsRequest{ + Labels: labels, + State: state, + }) + if err != nil { + handleError(w, err) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(GetAlertsResponse{ + Data: GetAlertsResponseData{ + Alerts: alerts, + }, + Warnings: hr.alertWarnings(ctx), + }); err != nil { + log.WithError(err).Warn("failed to encode alerts response") + } +} + +func (hr *httpRouter) alertWarnings(ctx context.Context) []string { + health, ok := hr.alertingHealth(ctx) + if !ok { + return nil + } + + warnings := []string{} + if health.UserWorkloadEnabled && health.UserWorkload != nil { + warnings = append(warnings, buildRouteWarnings(health.UserWorkload.Prometheus, k8s.UserWorkloadRouteName, "user workload Prometheus")...) + warnings = append(warnings, buildRouteWarnings(health.UserWorkload.Alertmanager, k8s.UserWorkloadAlertmanagerRouteName, "user workload Alertmanager")...) + } + + return warnings +} + +//nolint:unused // used by the rules listing handler in a subsequent branch +func (hr *httpRouter) rulesWarnings(ctx context.Context) []string { + health, ok := hr.alertingHealth(ctx) + if !ok { + return nil + } + + if health.UserWorkloadEnabled && health.UserWorkload != nil { + return buildRouteWarnings(health.UserWorkload.Prometheus, k8s.UserWorkloadRouteName, "user workload Prometheus") + } + + return nil +} + +func (hr *httpRouter) alertingHealth(ctx context.Context) (k8s.AlertingHealth, bool) { + if hr.managementClient == nil { + return k8s.AlertingHealth{}, false + } + + health, err := hr.managementClient.GetAlertingHealth(ctx) + if err != nil { + log.WithError(err).Warn("alerting health unavailable") + return k8s.AlertingHealth{}, false + } + + return health, true +} + +func buildRouteWarnings(route k8s.AlertingRouteHealth, expectedName string, friendlyName string) []string { + if route.Name != "" && route.Name != expectedName { + return nil + } + if route.FallbackReachable { + return nil + } + + switch route.Status { + case k8s.RouteNotFound: + return []string{friendlyName + " route is missing"} + case k8s.RouteUnreachable: + return []string{friendlyName + " route is unreachable"} + default: + return nil + } +} diff --git a/internal/managementrouter/alerts_get_test.go b/internal/managementrouter/alerts_get_test.go new file mode 100644 index 000000000..1c931a00b --- /dev/null +++ b/internal/managementrouter/alerts_get_test.go @@ -0,0 +1,380 @@ +package managementrouter_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus/prometheus/model/relabel" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/openshift/monitoring-plugin/internal/managementrouter" + alertrule "github.com/openshift/monitoring-plugin/pkg/alert_rule" + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management" + "github.com/openshift/monitoring-plugin/pkg/management/testutils" + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +// agFixture holds mocks and the router for GetAlerts handler tests. +type agFixture struct { + router http.Handler + mockK8s *testutils.MockClient + mockPrometheusAlerts *testutils.MockPrometheusAlertsInterface +} + +func newAGFixture(t *testing.T) *agFixture { + t.Helper() + f := &agFixture{ + mockPrometheusAlerts: &testutils.MockPrometheusAlertsInterface{}, + } + f.mockK8s = &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return f.mockPrometheusAlerts + }, + } + f.rebuild() + return f +} + +func (f *agFixture) rebuild() { + mgmt := management.New(context.Background(), f.mockK8s) + f.router = managementrouter.New(mgmt) +} + +func (f *agFixture) get(t *testing.T, url string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, url, nil) + req.Header.Set("Authorization", "Bearer test-token") + w := httptest.NewRecorder() + f.router.ServeHTTP(w, req) + return w +} + +func decodeAlertsResp(t *testing.T, w *httptest.ResponseRecorder) managementrouter.GetAlertsResponse { + t.Helper() + var resp managementrouter.GetAlertsResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + return resp +} + +func TestGetAlerts_ParsesFlatQueryParams(t *testing.T) { + f := newAGFixture(t) + var captured k8s.GetAlertsRequest + f.mockPrometheusAlerts.GetAlertsFunc = func(_ context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + captured = req + return []k8s.PrometheusAlert{}, nil + } + + w := f.get(t, "/api/v1/alerting/alerts?namespace=ns1&severity=critical&state=firing&team=sre") + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + if captured.State != "firing" { + t.Errorf("expected state=firing, got %q", captured.State) + } + if captured.Labels["namespace"] != "ns1" { + t.Errorf("expected namespace=ns1, got %q", captured.Labels["namespace"]) + } + if captured.Labels["severity"] != "critical" { + t.Errorf("expected severity=critical, got %q", captured.Labels["severity"]) + } + if captured.Labels["team"] != "sre" { + t.Errorf("expected team=sre, got %q", captured.Labels["team"]) + } +} + +func TestGetAlerts_ReturnsAllAlerts(t *testing.T) { + f := newAGFixture(t) + testAlerts := []k8s.PrometheusAlert{ + { + Labels: map[string]string{managementlabels.AlertNameLabel: "HighCPUUsage", "severity": "warning", "namespace": "default"}, + Annotations: map[string]string{"description": "CPU usage is high"}, + State: "firing", + ActiveAt: time.Now(), + }, + { + Labels: map[string]string{managementlabels.AlertNameLabel: "LowMemory", "severity": "critical", "namespace": "monitoring"}, + Annotations: map[string]string{"description": "Memory is running low"}, + State: "firing", + ActiveAt: time.Now(), + }, + } + f.mockPrometheusAlerts.SetActiveAlerts(testAlerts) + + w := f.get(t, "/api/v1/alerting/alerts") + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + if ct := w.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("expected Content-Type application/json, got %q", ct) + } + resp := decodeAlertsResp(t, w) + if len(resp.Data.Alerts) != 2 { + t.Fatalf("expected 2 alerts, got %d", len(resp.Data.Alerts)) + } + if resp.Data.Alerts[0].Labels[managementlabels.AlertNameLabel] != "HighCPUUsage" { + t.Errorf("alert[0] name mismatch: %s", resp.Data.Alerts[0].Labels[managementlabels.AlertNameLabel]) + } + if resp.Data.Alerts[1].Labels[managementlabels.AlertNameLabel] != "LowMemory" { + t.Errorf("alert[1] name mismatch: %s", resp.Data.Alerts[1].Labels[managementlabels.AlertNameLabel]) + } +} + +func TestGetAlerts_WarningsWhenUserWorkloadRoutesMissing(t *testing.T) { + f := newAGFixture(t) + f.mockK8s.AlertingHealthFunc = func(_ context.Context) (k8s.AlertingHealth, error) { + return k8s.AlertingHealth{ + UserWorkloadEnabled: true, + UserWorkload: &k8s.AlertingStackHealth{ + Prometheus: k8s.AlertingRouteHealth{Status: k8s.RouteNotFound}, + Alertmanager: k8s.AlertingRouteHealth{Status: k8s.RouteNotFound}, + }, + }, nil + } + f.rebuild() + + w := f.get(t, "/api/v1/alerting/alerts") + resp := decodeAlertsResp(t, w) + + warnSet := make(map[string]bool) + for _, w := range resp.Warnings { + warnSet[w] = true + } + if !warnSet["user workload Prometheus route is missing"] { + t.Errorf("expected Prometheus route warning, got: %v", resp.Warnings) + } + if !warnSet["user workload Alertmanager route is missing"] { + t.Errorf("expected Alertmanager route warning, got: %v", resp.Warnings) + } +} + +func TestGetAlerts_SuppressesWarningsWhenFallbacksHealthy(t *testing.T) { + f := newAGFixture(t) + f.mockK8s.AlertingHealthFunc = func(_ context.Context) (k8s.AlertingHealth, error) { + return k8s.AlertingHealth{ + UserWorkloadEnabled: true, + UserWorkload: &k8s.AlertingStackHealth{ + Prometheus: k8s.AlertingRouteHealth{Status: k8s.RouteUnreachable, FallbackReachable: true}, + Alertmanager: k8s.AlertingRouteHealth{Status: k8s.RouteUnreachable, FallbackReachable: true}, + }, + }, nil + } + f.rebuild() + + w := f.get(t, "/api/v1/alerting/alerts") + resp := decodeAlertsResp(t, w) + if len(resp.Warnings) != 0 { + t.Errorf("expected no warnings, got: %v", resp.Warnings) + } +} + +func TestGetAlerts_ReturnsEmptyWhenNoAlerts(t *testing.T) { + f := newAGFixture(t) + f.mockPrometheusAlerts.SetActiveAlerts([]k8s.PrometheusAlert{}) + + w := f.get(t, "/api/v1/alerting/alerts") + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + resp := decodeAlertsResp(t, w) + if len(resp.Data.Alerts) != 0 { + t.Errorf("expected empty alerts, got %d", len(resp.Data.Alerts)) + } +} + +func TestGetAlerts_Returns500OnError(t *testing.T) { + f := newAGFixture(t) + f.mockPrometheusAlerts.GetAlertsFunc = func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return nil, fmt.Errorf("connection error") + } + + w := f.get(t, "/api/v1/alerting/alerts") + if w.Code != http.StatusInternalServerError { + t.Fatalf("expected 500, got %d: %s", w.Code, w.Body) + } + if body := w.Body.String(); !strings.Contains(body, "An unexpected error occurred") { + t.Errorf("expected error message, got: %s", body) + } +} + +func TestGetAlerts_ForwardsBearerToken(t *testing.T) { + f := newAGFixture(t) + var capturedCtx context.Context + f.mockPrometheusAlerts.GetAlertsFunc = func(ctx context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + capturedCtx = ctx + return []k8s.PrometheusAlert{}, nil + } + + req := httptest.NewRequest(http.MethodGet, "/api/v1/alerting/alerts", nil) + req.Header.Set("Authorization", "Bearer test-token-abc123") + w := httptest.NewRecorder() + f.router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + if token := k8s.BearerTokenFromContext(capturedCtx); token != "test-token-abc123" { + t.Errorf("expected token test-token-abc123, got %q", token) + } +} + +func TestGetAlerts_MissingAuthHeaderReturns401(t *testing.T) { + f := newAGFixture(t) + req := httptest.NewRequest(http.MethodGet, "/api/v1/alerting/alerts", nil) + w := httptest.NewRecorder() + f.router.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("expected 401, got %d: %s", w.Code, w.Body) + } +} + +func TestGetAlerts_EnrichesAlertWithRuleId(t *testing.T) { + f := newAGFixture(t) + baseRule := monitoringv1.Rule{ + Alert: "HighCPU", + Expr: intstr.FromString("node_cpu > 0.9"), + Labels: map[string]string{"severity": "critical"}, + } + ruleId := alertrule.GetAlertingRuleId(&baseRule) + + relabeledRule := monitoringv1.Rule{ + Alert: "HighCPU", + Expr: intstr.FromString("node_cpu > 0.9"), + Labels: map[string]string{ + managementlabels.AlertNameLabel: "HighCPU", + "severity": "critical", + k8s.AlertRuleLabelId: ruleId, + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "cluster-cpu-rules", + managementlabels.AlertingRuleLabelName: "my-alerting-rule", + }, + } + + f.mockK8s.RelabeledRulesFunc = func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{relabeledRule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == ruleId { + return relabeledRule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + } + f.mockK8s.NamespaceFunc = func() k8s.NamespaceInterface { + return &testutils.MockNamespaceInterface{ + IsClusterMonitoringNamespaceFunc: func(name string) bool { return name == "openshift-monitoring" }, + } + } + f.mockPrometheusAlerts.SetActiveAlerts([]k8s.PrometheusAlert{ + { + Labels: map[string]string{ + managementlabels.AlertNameLabel: "HighCPU", + "severity": "critical", + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.AlertBackendLabel: "alertmanager", + }, + Annotations: map[string]string{"summary": "CPU is high"}, + State: "firing", + ActiveAt: time.Now(), + }, + }) + f.rebuild() + + w := f.get(t, "/api/v1/alerting/alerts") + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + resp := decodeAlertsResp(t, w) + if len(resp.Data.Alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(resp.Data.Alerts)) + } + alert := resp.Data.Alerts[0] + if alert.AlertRuleId != ruleId { + t.Errorf("expected ruleId %s, got %s", ruleId, alert.AlertRuleId) + } + if alert.AlertComponent == "" { + t.Error("expected non-empty AlertComponent") + } + if alert.AlertLayer == "" { + t.Error("expected non-empty AlertLayer") + } +} + +func TestGetAlerts_EnrichesWithoutAlertingRuleCR(t *testing.T) { + f := newAGFixture(t) + baseRule := monitoringv1.Rule{ + Alert: "KubePodCrashLooping", + Expr: intstr.FromString("rate(kube_pod_restart_total[5m]) > 0"), + Labels: map[string]string{"severity": "warning"}, + } + ruleId := alertrule.GetAlertingRuleId(&baseRule) + + relabeledRule := monitoringv1.Rule{ + Alert: "KubePodCrashLooping", + Expr: intstr.FromString("rate(kube_pod_restart_total[5m]) > 0"), + Labels: map[string]string{ + managementlabels.AlertNameLabel: "KubePodCrashLooping", + "severity": "warning", + k8s.AlertRuleLabelId: ruleId, + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "kube-state-metrics", + }, + } + + f.mockK8s.RelabeledRulesFunc = func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{relabeledRule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == ruleId { + return relabeledRule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + } + f.mockK8s.NamespaceFunc = func() k8s.NamespaceInterface { + return &testutils.MockNamespaceInterface{ + IsClusterMonitoringNamespaceFunc: func(name string) bool { return name == "openshift-monitoring" }, + } + } + f.mockPrometheusAlerts.SetActiveAlerts([]k8s.PrometheusAlert{ + { + Labels: map[string]string{ + managementlabels.AlertNameLabel: "KubePodCrashLooping", + "severity": "warning", + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.AlertBackendLabel: "alertmanager", + }, + State: "firing", + ActiveAt: time.Now(), + }, + }) + f.rebuild() + + w := f.get(t, "/api/v1/alerting/alerts") + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + resp := decodeAlertsResp(t, w) + if len(resp.Data.Alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(resp.Data.Alerts)) + } + if resp.Data.Alerts[0].AlertRuleId != ruleId { + t.Errorf("expected ruleId %s, got %s", ruleId, resp.Data.Alerts[0].AlertRuleId) + } +} diff --git a/internal/managementrouter/health_get.go b/internal/managementrouter/health_get.go new file mode 100644 index 000000000..2db846c85 --- /dev/null +++ b/internal/managementrouter/health_get.go @@ -0,0 +1,32 @@ +package managementrouter + +import ( + "encoding/json" + "net/http" + + "github.com/openshift/monitoring-plugin/pkg/k8s" +) + +type GetHealthResponse struct { + Alerting *k8s.AlertingHealth `json:"alerting,omitempty"` +} + +func (hr *httpRouter) GetHealth(w http.ResponseWriter, r *http.Request) { + resp := GetHealthResponse{} + + if hr.managementClient != nil { + health, err := hr.managementClient.GetAlertingHealth(r.Context()) + if err != nil { + handleError(w, err) + return + } + resp.Alerting = &health + } + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + log.WithError(err).Warn("failed to encode health response") + } +} diff --git a/internal/managementrouter/health_get_test.go b/internal/managementrouter/health_get_test.go new file mode 100644 index 000000000..0edd58c18 --- /dev/null +++ b/internal/managementrouter/health_get_test.go @@ -0,0 +1,278 @@ +package managementrouter_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/client-go/rest" + + "github.com/openshift/monitoring-plugin/internal/managementrouter" + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management" +) + +// stubClient is a configurable stub implementing management.Client. +// Fields are set per-test; all methods default to no-op returns. +type stubClient struct { + getRules func(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) + alertingHealth func(ctx context.Context) (k8s.AlertingHealth, error) +} + +func (s *stubClient) ListRules(_ context.Context, _ management.PrometheusRuleOptions, _ management.AlertRuleOptions, _ management.PaginationOptions) (management.ListRulesResult, error) { + return management.ListRulesResult{}, nil +} + +func (s *stubClient) GetRuleById(_ context.Context, _ string) (monitoringv1.Rule, error) { + return monitoringv1.Rule{}, nil +} + +func (s *stubClient) CreateUserDefinedAlertRule(_ context.Context, _ monitoringv1.Rule, _ management.PrometheusRuleOptions) (string, error) { + return "", nil +} + +func (s *stubClient) CreatePlatformAlertRule(_ context.Context, _ monitoringv1.Rule) (string, error) { + return "", nil +} + +func (s *stubClient) UpdateUserDefinedAlertRule(_ context.Context, _ string, _ monitoringv1.Rule) (string, error) { + return "", nil +} +func (s *stubClient) DeleteAlertRuleById(_ context.Context, _ string) error { return nil } +func (s *stubClient) UpdatePlatformAlertRule(_ context.Context, _ string, _ monitoringv1.Rule) error { + return nil +} +func (s *stubClient) DropAlertRule(_ context.Context, _ string) error { return nil } +func (s *stubClient) RestoreAlertRule(_ context.Context, _ string) error { return nil } +func (s *stubClient) UpdateAlertRuleLabels(_ context.Context, _ string, _ map[string]*string) (string, error) { + return "", nil +} + +func (s *stubClient) GetAlerts(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return nil, nil +} + +func (s *stubClient) GetRules(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + if s.getRules != nil { + return s.getRules(ctx, req) + } + return []k8s.PrometheusRuleGroup{}, nil +} + +func (s *stubClient) GetAlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) { + if s.alertingHealth != nil { + return s.alertingHealth(ctx) + } + return k8s.AlertingHealth{}, nil +} + +func (s *stubClient) UpdateAlertRuleClassification(_ context.Context, _ management.UpdateRuleClassificationRequest) error { + return nil +} + +func (s *stubClient) BulkUpdateAlertRuleClassification(_ context.Context, _ []management.UpdateRuleClassificationRequest) []error { + return nil +} + +func (s *stubClient) MetricsHandler(_ context.Context, _ *rest.Config) (http.Handler, error) { + return nil, nil +} + +// newStubRouter builds a router backed by stub and adds a Bearer token header +// to requests via the helper get/getNoAuth methods. +func newStubRouter(stub *stubClient) http.Handler { + return managementrouter.New(stub) +} + +func stubGet(router http.Handler, url string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, url, nil) + req.Header.Set("Authorization", "Bearer test-token") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + return w +} + +// --- health_get tests --- + +func healthStub() *stubClient { + return &stubClient{ + alertingHealth: func(_ context.Context) (k8s.AlertingHealth, error) { + return k8s.AlertingHealth{ + Platform: &k8s.AlertingStackHealth{ + Prometheus: k8s.AlertingRouteHealth{Name: "prometheus-k8s", Namespace: "openshift-monitoring", Status: k8s.RouteReachable}, + Alertmanager: k8s.AlertingRouteHealth{Name: "alertmanager-main", Namespace: "openshift-monitoring", Status: k8s.RouteReachable}, + }, + UserWorkloadEnabled: true, + UserWorkload: &k8s.AlertingStackHealth{ + Prometheus: k8s.AlertingRouteHealth{Name: "prometheus-user-workload", Namespace: "openshift-user-workload-monitoring", Status: k8s.RouteReachable}, + Alertmanager: k8s.AlertingRouteHealth{Name: "alertmanager-user-workload", Namespace: "openshift-user-workload-monitoring", Status: k8s.RouteReachable}, + }, + }, nil + }, + } +} + +func TestGetHealth_Returns200(t *testing.T) { + router := newStubRouter(healthStub()) + w := stubGet(router, "/api/v1/alerting/health") + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } +} + +func TestGetHealth_ReturnsAlertingStructure(t *testing.T) { + router := newStubRouter(healthStub()) + w := stubGet(router, "/api/v1/alerting/health") + + var response managementrouter.GetHealthResponse + if err := json.NewDecoder(w.Body).Decode(&response); err != nil { + t.Fatalf("decode error: %v", err) + } + if response.Alerting == nil { + t.Error("expected non-nil Alerting in response") + } +} + +func TestGetHealth_Returns500OnError(t *testing.T) { + stub := &stubClient{ + alertingHealth: func(_ context.Context) (k8s.AlertingHealth, error) { + return k8s.AlertingHealth{}, fmt.Errorf("connection refused") + }, + } + router := newStubRouter(stub) + w := stubGet(router, "/api/v1/alerting/health") + + if w.Code != http.StatusInternalServerError { + t.Fatalf("expected 500, got %d: %s", w.Code, w.Body) + } + var errResp map[string]string + if err := json.NewDecoder(w.Body).Decode(&errResp); err != nil { + t.Fatalf("decode error: %v", err) + } + if errResp["error"] != "An unexpected error occurred" { + t.Errorf("unexpected error message: %q", errResp["error"]) + } +} + +// --- rules_get tests --- + +func TestGetRules_ParsesFlatQueryParams(t *testing.T) { + stub := &stubClient{} + var captured k8s.GetRulesRequest + stub.getRules = func(_ context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + captured = req + return []k8s.PrometheusRuleGroup{}, nil + } + router := newStubRouter(stub) + w := stubGet(router, "/api/v1/alerting/rules?namespace=ns1&severity=critical&state=firing&team=sre") + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + if captured.State != "firing" { + t.Errorf("expected state=firing, got %q", captured.State) + } + for k, want := range map[string]string{"namespace": "ns1", "severity": "critical", "team": "sre"} { + if got := captured.Labels[k]; got != want { + t.Errorf("label[%s]: want %q, got %q", k, want, got) + } + } +} + +func TestGetRules_ReturnsGroupsInResponse(t *testing.T) { + stub := &stubClient{ + getRules: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return []k8s.PrometheusRuleGroup{{Name: "group-a"}}, nil + }, + } + router := newStubRouter(stub) + w := stubGet(router, "/api/v1/alerting/rules") + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body) + } + if ct := w.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("expected application/json, got %q", ct) + } + var response managementrouter.GetRulesResponse + if err := json.NewDecoder(w.Body).Decode(&response); err != nil { + t.Fatalf("decode error: %v", err) + } + if len(response.Data.Groups) != 1 || response.Data.Groups[0].Name != "group-a" { + t.Errorf("unexpected groups: %v", response.Data.Groups) + } +} + +func TestGetRules_WarningWhenUserWorkloadPromRouteMissing(t *testing.T) { + stub := &stubClient{ + alertingHealth: func(_ context.Context) (k8s.AlertingHealth, error) { + return k8s.AlertingHealth{ + UserWorkloadEnabled: true, + UserWorkload: &k8s.AlertingStackHealth{ + Prometheus: k8s.AlertingRouteHealth{Status: k8s.RouteNotFound}, + }, + }, nil + }, + } + router := newStubRouter(stub) + w := stubGet(router, "/api/v1/alerting/rules") + + var response managementrouter.GetRulesResponse + if err := json.NewDecoder(w.Body).Decode(&response); err != nil { + t.Fatalf("decode error: %v", err) + } + found := false + for _, warn := range response.Warnings { + if warn == "user workload Prometheus route is missing" { + found = true + break + } + } + if !found { + t.Errorf("expected Prometheus route warning, got: %v", response.Warnings) + } +} + +func TestGetRules_SuppressesWarningWhenFallbackHealthy(t *testing.T) { + stub := &stubClient{ + alertingHealth: func(_ context.Context) (k8s.AlertingHealth, error) { + return k8s.AlertingHealth{ + UserWorkloadEnabled: true, + UserWorkload: &k8s.AlertingStackHealth{ + Prometheus: k8s.AlertingRouteHealth{Status: k8s.RouteUnreachable, FallbackReachable: true}, + }, + }, nil + }, + } + router := newStubRouter(stub) + w := stubGet(router, "/api/v1/alerting/rules") + + var response managementrouter.GetRulesResponse + if err := json.NewDecoder(w.Body).Decode(&response); err != nil { + t.Fatalf("decode error: %v", err) + } + if len(response.Warnings) != 0 { + t.Errorf("expected no warnings, got: %v", response.Warnings) + } +} + +func TestGetRules_Returns500OnError(t *testing.T) { + stub := &stubClient{ + getRules: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return nil, fmt.Errorf("connection error") + }, + } + router := newStubRouter(stub) + w := stubGet(router, "/api/v1/alerting/rules") + + if w.Code != http.StatusInternalServerError { + t.Fatalf("expected 500, got %d: %s", w.Code, w.Body) + } + if body := w.Body.String(); !containsStr(body, "An unexpected error occurred") { + t.Errorf("expected error message in body, got: %s", body) + } +} diff --git a/internal/managementrouter/query_filters.go b/internal/managementrouter/query_filters.go new file mode 100644 index 000000000..5f1d58498 --- /dev/null +++ b/internal/managementrouter/query_filters.go @@ -0,0 +1,67 @@ +package managementrouter + +import ( + "fmt" + "net/url" + "strings" +) + +var validStates = map[string]bool{ + "": true, + "pending": true, + "firing": true, + "silenced": true, +} + +// reservedQueryKeys lists query parameter names that have special meaning +// and must not be treated as label equality filters. +var reservedQueryKeys = map[string]bool{ + "state": true, + "match[]": true, + "limit": true, + "next_token": true, +} + +// parseStateLabelsAndMatchers returns the optional state filter, label equality +// matches, and Prometheus-style label matchers from the query string. +// +// Reserved keys ("state", "match[]") are handled specially. Every other key is +// treated as a label equality filter (e.g. ?severity=critical). +// +// match[] values follow upstream Prometheus API conventions and may contain +// equality, inequality, regex, or negative-regex matchers: +// +// ?match[]=severity="critical"&match[]=alertname=~"Kube.*" +func parseStateLabelsAndMatchers(q url.Values) (string, map[string]string, []string, error) { + state := strings.ToLower(strings.TrimSpace(q.Get("state"))) + if !validStates[state] { + return "", nil, nil, fmt.Errorf("invalid state filter %q: must be one of pending, firing, silenced", q.Get("state")) + } + + labels := make(map[string]string) + for key, vals := range q { + if reservedQueryKeys[key] { + continue + } + if len(vals) > 0 && strings.TrimSpace(vals[0]) != "" { + labels[strings.TrimSpace(key)] = strings.TrimSpace(vals[0]) + } + } + + var matchers []string + for _, raw := range q["match[]"] { + v := strings.TrimSpace(raw) + if v != "" { + matchers = append(matchers, v) + } + } + + return state, labels, matchers, nil +} + +// parseStateAndLabels returns the optional state filter and label matches. +// Any query param other than reserved keys is treated as a label match. +func parseStateAndLabels(q url.Values) (string, map[string]string, error) { + state, labels, _, err := parseStateLabelsAndMatchers(q) + return state, labels, err +} diff --git a/internal/managementrouter/query_filters_test.go b/internal/managementrouter/query_filters_test.go new file mode 100644 index 000000000..e417245e1 --- /dev/null +++ b/internal/managementrouter/query_filters_test.go @@ -0,0 +1,166 @@ +package managementrouter + +import ( + "net/url" + "testing" +) + +func TestParseStateLabelsAndMatchers(t *testing.T) { + tests := []struct { + name string + query string + wantState string + wantLabels map[string]string + wantMatchers []string + wantMatchersLen int + wantErr bool + }{ + { + name: "empty query", + query: "", + wantState: "", + wantLabels: map[string]string{}, + wantMatchers: nil, + }, + { + name: "state only", + query: "state=firing", + wantState: "firing", + wantLabels: map[string]string{}, + }, + { + name: "flat labels only", + query: "severity=critical&namespace=openshift-monitoring", + wantState: "", + wantLabels: map[string]string{ + "severity": "critical", + "namespace": "openshift-monitoring", + }, + }, + { + name: "match[] only with equality", + query: `match[]=severity="critical"`, + wantState: "", + wantLabels: map[string]string{}, + wantMatchers: []string{ + `severity="critical"`, + }, + }, + { + name: "match[] with regex", + query: `match[]=alertname=~"Kube.*CPU.*"`, + wantState: "", + wantLabels: map[string]string{}, + wantMatchers: []string{ + `alertname=~"Kube.*CPU.*"`, + }, + }, + { + name: "multiple match[] values", + query: `match[]=severity="critical"&match[]=namespace="openshift-monitoring"`, + wantState: "", + wantLabels: map[string]string{}, + wantMatchersLen: 2, + }, + { + name: "mixed flat labels and match[]", + query: `state=firing&team=sre&match[]=severity=~"critical|warning"`, + wantState: "firing", + wantLabels: map[string]string{ + "team": "sre", + }, + wantMatchers: []string{ + `severity=~"critical|warning"`, + }, + }, + { + name: "match[] is not treated as a label", + query: `match[]=severity="critical"`, + wantState: "", + wantLabels: map[string]string{}, + }, + { + name: "invalid state", + query: "state=invalid", + wantErr: true, + }, + { + name: "empty match[] values are skipped", + query: `match[]=&match[]=%20&match[]=severity="warning"`, + wantState: "", + wantLabels: map[string]string{}, + wantMatchersLen: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + q, err := url.ParseQuery(tt.query) + if err != nil { + t.Fatalf("invalid test query: %v", err) + } + + state, labels, matchers, err := parseStateLabelsAndMatchers(q) + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if state != tt.wantState { + t.Errorf("state = %q, want %q", state, tt.wantState) + } + + if tt.wantLabels != nil { + if len(labels) != len(tt.wantLabels) { + t.Errorf("labels length = %d, want %d", len(labels), len(tt.wantLabels)) + } + for k, v := range tt.wantLabels { + if labels[k] != v { + t.Errorf("labels[%q] = %q, want %q", k, labels[k], v) + } + } + if _, found := labels["match[]"]; found { + t.Error("match[] should not appear in labels map") + } + } + + if tt.wantMatchers != nil { + if len(matchers) != len(tt.wantMatchers) { + t.Errorf("matchers length = %d, want %d", len(matchers), len(tt.wantMatchers)) + } + for i, want := range tt.wantMatchers { + if i < len(matchers) && matchers[i] != want { + t.Errorf("matchers[%d] = %q, want %q", i, matchers[i], want) + } + } + } + + if tt.wantMatchersLen > 0 && len(matchers) != tt.wantMatchersLen { + t.Errorf("matchers length = %d, want %d", len(matchers), tt.wantMatchersLen) + } + }) + } +} + +func TestParseStateAndLabelsBackcompat(t *testing.T) { + q, _ := url.ParseQuery(`state=firing&severity=critical&match[]=alertname=~"Foo.*"`) + + state, labels, err := parseStateAndLabels(q) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != "firing" { + t.Errorf("state = %q, want %q", state, "firing") + } + if labels["severity"] != "critical" { + t.Errorf("severity = %q, want %q", labels["severity"], "critical") + } + if _, found := labels["match[]"]; found { + t.Error("match[] should not appear in labels map") + } +} diff --git a/internal/managementrouter/router.go b/internal/managementrouter/router.go index fd0e0cda7..915b62f8c 100644 --- a/internal/managementrouter/router.go +++ b/internal/managementrouter/router.go @@ -43,6 +43,12 @@ func New(managementClient management.Client) *mux.Router { BaseURL: "/api/v1/alerting", BaseRouter: r, }) + // GET /alerts, GET /rules, and GET /health are not yet in the OpenAPI + // spec; registered manually until their respective branches add the spec + // entries. + r.HandleFunc("/api/v1/alerting/alerts", hr.GetAlerts).Methods(http.MethodGet) + r.HandleFunc("/api/v1/alerting/rules", hr.GetRules).Methods(http.MethodGet) + r.HandleFunc("/api/v1/alerting/health", hr.GetHealth).Methods(http.MethodGet) return r } diff --git a/internal/managementrouter/rules_get.go b/internal/managementrouter/rules_get.go new file mode 100644 index 000000000..fe9a59409 --- /dev/null +++ b/internal/managementrouter/rules_get.go @@ -0,0 +1,48 @@ +package managementrouter + +import ( + "encoding/json" + "net/http" + + "github.com/openshift/monitoring-plugin/pkg/k8s" +) + +type GetRulesResponse struct { + Data GetRulesResponseData `json:"data"` + Warnings []string `json:"warnings,omitempty"` +} + +type GetRulesResponseData struct { + Groups []k8s.PrometheusRuleGroup `json:"groups"` +} + +func (hr *httpRouter) GetRules(w http.ResponseWriter, req *http.Request) { + state, labels, matchers, err := parseStateLabelsAndMatchers(req.URL.Query()) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + ctx := req.Context() + + groups, err := hr.managementClient.GetRules(ctx, k8s.GetRulesRequest{ + Labels: labels, + Matchers: matchers, + State: state, + }) + if err != nil { + handleError(w, err) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(GetRulesResponse{ + Data: GetRulesResponseData{ + Groups: groups, + }, + Warnings: hr.rulesWarnings(ctx), + }); err != nil { + log.WithError(err).Warn("failed to encode rules response") + } +} diff --git a/pkg/alertcomponent/matcher.go b/pkg/alertcomponent/matcher.go new file mode 100644 index 000000000..8aa6f9227 --- /dev/null +++ b/pkg/alertcomponent/matcher.go @@ -0,0 +1,381 @@ +package alertcomponent + +import ( + "regexp" + + "github.com/prometheus/common/model" + + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +const ( + labelNamespace = "namespace" + labelSeverity = "severity" +) + +func ns(values ...string) LabelsMatcher { + return NewLabelsMatcher(labelNamespace, NewStringValuesMatcher(values...)) +} + +func alertNames(values ...string) LabelsMatcher { + return NewLabelsMatcher(managementlabels.AlertNameLabel, NewStringValuesMatcher(values...)) +} + +func regexAlertNames(regexes ...*regexp.Regexp) LabelsMatcher { + return NewLabelsMatcher(managementlabels.AlertNameLabel, NewRegexValuesMatcher(regexes...)) +} + +func labelValues(key string, values ...string) LabelsMatcher { + return NewLabelsMatcher(key, NewStringValuesMatcher(values...)) +} + +func comp(component string, ms ...LabelsMatcher) componentMatcher { + return componentMatcher{component: component, matchers: ms} +} + +// LabelsMatcher represents a matcher definition for a set of labels. +// It matches if all of the label matchers match the labels. +type LabelsMatcher interface { + Matches(labels model.LabelSet) (match bool, keys []model.LabelName) + Equals(other LabelsMatcher) bool +} + +func NewLabelsMatcher(key string, matcher ValueMatcher) LabelsMatcher { + return labelMatcher{key: key, matcher: matcher} +} + +func NewStringValuesMatcher(keys ...string) ValueMatcher { + return stringMatcher(keys) +} + +func NewRegexValuesMatcher(regexes ...*regexp.Regexp) ValueMatcher { + return regexpMatcher(regexes) +} + +// labelMatcher represents a matcher definition for a label. +type labelMatcher struct { + key string + matcher ValueMatcher +} + +// Matches implements the LabelsMatcher interface. +func (l labelMatcher) Matches(labels model.LabelSet) (bool, []model.LabelName) { + if l.matcher.Matches(string(labels[model.LabelName(l.key)])) { + return true, []model.LabelName{model.LabelName(l.key)} + } + return false, nil +} + +// Equals implements the LabelsMatcher interface. +func (l labelMatcher) Equals(other LabelsMatcher) bool { + ol, ok := other.(labelMatcher) + if !ok { + return false + } + return l.key == ol.key && l.matcher.Equals(ol.matcher) +} + +// ValueMatcher represents a matcher for a specific value. +// +// Multiple implementations are provided for different types of matchers. +type ValueMatcher interface { + Matches(value string) bool + Equals(other ValueMatcher) bool +} + +// stringMatcher is a matcher for a list of strings. +// +// It matches if the value is in the list of strings. +type stringMatcher []string + +func (s stringMatcher) Matches(value string) bool { + for _, v := range s { + if v == value { + return true + } + } + return false +} + +// Equals implements the ValueMatcher interface. +func (s stringMatcher) Equals(other ValueMatcher) bool { + o, ok := other.(stringMatcher) + if !ok { + return false + } + return equalsNoOrder(s, o) +} + +// regexpMatcher is a matcher for a list of regular expressions. +// +// It matches if the value matches any of the regular expressions. +type regexpMatcher []*regexp.Regexp + +func (r regexpMatcher) Matches(value string) bool { + for _, re := range r { + if re.MatchString(value) { + return true + } + } + return false +} + +// Equals implements the ValueMatcher interface. +func (r regexpMatcher) Equals(other ValueMatcher) bool { + o, ok := other.(regexpMatcher) + if !ok { + return false + } + s1 := make([]string, 0, len(r)) + for _, re := range r { + s1 = append(s1, re.String()) + } + s2 := make([]string, 0, len(o)) + for _, re := range o { + s2 = append(s2, re.String()) + } + return equalsNoOrder(s1, s2) +} + +func equalsNoOrder(a, b []string) bool { + if len(a) != len(b) { + return false + } + + seen := make(map[string]int, len(a)) + for _, v := range a { + seen[v]++ + } + for _, v := range b { + if seen[v] == 0 { + return false + } + seen[v]-- + } + return true +} + +// componentMatcher represents a matcher definition for a component. +// +// It matches if any of the label matchers match the labels. +type componentMatcher struct { + component string + matchers []LabelsMatcher +} + +// findComponent tries to determine a component for given labels using the provided matchers. +// +// It returns the component and the keys that matched. +// If no match is found, it returns an empty component and nil keys. +func findComponent(compMatchers []componentMatcher, labels model.LabelSet) ( + component string, keys []model.LabelName) { + for _, compMatcher := range compMatchers { + for _, labelsMatcher := range compMatcher.matchers { + if matches, keys := labelsMatcher.Matches(labels); matches { + return compMatcher.component, keys + } + } + } + return "", nil +} + +// componentMatcherFn is a function that tries matching provided labels to a component. +// It returns the layer, component and the keys from the labels that were used for matching. +// If no match is found, it returns an empty layer, component and nil keys. +type componentMatcherFn func(labels model.LabelSet) (layer, comp model.LabelValue, keys []model.LabelName) + +func evalMatcherFns(fns []componentMatcherFn, labels model.LabelSet) ( + layer, comp string, labelsSubset model.LabelSet) { + for _, fn := range fns { + if layer, comp, keys := fn(labels); layer != "" { + return string(layer), string(comp), getLabelsSubset(labels, keys...) + } + } + return "Others", "Others", getLabelsSubset(labels) +} + +// getLabelsSubset returns a subset of the labels with given keys. +func getLabelsSubset(m model.LabelSet, keys ...model.LabelName) model.LabelSet { + keys = append([]model.LabelName{ + model.LabelName(labelNamespace), + model.LabelName(managementlabels.AlertNameLabel), + model.LabelName(labelSeverity), + }, keys...) + return getMapSubset(m, keys...) +} + +// getMapSubset returns a subset of the labels with given keys. +func getMapSubset(m model.LabelSet, keys ...model.LabelName) model.LabelSet { + subset := make(model.LabelSet, len(keys)) + for _, key := range keys { + if val, ok := m[key]; ok { + subset[key] = val + } + } + return subset +} + +var ( + nodeAlerts []model.LabelValue = []model.LabelValue{ + "NodeClockNotSynchronising", + "KubeNodeNotReady", + "KubeNodeUnreachable", + "NodeSystemSaturation", + "NodeFilesystemSpaceFillingUp", + "NodeFilesystemAlmostOutOfSpace", + "NodeMemoryMajorPagesFaults", + "NodeNetworkTransmitErrs", + "NodeTextFileCollectorScrapeError", + "NodeFilesystemFilesFillingUp", + "NodeNetworkReceiveErrs", + "NodeClockSkewDetected", + "NodeFilesystemAlmostOutOfFiles", + "NodeWithoutOVNKubeNodePodRunning", + "InfraNodesNeedResizingSRE", + "NodeHighNumberConntrackEntriesUsed", + "NodeMemHigh", + "NodeNetworkInterfaceFlapping", + "NodeWithoutSDNPod", + "NodeCpuHigh", + "CriticalNodeNotReady", + "NodeFileDescriptorLimit", + "MCCPoolAlert", + "MCCDrainError", + "MCDRebootError", + "MCDPivotError", + } + + coreMatchers = []componentMatcher{ + comp("etcd", ns("openshift-etcd", "openshift-etcd-operator")), + comp("kube-apiserver", ns("openshift-kube-apiserver", "openshift-kube-apiserver-operator")), + comp("kube-controller-manager", ns("openshift-kube-controller-manager", "openshift-kube-controller-manager-operator", "kube-system")), + comp("kube-scheduler", ns("openshift-kube-scheduler", "openshift-kube-scheduler-operator")), + comp("machine-approver", ns("openshift-cluster-machine-approver", "openshift-machine-approver-operator")), + comp("machine-config", + ns("openshift-machine-config-operator"), + alertNames( + "HighOverallControlPlaneMemory", + "ExtremelyHighIndividualControlPlaneMemory", + "MissingMachineConfig", + "MCCBootImageUpdateError", + "KubeletHealthState", + "SystemMemoryExceedsReservation", + ), + ), + comp("version", + ns("openshift-cluster-version", "openshift-version-operator"), + alertNames("ClusterNotUpgradeable", "UpdateAvailable"), + ), + comp("dns", ns("openshift-dns", "openshift-dns-operator")), + comp("authentication", ns("openshift-authentication", "openshift-oauth-apiserver", "openshift-authentication-operator")), + comp("cert-manager", ns("openshift-cert-manager", "openshift-cert-manager-operator")), + comp("cloud-controller-manager", ns("openshift-cloud-controller-manager", "openshift-cloud-controller-manager-operator")), + comp("cloud-credential", ns("openshift-cloud-credential-operator")), + comp("cluster-api", ns("openshift-cluster-api", "openshift-cluster-api-operator")), + comp("config-operator", ns("openshift-config-operator")), + comp("kube-storage-version-migrator", ns("openshift-kube-storage-version-migrator", "openshift-kube-storage-version-migrator-operator")), + comp("image-registry", ns("openshift-image-registry", "openshift-image-registry-operator")), + comp("ingress", ns("openshift-ingress", "openshift-route-controller-manager", "openshift-ingress-canary", "openshift-ingress-operator")), + comp("console", ns("openshift-console", "openshift-console-operator")), + comp("insights", ns("openshift-insights", "openshift-insights-operator")), + comp("machine-api", ns("openshift-machine-api", "openshift-machine-api-operator")), + comp("monitoring", ns("openshift-monitoring", "openshift-monitoring-operator")), + comp("network", ns("openshift-network-operator", "openshift-ovn-kubernetes", "openshift-multus", "openshift-network-diagnostics", "openshift-sdn")), + comp("node-tuning", ns("openshift-cluster-node-tuning-operator", "openshift-node-tuning-operator")), + comp("openshift-apiserver", ns("openshift-apiserver", "openshift-apiserver-operator")), + comp("openshift-controller-manager", ns("openshift-controller-manager", "openshift-controller-manager-operator")), + comp("openshift-samples", ns("openshift-cluster-samples-operator", "openshift-samples-operator")), + comp("operator-lifecycle-manager", ns("openshift-operator-lifecycle-manager")), + comp("service-ca", ns("openshift-service-ca", "openshift-service-ca-operator")), + comp("storage", ns("openshift-storage", "openshift-cluster-csi-drivers", "openshift-cluster-storage-operator", "openshift-storage-operator")), + comp("vertical-pod-autoscaler", ns("openshift-vertical-pod-autoscaler", "openshift-vertical-pod-autoscaler-operator")), + comp("marketplace", ns("openshift-marketplace", "openshift-marketplace-operator")), + } + + workloadMatchers = []componentMatcher{ + comp("openshift-compliance", ns("openshift-compliance")), + comp("openshift-file-integrity", ns("openshift-file-integrity")), + comp("openshift-logging", ns("openshift-logging")), + comp("openshift-user-workload-monitoring", ns("openshift-user-workload-monitoring")), + comp("openshift-gitops", ns("openshift-gitops", "openshift-gitops-operator")), + comp("openshift-operators", ns("openshift-operators")), + comp("openshift-local-storage", ns("openshift-local-storage")), + comp("quay", labelValues("container", "quay-app", "quay-mirror", "quay-app-upgrade")), + comp("Argo", regexAlertNames(regexp.MustCompile("^Argo"))), + } +) + +var cvoAlerts = []model.LabelValue{"ClusterOperatorDown", "ClusterOperatorDegraded"} + +func cvoAlertsMatcher(labels model.LabelSet) (layer, comp model.LabelValue, keys []model.LabelName) { + for _, v := range cvoAlerts { + if labels[managementlabels.AlertNameLabel] == v { + component := labels["name"] + if component == "" { + component = "version" + } + return "cluster", component, nil + } + } + return "", "", nil +} + +func kubevirtOperatorMatcher(labels model.LabelSet) (layer, comp model.LabelValue, keys []model.LabelName) { + if labels["kubernetes_operator_part_of"] != "kubevirt" { + return "", "", nil + } + if labels["kubernetes_operator_component"] == "cnv-observability" { + return "", "", nil + } + if labels["operator_health_impact"] == "none" && labels["kubernetes_operator_component"] == "kubevirt" { + return "namespace", "OpenShift Virtualization Virtual Machine", []model.LabelName{ + "kubernetes_operator_part_of", + "kubernetes_operator_component", + "operator_health_impact", + } + } + return "cluster", "OpenShift Virtualization Operator", []model.LabelName{ + "kubernetes_operator_part_of", + "kubernetes_operator_component", + "operator_health_impact", + } +} + +func computeMatcher(labels model.LabelSet) (layer, comp model.LabelValue, keys []model.LabelName) { + for _, nodeAlert := range nodeAlerts { + if labels[managementlabels.AlertNameLabel] == nodeAlert { + component := "compute" + return "cluster", model.LabelValue(component), nil + } + } + return "", "", nil +} + +func coreMatcher(labels model.LabelSet) (layer, comp model.LabelValue, keys []model.LabelName) { + // Try matching against core components. + if component, keys := findComponent(coreMatchers, labels); component != "" { + return "cluster", model.LabelValue(component), keys + } + return "", "", nil +} + +func workloadMatcher(labels model.LabelSet) (layer, comp model.LabelValue, keys []model.LabelName) { + // Try matching against workload components. + if component, keys := findComponent(workloadMatchers, labels); component != "" { + return "namespace", model.LabelValue(component), keys + } + return "", "", nil +} + +// DetermineComponent determines the component for a given set of labels. +// It returns the layer and component strings. +func DetermineComponent(labels model.LabelSet) (layer, component string) { + layer, component, _ = evalMatcherFns([]componentMatcherFn{ + cvoAlertsMatcher, + kubevirtOperatorMatcher, + computeMatcher, + coreMatcher, + workloadMatcher, + }, labels) + return layer, component +} diff --git a/pkg/k8s/alert_relabel_config_gc.go b/pkg/k8s/alert_relabel_config_gc.go new file mode 100644 index 000000000..7c9e92a2e --- /dev/null +++ b/pkg/k8s/alert_relabel_config_gc.go @@ -0,0 +1,52 @@ +package k8s + +import ( + "context" + + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +// gcOrphanedARCs deletes AlertRelabelConfigs whose associated alert rule no +// longer exists. This handles the case where an operator (or manual action) +// removes rules from a PrometheusRule or deletes the CR entirely — the ARCs +// that were created by the plugin for classification/drop/stamp become orphans. +// +// Only ARCs carrying the plugin's alertRuleId annotation are considered. +// GitOps-managed ARCs are never deleted automatically; a warning is logged +// so that operators can clean them up manually. +func (rrm *relabeledRulesManager) gcOrphanedARCs(ctx context.Context, liveRuleIDs map[string]struct{}) { + if rrm.alertRelabelConfigs == nil { + return + } + + arcs, err := rrm.alertRelabelConfigs.List(ctx, "") + if err != nil { + log.Errorf("orphan ARC GC: failed to list ARCs: %v", err) + return + } + + for i := range arcs { + arc := &arcs[i] + + ruleID, ok := arc.Annotations[managementlabels.ARCAnnotationAlertRuleIDKey] + if !ok || ruleID == "" { + continue + } + + if _, alive := liveRuleIDs[ruleID]; alive { + continue + } + + if IsManagedByGitOps(arc.Annotations, arc.Labels) { + log.Warnf("orphan ARC GC: ARC %s/%s (ruleId=%s) is orphaned but GitOps-managed — skipping deletion, manual cleanup required", arc.Namespace, arc.Name, ruleID) + continue + } + + if err := rrm.alertRelabelConfigs.Delete(ctx, arc.Namespace, arc.Name); err != nil { + log.Errorf("orphan ARC GC: failed to delete ARC %s/%s: %v", arc.Namespace, arc.Name, err) + continue + } + + log.Infof("orphan ARC GC: deleted orphaned ARC %s/%s (ruleId=%s)", arc.Namespace, arc.Name, ruleID) + } +} diff --git a/pkg/k8s/alert_relabel_config_gc_test.go b/pkg/k8s/alert_relabel_config_gc_test.go new file mode 100644 index 000000000..e139ad965 --- /dev/null +++ b/pkg/k8s/alert_relabel_config_gc_test.go @@ -0,0 +1,168 @@ +package k8s + +import ( + "context" + "testing" + + osmv1 "github.com/openshift/api/monitoring/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +type mockARCInterface struct { + arcs map[string]*osmv1.AlertRelabelConfig + deleted []string +} + +func (m *mockARCInterface) List(_ context.Context, _ string) ([]osmv1.AlertRelabelConfig, error) { + var result []osmv1.AlertRelabelConfig + for _, arc := range m.arcs { + result = append(result, *arc) + } + return result, nil +} + +func (m *mockARCInterface) Get(_ context.Context, ns, name string) (*osmv1.AlertRelabelConfig, bool, error) { + if arc, ok := m.arcs[ns+"/"+name]; ok { + return arc, true, nil + } + return nil, false, nil +} + +func (m *mockARCInterface) Create(_ context.Context, arc osmv1.AlertRelabelConfig) (*osmv1.AlertRelabelConfig, error) { + return &arc, nil +} + +func (m *mockARCInterface) Update(_ context.Context, _ osmv1.AlertRelabelConfig) error { return nil } + +func (m *mockARCInterface) Delete(_ context.Context, ns, name string) error { + m.deleted = append(m.deleted, ns+"/"+name) + delete(m.arcs, ns+"/"+name) + return nil +} + +func newARC(ns, name, ruleID string, annotations, labels map[string]string) *osmv1.AlertRelabelConfig { + if annotations == nil { + annotations = map[string]string{} + } + if ruleID != "" { + annotations[managementlabels.ARCAnnotationAlertRuleIDKey] = ruleID + } + return &osmv1.AlertRelabelConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Annotations: annotations, + Labels: labels, + }, + } +} + +func TestGCOrphanedARCs_DeletesOrphan(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-orphan": newARC("openshift-monitoring", "arc-orphan", "rule-gone", nil, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) + + if len(mock.deleted) != 1 || mock.deleted[0] != "openshift-monitoring/arc-orphan" { + t.Fatalf("expected orphan ARC to be deleted, got deleted=%v", mock.deleted) + } +} + +func TestGCOrphanedARCs_KeepsLiveRule(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-live": newARC("openshift-monitoring", "arc-live", "rule-alive", nil, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{"rule-alive": {}}) + + if len(mock.deleted) != 0 { + t.Fatalf("expected no deletions, got deleted=%v", mock.deleted) + } +} + +func TestGCOrphanedARCs_SkipsGitOpsManaged(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-gitops": newARC("openshift-monitoring", "arc-gitops", "rule-gone", + map[string]string{"argocd.argoproj.io/tracking-id": "some-id"}, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) + + if len(mock.deleted) != 0 { + t.Fatalf("expected GitOps-managed ARC to be preserved, got deleted=%v", mock.deleted) + } +} + +func TestGCOrphanedARCs_SkipsARCWithoutAnnotation(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-manual": newARC("openshift-monitoring", "arc-manual", "", nil, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) + + if len(mock.deleted) != 0 { + t.Fatalf("expected ARC without annotation to be preserved, got deleted=%v", mock.deleted) + } +} + +func TestGCOrphanedARCs_MixedScenario(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-live": newARC("openshift-monitoring", "arc-live", "rule-1", nil, nil), + "openshift-monitoring/arc-orphan1": newARC("openshift-monitoring", "arc-orphan1", "rule-deleted-1", nil, nil), + "openshift-monitoring/arc-orphan2": newARC("openshift-monitoring", "arc-orphan2", "rule-deleted-2", nil, nil), + "openshift-monitoring/arc-gitops": newARC("openshift-monitoring", "arc-gitops", "rule-deleted-3", + map[string]string{"argocd.argoproj.io/tracking-id": "t"}, nil), + "openshift-monitoring/arc-manual": newARC("openshift-monitoring", "arc-manual", "", nil, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + liveIDs := map[string]struct{}{"rule-1": {}} + rrm.gcOrphanedARCs(context.Background(), liveIDs) + + deletedSet := map[string]bool{} + for _, d := range mock.deleted { + deletedSet[d] = true + } + + if len(mock.deleted) != 2 { + t.Fatalf("expected 2 deletions, got %d: %v", len(mock.deleted), mock.deleted) + } + if !deletedSet["openshift-monitoring/arc-orphan1"] { + t.Error("expected arc-orphan1 to be deleted") + } + if !deletedSet["openshift-monitoring/arc-orphan2"] { + t.Error("expected arc-orphan2 to be deleted") + } + if deletedSet["openshift-monitoring/arc-live"] { + t.Error("arc-live should not have been deleted") + } + if deletedSet["openshift-monitoring/arc-gitops"] { + t.Error("arc-gitops should not have been deleted (GitOps-managed)") + } + if deletedSet["openshift-monitoring/arc-manual"] { + t.Error("arc-manual should not have been deleted (no annotation)") + } +} + +func TestGCOrphanedARCs_NilInterface(t *testing.T) { + rrm := &relabeledRulesManager{alertRelabelConfigs: nil} + // Should not panic + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) +} diff --git a/pkg/k8s/alerting_health.go b/pkg/k8s/alerting_health.go new file mode 100644 index 000000000..0fdc40880 --- /dev/null +++ b/pkg/k8s/alerting_health.go @@ -0,0 +1,127 @@ +package k8s + +import ( + "context" + "fmt" + "strings" + "sync" + + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +const ( + clusterMonitoringConfigMap = "cluster-monitoring-config" + clusterMonitoringConfigKey = "config.yaml" +) + +type clusterMonitoringConfig struct { + EnableUserWorkload bool `yaml:"enableUserWorkload"` +} + +// clusterMonitoringConfigManager watches the cluster-monitoring-config ConfigMap +// via an informer and caches the parsed enableUserWorkload value so that +// AlertingHealth never needs a live API call. +type clusterMonitoringConfigManager struct { + informer cache.SharedIndexInformer + + mu sync.RWMutex + enabled bool + err error +} + +func newClusterMonitoringConfigManager(ctx context.Context, clientset *kubernetes.Clientset) (*clusterMonitoringConfigManager, error) { + informer := cache.NewSharedIndexInformer( + cache.NewListWatchFromClient( + clientset.CoreV1().RESTClient(), + "configmaps", + ClusterMonitoringNamespace, + fields.OneTermEqualSelector("metadata.name", clusterMonitoringConfigMap), + ), + &corev1.ConfigMap{}, + 0, + cache.Indexers{}, + ) + + m := &clusterMonitoringConfigManager{ + informer: informer, + } + + _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + cm, ok := obj.(*corev1.ConfigMap) + if !ok { + return + } + m.handleUpdate(cm) + }, + UpdateFunc: func(_, newObj interface{}) { + cm, ok := newObj.(*corev1.ConfigMap) + if !ok { + return + } + m.handleUpdate(cm) + }, + DeleteFunc: func(_ interface{}) { + m.mu.Lock() + defer m.mu.Unlock() + m.enabled = false + m.err = nil + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to add event handler to cluster-monitoring-config informer: %w", err) + } + + go informer.Run(ctx.Done()) + + if !cache.WaitForNamedCacheSync("ClusterMonitoringConfig informer", ctx.Done(), informer.HasSynced) { + return nil, fmt.Errorf("failed to sync ClusterMonitoringConfig informer") + } + + return m, nil +} + +func (m *clusterMonitoringConfigManager) handleUpdate(cm *corev1.ConfigMap) { + m.mu.Lock() + defer m.mu.Unlock() + + raw, ok := cm.Data[clusterMonitoringConfigKey] + if !ok || strings.TrimSpace(raw) == "" { + m.enabled = false + m.err = nil + return + } + + var cfg clusterMonitoringConfig + if err := yaml.Unmarshal([]byte(raw), &cfg); err != nil { + m.enabled = false + m.err = fmt.Errorf("parse cluster monitoring config.yaml: %w", err) + return + } + + m.enabled = cfg.EnableUserWorkload + m.err = nil +} + +func (m *clusterMonitoringConfigManager) userWorkloadEnabled() (bool, error) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.enabled, m.err +} + +// AlertingHealth returns alerting route health and UWM enablement status. +func (c *client) AlertingHealth(ctx context.Context) (AlertingHealth, error) { + health := c.prometheusAlerts.alertingHealth(ctx) + + enabled, err := c.clusterMonitoringConfig.userWorkloadEnabled() + if err != nil { + return health, fmt.Errorf("failed to determine user workload enablement: %w", err) + } + health.UserWorkloadEnabled = enabled + + return health, nil +} diff --git a/pkg/k8s/client.go b/pkg/k8s/client.go index 6370270ff..e16be6dd2 100644 --- a/pkg/k8s/client.go +++ b/pkg/k8s/client.go @@ -5,6 +5,7 @@ import ( "fmt" osmv1client "github.com/openshift/client-go/monitoring/clientset/versioned" + routeclient "github.com/openshift/client-go/route/clientset/versioned" monitoringv1client "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" @@ -21,11 +22,14 @@ type client struct { osmv1clientset *osmv1client.Clientset config *rest.Config + prometheusAlerts *prometheusAlerts + prometheusRuleManager *prometheusRuleManager alertRelabelConfigManager *alertRelabelConfigManager alertingRuleManager *alertingRuleManager namespaceManager *namespaceManager relabeledRulesManager *relabeledRulesManager + clusterMonitoringConfig *clusterMonitoringConfigManager } func NewClient(ctx context.Context, config *rest.Config) (Client, error) { @@ -44,6 +48,11 @@ func NewClient(ctx context.Context, config *rest.Config) (Client, error) { return nil, fmt.Errorf("failed to create osmv1 clientset: %w", err) } + routeClientset, err := routeclient.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create route clientset: %w", err) + } + c := &client{ clientset: clientset, monitoringv1clientset: monitoringv1clientset, @@ -56,6 +65,8 @@ func NewClient(ctx context.Context, config *rest.Config) (Client, error) { return nil, fmt.Errorf("failed to create PrometheusRule manager: %w", err) } + c.prometheusAlerts = newPrometheusAlerts(routeClientset, clientset.CoreV1(), config, c.prometheusRuleManager) + c.alertRelabelConfigManager, err = newAlertRelabelConfigManager(ctx, osmv1clientset, config) if err != nil { return nil, fmt.Errorf("failed to create alert relabel config manager: %w", err) @@ -71,6 +82,11 @@ func NewClient(ctx context.Context, config *rest.Config) (Client, error) { return nil, fmt.Errorf("failed to create namespace manager: %w", err) } + c.clusterMonitoringConfig, err = newClusterMonitoringConfigManager(ctx, clientset) + if err != nil { + return nil, fmt.Errorf("failed to create cluster monitoring config manager: %w", err) + } + c.relabeledRulesManager, err = newRelabeledRulesManager(ctx, c.namespaceManager, c.alertRelabelConfigManager, monitoringv1clientset, clientset) if err != nil { return nil, fmt.Errorf("failed to create relabeled rules config manager: %w", err) @@ -87,6 +103,10 @@ func (c *client) TestConnection(_ context.Context) error { return nil } +func (c *client) PrometheusAlerts() PrometheusAlertsInterface { + return c.prometheusAlerts +} + func (c *client) PrometheusRules() PrometheusRuleInterface { return c.prometheusRuleManager } diff --git a/pkg/k8s/const.go b/pkg/k8s/const.go index 243cea8d8..ff9eaf4c5 100644 --- a/pkg/k8s/const.go +++ b/pkg/k8s/const.go @@ -1,5 +1,31 @@ package k8s const ( - ClusterMonitoringNamespace = "openshift-monitoring" + ClusterMonitoringNamespace = "openshift-monitoring" + UserWorkloadMonitoringNamespace = "openshift-user-workload-monitoring" + + PlatformRouteName = "prometheus-k8s" + PlatformAlertmanagerRouteName = "alertmanager-main" + UserWorkloadRouteName = "prometheus-user-workload" + UserWorkloadAlertmanagerRouteName = "alertmanager-user-workload" + PrometheusAlertsPath = "/v1/alerts" + PrometheusRulesPath = "/v1/rules" + AlertmanagerAlertsPath = "/api/v2/alerts" + UserWorkloadAlertmanagerPort = 9095 + UserWorkloadPrometheusServiceName = "prometheus-user-workload-web" + UserWorkloadPrometheusPort = 9090 + + ThanosQuerierServiceName = "thanos-querier" + DefaultThanosQuerierTenancyRulesPort = 9093 + ThanosQuerierTenancyAlertsPath = "/api/v1/alerts" + ThanosQuerierTenancyRulesPath = "/api/v1/rules" + ServiceCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" + + AlertSourceLabel = "openshift_io_alert_source" + AlertSourcePlatform = "platform" + AlertSourceUser = "user" + AlertBackendLabel = "openshift_io_alert_backend" + AlertBackendAM = "alertmanager" + AlertBackendProm = "prometheus" + AlertBackendThanos = "thanos" ) diff --git a/pkg/k8s/enrich_active_at_test.go b/pkg/k8s/enrich_active_at_test.go new file mode 100644 index 000000000..95e205591 --- /dev/null +++ b/pkg/k8s/enrich_active_at_test.go @@ -0,0 +1,106 @@ +package k8s + +import ( + "testing" + "time" +) + +func TestEnrichActiveAt_ReplacesAlertmanagerTimestamp(t *testing.T) { + amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + promTime := time.Date(2026, 3, 9, 8, 0, 0, 0, time.UTC) + + amAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU", "severity": "critical", AlertSourceLabel: "platform", AlertBackendLabel: "am"}, + ActiveAt: amTime, + }} + promAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU", "severity": "critical", AlertSourceLabel: "platform", AlertBackendLabel: "prom"}, + ActiveAt: promTime, + }} + + enrichActiveAt(amAlerts, promAlerts) + + if !amAlerts[0].ActiveAt.Equal(promTime) { + t.Errorf("expected ActiveAt=%v, got %v", promTime, amAlerts[0].ActiveAt) + } +} + +func TestEnrichActiveAt_NoMatchKeepsOriginal(t *testing.T) { + amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + + amAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU", "severity": "critical"}, + ActiveAt: amTime, + }} + promAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "DiskFull", "severity": "warning"}, + ActiveAt: time.Date(2026, 3, 9, 8, 0, 0, 0, time.UTC), + }} + + enrichActiveAt(amAlerts, promAlerts) + + if !amAlerts[0].ActiveAt.Equal(amTime) { + t.Errorf("expected ActiveAt to stay %v, got %v", amTime, amAlerts[0].ActiveAt) + } +} + +func TestEnrichActiveAt_EmptyPromAlerts(t *testing.T) { + amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + + amAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU"}, + ActiveAt: amTime, + }} + + enrichActiveAt(amAlerts, nil) + + if !amAlerts[0].ActiveAt.Equal(amTime) { + t.Errorf("expected ActiveAt to stay %v, got %v", amTime, amAlerts[0].ActiveAt) + } +} + +func TestEnrichActiveAt_SkipsZeroPromActiveAt(t *testing.T) { + amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + + amAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU"}, + ActiveAt: amTime, + }} + promAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU"}, + }} + + enrichActiveAt(amAlerts, promAlerts) + + if !amAlerts[0].ActiveAt.Equal(amTime) { + t.Errorf("expected ActiveAt to stay %v when prom has zero time, got %v", amTime, amAlerts[0].ActiveAt) + } +} + +func TestAlertFingerprint_IgnoresMetadataLabels(t *testing.T) { + fp1 := alertFingerprint(map[string]string{ + "alertname": "HighCPU", + "severity": "critical", + AlertSourceLabel: "platform", + AlertBackendLabel: "am", + }) + fp2 := alertFingerprint(map[string]string{ + "alertname": "HighCPU", + "severity": "critical", + AlertSourceLabel: "platform", + AlertBackendLabel: "prom", + }) + + if fp1 != fp2 { + t.Errorf("fingerprints should match when only metadata labels differ:\n fp1=%q\n fp2=%q", fp1, fp2) + } +} + +func TestAlertFingerprint_DifferentLabelsProduceDifferentKeys(t *testing.T) { + fp1 := alertFingerprint(map[string]string{"alertname": "HighCPU", "severity": "critical"}) + fp2 := alertFingerprint(map[string]string{"alertname": "HighCPU", "severity": "warning"}) + + if fp1 == fp2 { + t.Error("fingerprints should differ when label values differ") + } +} diff --git a/pkg/k8s/prometheus_alerts.go b/pkg/k8s/prometheus_alerts.go new file mode 100644 index 000000000..3077799c9 --- /dev/null +++ b/pkg/k8s/prometheus_alerts.go @@ -0,0 +1,1015 @@ +package k8s + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "sort" + "strings" + "sync" + "time" + + routev1 "github.com/openshift/api/route/v1" + routeclient "github.com/openshift/client-go/route/clientset/versioned" + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" +) + +var prometheusLog = logrus.WithField("module", "k8s-prometheus") + +const ( + namespaceCacheTTL = 30 * time.Second + serviceHealthTimeout = 5 * time.Second + serviceRequestTimeout = 10 * time.Second + maxTenancyProbeTargets = 3 +) + +type namespaceCache struct { + mu sync.Mutex + expiresAt time.Time + ttl time.Duration + value []string +} + +func newNamespaceCache(ttl time.Duration) *namespaceCache { + return &namespaceCache{ttl: ttl} +} + +func (c *namespaceCache) get() ([]string, bool) { + if c == nil { + return nil, false + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.expiresAt.IsZero() || time.Now().After(c.expiresAt) { + return nil, false + } + return copyStringSlice(c.value), true +} + +func (c *namespaceCache) set(namespaces []string) { + if c == nil { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + c.value = copyStringSlice(namespaces) + c.expiresAt = time.Now().Add(c.ttl) +} + +type prometheusAlerts struct { + routeClient routeclient.Interface + coreClient corev1client.CoreV1Interface + config *rest.Config + ruleManager PrometheusRuleInterface + nsCache *namespaceCache +} + +// GetAlertsRequest holds parameters for filtering alerts +type GetAlertsRequest struct { + // Labels filters alerts by labels + Labels map[string]string + // State filters alerts by state: "firing", "pending", "silenced", or "" for all states + State string +} + +type PrometheusAlert struct { + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` + State string `json:"state"` + ActiveAt time.Time `json:"activeAt"` + Value string `json:"value"` + + AlertRuleId string `json:"alertRuleId,omitempty"` + AlertComponent string `json:"alertComponent,omitempty"` + AlertLayer string `json:"alertLayer,omitempty"` +} + +type prometheusAlertsData struct { + Alerts []PrometheusAlert `json:"alerts"` +} + +type prometheusAlertsResponse struct { + Status string `json:"status"` + Data prometheusAlertsData `json:"data"` +} + +type prometheusRulesData struct { + Groups []PrometheusRuleGroup `json:"groups"` +} + +type prometheusRulesResponse struct { + Status string `json:"status"` + Data prometheusRulesData `json:"data"` +} + +type alertmanagerAlertStatus struct { + State string `json:"state"` +} + +type alertmanagerAlert struct { + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` + StartsAt time.Time `json:"startsAt"` + EndsAt time.Time `json:"endsAt"` + GeneratorURL string `json:"generatorURL"` + Status alertmanagerAlertStatus `json:"status"` +} + +func newPrometheusAlerts(routeClient routeclient.Interface, coreClient corev1client.CoreV1Interface, config *rest.Config, ruleManager PrometheusRuleInterface) *prometheusAlerts { + return &prometheusAlerts{ + routeClient: routeClient, + coreClient: coreClient, + config: config, + ruleManager: ruleManager, + nsCache: newNamespaceCache(namespaceCacheTTL), + } +} + +func (pa *prometheusAlerts) GetAlerts(ctx context.Context, req GetAlertsRequest) ([]PrometheusAlert, error) { + platformAlerts, err := pa.getAlertsForSource(ctx, ClusterMonitoringNamespace, PlatformRouteName, PlatformAlertmanagerRouteName, AlertSourcePlatform) + if err != nil { + // Namespace-scoped callers (Thanos tenancy) often lack platform + // Prometheus access. Soft-fail so tenancy results are still returned. + if namespaceFromLabels(req.Labels) == "" { + return nil, err + } + prometheusLog.Warnf("failed to get platform alerts (continuing with namespace filter): %v", err) + } + + userAlerts, err := pa.getUserWorkloadAlerts(ctx, req) + if err != nil { + prometheusLog.Warnf("failed to get user workload alerts: %v", err) + } + + mergedAlerts := append(platformAlerts, userAlerts...) + + out := make([]PrometheusAlert, 0, len(mergedAlerts)) + for _, a := range mergedAlerts { + // Filter alerts based on state if provided + if !matchesAlertState(req.State, a.State) { + continue + } + + // Filter alerts based on labels if provided + if !labelsMatch(&req, &a) { + continue + } + + out = append(out, a) + } + return out, nil +} + +func matchesAlertState(requestedState string, alertState string) bool { + if requestedState == "" { + return true + } + if requestedState == "firing" { + return alertState == "firing" || alertState == "silenced" + } + return alertState == requestedState +} + +func (pa *prometheusAlerts) GetRules(ctx context.Context, req GetRulesRequest) ([]PrometheusRuleGroup, error) { + platformRules, err := pa.getRulesViaProxy(ctx, ClusterMonitoringNamespace, PlatformRouteName, AlertSourcePlatform) + if err != nil { + // Namespace-scoped callers (Thanos tenancy) often lack platform + // Prometheus access. Soft-fail so tenancy results are still returned. + if namespaceFromLabels(req.Labels) == "" { + return nil, err + } + prometheusLog.Warnf("failed to get platform rules (continuing with namespace filter): %v", err) + } + + userRules, err := pa.getUserWorkloadRules(ctx, req) + if err != nil { + prometheusLog.Warnf("failed to get user workload rules: %v", err) + } + + groups := append(platformRules, userRules...) + + matchers, err := compileRuleLabelMatchers(req) + if err != nil { + return nil, err + } + if len(matchers) == 0 { + return groups, nil + } + + return filterRuleGroupsByLabelMatchers(groups, matchers), nil +} + +func (pa *prometheusAlerts) alertingHealth(ctx context.Context) AlertingHealth { + userPrometheus := pa.routeHealth(ctx, UserWorkloadMonitoringNamespace, UserWorkloadRouteName, PrometheusRulesPath) + if userPrometheus.Status != RouteReachable { + if ok := pa.thanosTenancyReachable(ctx, ThanosQuerierTenancyAlertsPath); ok { + userPrometheus.FallbackReachable = true + } + } + + userAlertmanager := pa.routeHealth(ctx, UserWorkloadMonitoringNamespace, UserWorkloadAlertmanagerRouteName, AlertmanagerAlertsPath) + if userAlertmanager.Status != RouteReachable { + if ok := pa.serviceReachable(ctx, UserWorkloadMonitoringNamespace, UserWorkloadAlertmanagerRouteName, UserWorkloadAlertmanagerPort, AlertmanagerAlertsPath); ok { + userAlertmanager.FallbackReachable = true + } + } + + platformStack := pa.stackHealth(ctx, ClusterMonitoringNamespace, PlatformRouteName, PlatformAlertmanagerRouteName) + userWorkloadStack := AlertingStackHealth{ + Prometheus: userPrometheus, + Alertmanager: userAlertmanager, + } + + return AlertingHealth{ + Platform: &platformStack, + UserWorkload: &userWorkloadStack, + } +} + +func (pa *prometheusAlerts) stackHealth(ctx context.Context, namespace string, promRouteName string, amRouteName string) AlertingStackHealth { + return AlertingStackHealth{ + Prometheus: pa.routeHealth(ctx, namespace, promRouteName, PrometheusRulesPath), + Alertmanager: pa.routeHealth(ctx, namespace, amRouteName, AlertmanagerAlertsPath), + } +} + +func (pa *prometheusAlerts) routeHealth(ctx context.Context, namespace string, routeName string, path string) AlertingRouteHealth { + health := AlertingRouteHealth{ + Name: routeName, + Namespace: namespace, + } + + if pa.routeClient == nil { + health.Error = "route client is not configured" + return health + } + + route, err := pa.routeClient.RouteV1().Routes(namespace).Get(ctx, routeName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + health.Status = RouteNotFound + health.Error = err.Error() + return health + } + health.Error = err.Error() + return health + } + + url := buildRouteURL(route.Spec.Host, route.Spec.Path, path) + client, err := pa.createHTTPClient() + if err != nil { + health.Status = RouteUnreachable + health.Error = err.Error() + return health + } + + if _, err := pa.executeRequest(ctx, client, url); err != nil { + health.Status = RouteUnreachable + health.Error = err.Error() + return health + } + + health.Status = RouteReachable + return health +} + +// getAlertsForSource fetches alerts from both Alertmanager and Prometheus in +// parallel and merges the results. The fallback strategy is: +// - Both succeed: AM (firing+silenced) + Prom pending, with AM timestamps +// enriched from Prometheus activeAt. +// - AM only: AM alerts returned as-is (no Prom data to enrich from). +// - Prom only: all Prom alerts returned (AM was unreachable). +// - Both fail: error propagated from Prometheus. +func (pa *prometheusAlerts) getAlertsForSource(ctx context.Context, namespace string, promRouteName string, amRouteName string, source string) ([]PrometheusAlert, error) { + amAlerts, amErr := pa.getAlertmanagerAlerts(ctx, namespace, amRouteName, source) + promAlerts, promErr := pa.getAlertsViaProxy(ctx, namespace, promRouteName, source) + + if amErr == nil { + if promErr == nil { + enrichActiveAt(amAlerts, promAlerts) + } + pending := filterAlertsByState(promAlerts, "pending") + return append(amAlerts, pending...), nil + } + + if promErr != nil { + return nil, promErr + } + + return promAlerts, nil +} + +func (pa *prometheusAlerts) getUserWorkloadAlerts(ctx context.Context, req GetAlertsRequest) ([]PrometheusAlert, error) { + if shouldPreferUserAlertmanager(req.State) { + alerts, err := pa.getUserWorkloadAlertsViaAlertmanager(ctx) + if err == nil { + return alerts, nil + } + prometheusLog.Warnf("failed to get user workload alerts via alertmanager: %v", err) + } + + namespace := namespaceFromLabels(req.Labels) + if namespace != "" { + alerts, err := pa.getAlertsViaThanosTenancy(ctx, namespace, AlertSourceUser) + if err == nil { + return alerts, nil + } + prometheusLog.Warnf("failed to get user workload alerts via thanos tenancy: %v", err) + } + + userNamespaces := pa.userRuleNamespaces(ctx) + if len(userNamespaces) > 0 { + alerts, err := pa.getAlertsViaThanosTenancyNamespaces(ctx, userNamespaces, AlertSourceUser) + if err == nil { + return alerts, nil + } + prometheusLog.Warnf("failed to get user workload alerts via thanos tenancy namespaces: %v", err) + } + + return pa.getAlertsForSource(ctx, UserWorkloadMonitoringNamespace, UserWorkloadRouteName, UserWorkloadAlertmanagerRouteName, AlertSourceUser) +} + +func shouldPreferUserAlertmanager(state string) bool { + return state == "firing" || state == "silenced" +} + +func (pa *prometheusAlerts) getUserWorkloadAlertsViaAlertmanager(ctx context.Context) ([]PrometheusAlert, error) { + alerts, err := pa.getAlertmanagerAlerts(ctx, UserWorkloadMonitoringNamespace, UserWorkloadAlertmanagerRouteName, AlertSourceUser) + if err != nil { + alerts, err = pa.getAlertmanagerAlertsViaService(ctx, UserWorkloadMonitoringNamespace, UserWorkloadAlertmanagerRouteName, UserWorkloadAlertmanagerPort, AlertSourceUser) + if err != nil { + return nil, err + } + } + + promAlerts, err := pa.getAlertsViaProxy(ctx, UserWorkloadMonitoringNamespace, UserWorkloadRouteName, AlertSourceUser) + if err != nil { + promAlerts, err = pa.getPrometheusAlertsViaService(ctx, UserWorkloadMonitoringNamespace, UserWorkloadPrometheusServiceName, UserWorkloadPrometheusPort, AlertSourceUser) + if err != nil { + return alerts, nil + } + } + + // Enrich before filtering: AM alerts need activeAt from all Prom states. + enrichActiveAt(alerts, promAlerts) + return append(alerts, filterAlertsByState(promAlerts, "pending")...), nil +} + +func (pa *prometheusAlerts) getPrometheusAlertsViaService(ctx context.Context, namespace string, serviceName string, port int32, source string) ([]PrometheusAlert, error) { + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + timeoutCtx, cancel := context.WithTimeout(ctx, serviceRequestTimeout) + defer cancel() + ctx = timeoutCtx + } + + raw, err := pa.getServiceResponse(ctx, namespace, serviceName, port, PrometheusAlertsPath) + if err != nil { + return nil, err + } + + var alertsResp prometheusAlertsResponse + if err := json.Unmarshal(raw, &alertsResp); err != nil { + return nil, fmt.Errorf("decode prometheus response: %w", err) + } + + if alertsResp.Status != "success" { + return nil, fmt.Errorf("prometheus API returned non-success status: %s", alertsResp.Status) + } + + applyAlertMetadata(alertsResp.Data.Alerts, source, AlertBackendProm) + return alertsResp.Data.Alerts, nil +} + +func (pa *prometheusAlerts) getAlertmanagerAlertsViaService(ctx context.Context, namespace string, serviceName string, port int32, source string) ([]PrometheusAlert, error) { + raw, err := pa.getServiceResponse(ctx, namespace, serviceName, port, AlertmanagerAlertsPath) + if err != nil { + return nil, err + } + + converted, err := parseAlertmanagerResponse(raw) + if err != nil { + return nil, err + } + + applyAlertMetadata(converted, source, AlertBackendAM) + if len(converted) == 0 { + return []PrometheusAlert{}, nil + } + return converted, nil +} + +// parseAlertmanagerResponse unmarshals a raw Alertmanager GET /api/v2/alerts +// response and converts it to PrometheusAlert structs. No routing labels are +// added — callers that need them should call applyAlertMetadata. +func parseAlertmanagerResponse(raw []byte) ([]PrometheusAlert, error) { + var amAlerts []alertmanagerAlert + if err := json.Unmarshal(raw, &amAlerts); err != nil { + return nil, fmt.Errorf("decode alertmanager response: %w", err) + } + + converted := make([]PrometheusAlert, 0, len(amAlerts)) + for _, alert := range amAlerts { + state := mapAlertmanagerState(alert.Status.State) + if state == "" { + continue + } + converted = append(converted, PrometheusAlert{ + Labels: alert.Labels, + Annotations: alert.Annotations, + State: state, + ActiveAt: alert.StartsAt, + }) + } + return converted, nil +} + +func (pa *prometheusAlerts) serviceReachable(ctx context.Context, namespace string, serviceName string, port int32, path string) bool { + healthCtx, cancel := context.WithTimeout(ctx, serviceHealthTimeout) + defer cancel() + + _, err := pa.getServiceResponse(healthCtx, namespace, serviceName, port, path) + return err == nil +} + +func (pa *prometheusAlerts) getServiceResponse(ctx context.Context, namespace string, serviceName string, port int32, path string) ([]byte, error) { + baseURL := fmt.Sprintf("https://%s.%s.svc:%d", serviceName, namespace, port) + requestURL := fmt.Sprintf("%s%s", baseURL, path) + + client, err := pa.createHTTPClient() + if err != nil { + return nil, err + } + + return pa.executeRequest(ctx, client, requestURL) +} + +func (pa *prometheusAlerts) thanosTenancyReachable(ctx context.Context, path string) bool { + namespaces := pa.userRuleNamespaces(ctx) + if len(namespaces) == 0 { + return false + } + + limit := maxTenancyProbeTargets + if limit <= 0 || limit > len(namespaces) { + limit = len(namespaces) + } + + for i := 0; i < limit; i++ { + healthCtx, cancel := context.WithTimeout(ctx, serviceHealthTimeout) + _, err := pa.getThanosTenancyResponse(healthCtx, path, namespaces[i]) + cancel() + + if err == nil { + return true + } + if isTenancyExpectedError(err) { + continue + } + return false + } + + return false +} + +// isTenancyExpectedError returns true for errors that are expected when probing +// Thanos tenancy endpoints across user namespaces — e.g. the namespace has no +// rules (404), the SA lacks access (401/403), or the namespace is not yet +// instrumented. These are skipped; only a network/server error aborts the probe. +func isTenancyExpectedError(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "status 401") || + strings.Contains(msg, "status 403") || + strings.Contains(msg, "status 404") || + strings.Contains(msg, "unauthorized") || + strings.Contains(msg, "forbidden") || + strings.Contains(msg, "not found") +} + +func (pa *prometheusAlerts) getAlertsViaProxy(ctx context.Context, namespace string, routeName string, source string) ([]PrometheusAlert, error) { + raw, err := pa.getPrometheusResponse(ctx, namespace, routeName, PrometheusAlertsPath) + if err != nil { + return nil, err + } + + var alertsResp prometheusAlertsResponse + if err := json.Unmarshal(raw, &alertsResp); err != nil { + return nil, fmt.Errorf("decode prometheus response: %w", err) + } + + if alertsResp.Status != "success" { + return nil, fmt.Errorf("prometheus API returned non-success status: %s", alertsResp.Status) + } + + applyAlertMetadata(alertsResp.Data.Alerts, source, AlertBackendProm) + return alertsResp.Data.Alerts, nil +} + +func (pa *prometheusAlerts) getAlertsViaThanosTenancy(ctx context.Context, namespace string, source string) ([]PrometheusAlert, error) { + raw, err := pa.getThanosTenancyResponse(ctx, ThanosQuerierTenancyAlertsPath, namespace) + if err != nil { + return nil, err + } + + var alertsResp prometheusAlertsResponse + if err := json.Unmarshal(raw, &alertsResp); err != nil { + return nil, fmt.Errorf("decode thanos response: %w", err) + } + + if alertsResp.Status != "success" { + return nil, fmt.Errorf("thanos API returned non-success status: %s", alertsResp.Status) + } + + applyAlertMetadata(alertsResp.Data.Alerts, source, AlertBackendThanos) + return alertsResp.Data.Alerts, nil +} + +func (pa *prometheusAlerts) getAlertmanagerAlerts(ctx context.Context, namespace string, routeName string, source string) ([]PrometheusAlert, error) { + raw, err := pa.getPrometheusResponse(ctx, namespace, routeName, AlertmanagerAlertsPath) + if err != nil { + return nil, err + } + + converted, err := parseAlertmanagerResponse(raw) + if err != nil { + return nil, err + } + + applyAlertMetadata(converted, source, AlertBackendAM) + if len(converted) == 0 { + return []PrometheusAlert{}, nil + } + return converted, nil +} + +func (pa *prometheusAlerts) getUserWorkloadRules(ctx context.Context, req GetRulesRequest) ([]PrometheusRuleGroup, error) { + namespace := namespaceFromLabels(req.Labels) + if namespace != "" { + rules, err := pa.getRulesViaThanosTenancy(ctx, namespace, AlertSourceUser) + if err == nil { + return rules, nil + } + prometheusLog.Warnf("failed to get user workload rules via thanos tenancy: %v", err) + } + + userNamespaces := pa.userRuleNamespaces(ctx) + if len(userNamespaces) > 0 { + groups, err := pa.getRulesViaThanosTenancyNamespaces(ctx, userNamespaces, AlertSourceUser) + if err == nil { + return groups, nil + } + prometheusLog.Warnf("failed to get user workload rules via thanos tenancy namespaces: %v", err) + } + + return pa.getRulesViaProxy(ctx, UserWorkloadMonitoringNamespace, UserWorkloadRouteName, AlertSourceUser) +} + +func (pa *prometheusAlerts) userRuleNamespaces(ctx context.Context) []string { + if cached, ok := pa.nsCache.get(); ok { + return cached + } + + if pa.ruleManager == nil { + namespaces := pa.allNonPlatformNamespaces(ctx) + pa.nsCache.set(namespaces) + return namespaces + } + + prometheusRules, err := pa.ruleManager.List() + if err != nil { + prometheusLog.WithError(err).Warn("failed to list PrometheusRules for user namespace discovery") + namespaces := pa.allNonPlatformNamespaces(ctx) + pa.nsCache.set(namespaces) + return namespaces + } + + namespaces := map[string]struct{}{} + for _, pr := range prometheusRules { + if pr.Namespace == "" { + continue + } + if pr.Namespace == ClusterMonitoringNamespace || pr.Namespace == UserWorkloadMonitoringNamespace { + continue + } + namespaces[pr.Namespace] = struct{}{} + } + + out := make([]string, 0, len(namespaces)) + for ns := range namespaces { + out = append(out, ns) + } + pa.nsCache.set(out) + return out +} + +func (pa *prometheusAlerts) allNonPlatformNamespaces(ctx context.Context) []string { + if pa.coreClient == nil { + return nil + } + + namespaceList, err := pa.coreClient.Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + prometheusLog.WithError(err).Warn("failed to list namespaces for user namespace discovery") + return nil + } + + out := make([]string, 0, len(namespaceList.Items)) + for _, ns := range namespaceList.Items { + if ns.Name == ClusterMonitoringNamespace || ns.Name == UserWorkloadMonitoringNamespace { + continue + } + out = append(out, ns.Name) + } + return out +} + +// fanOutThanosTenancy calls fetch for each namespace, accumulates results, and +// returns combined results (or the last error if nothing succeeded). +func fanOutThanosTenancy[T any](namespaces []string, fetch func(string) ([]T, error)) ([]T, error) { + var out []T + var lastErr error + for _, namespace := range namespaces { + results, err := fetch(namespace) + if err != nil { + lastErr = err + continue + } + out = append(out, results...) + } + if len(out) > 0 { + return out, nil + } + return out, lastErr +} + +func (pa *prometheusAlerts) getAlertsViaThanosTenancyNamespaces(ctx context.Context, namespaces []string, source string) ([]PrometheusAlert, error) { + return fanOutThanosTenancy(namespaces, func(ns string) ([]PrometheusAlert, error) { + return pa.getAlertsViaThanosTenancy(ctx, ns, source) + }) +} + +func (pa *prometheusAlerts) getRulesViaThanosTenancyNamespaces(ctx context.Context, namespaces []string, source string) ([]PrometheusRuleGroup, error) { + return fanOutThanosTenancy(namespaces, func(ns string) ([]PrometheusRuleGroup, error) { + return pa.getRulesViaThanosTenancy(ctx, ns, source) + }) +} + +func (pa *prometheusAlerts) getRulesViaProxy(ctx context.Context, namespace string, routeName string, source string) ([]PrometheusRuleGroup, error) { + raw, err := pa.getPrometheusResponse(ctx, namespace, routeName, PrometheusRulesPath) + if err != nil { + return nil, err + } + + var rulesResp prometheusRulesResponse + if err := json.Unmarshal(raw, &rulesResp); err != nil { + return nil, fmt.Errorf("decode prometheus response: %w", err) + } + + if rulesResp.Status != "success" { + return nil, fmt.Errorf("prometheus API returned non-success status: %s", rulesResp.Status) + } + + applyRuleSource(rulesResp.Data.Groups, source) + return rulesResp.Data.Groups, nil +} + +func (pa *prometheusAlerts) getRulesViaThanosTenancy(ctx context.Context, namespace string, source string) ([]PrometheusRuleGroup, error) { + raw, err := pa.getThanosTenancyResponse(ctx, ThanosQuerierTenancyRulesPath, namespace) + if err != nil { + return nil, err + } + + var rulesResp prometheusRulesResponse + if err := json.Unmarshal(raw, &rulesResp); err != nil { + return nil, fmt.Errorf("decode thanos response: %w", err) + } + + if rulesResp.Status != "success" { + return nil, fmt.Errorf("thanos API returned non-success status: %s", rulesResp.Status) + } + + applyRuleSource(rulesResp.Data.Groups, source) + return rulesResp.Data.Groups, nil +} + +func (pa *prometheusAlerts) getPrometheusResponse(ctx context.Context, namespace string, routeName string, path string) ([]byte, error) { + url, err := pa.buildPrometheusURL(ctx, namespace, routeName, path) + if err != nil { + return nil, err + } + client, err := pa.createHTTPClient() + if err != nil { + return nil, err + } + + return pa.executeRequest(ctx, client, url) +} + +func (pa *prometheusAlerts) getThanosTenancyResponse(ctx context.Context, path string, namespace string) ([]byte, error) { + if namespace == "" { + return nil, fmt.Errorf("namespace is required for thanos tenancy requests") + } + + baseURL := fmt.Sprintf("https://%s.%s.svc:%d", ThanosQuerierServiceName, ClusterMonitoringNamespace, DefaultThanosQuerierTenancyRulesPort) + requestURL := fmt.Sprintf("%s%s?namespace=%s", baseURL, path, url.QueryEscape(namespace)) + + client, err := pa.createHTTPClient() + if err != nil { + return nil, err + } + + return pa.executeRequest(ctx, client, requestURL) +} + +func (pa *prometheusAlerts) buildPrometheusURL(ctx context.Context, namespace string, routeName string, path string) (string, error) { + route, err := pa.fetchPrometheusRoute(ctx, namespace, routeName) + if err != nil { + return "", err + } + + return buildRouteURL(route.Spec.Host, route.Spec.Path, path), nil +} + +func (pa *prometheusAlerts) fetchPrometheusRoute(ctx context.Context, namespace string, routeName string) (*routev1.Route, error) { + if pa.routeClient == nil { + return nil, fmt.Errorf("route client is not configured") + } + + route, err := pa.routeClient.RouteV1().Routes(namespace).Get(ctx, routeName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get prometheus route: %w", err) + } + + return route, nil +} + +func applyAlertMetadata(alerts []PrometheusAlert, source, backend string) { + for i := range alerts { + if alerts[i].Labels == nil { + alerts[i].Labels = map[string]string{} + } + alerts[i].Labels[AlertSourceLabel] = source + alerts[i].Labels[AlertBackendLabel] = backend + } +} + +func applyRuleSource(groups []PrometheusRuleGroup, source string) { + for gi := range groups { + for ri := range groups[gi].Rules { + rule := &groups[gi].Rules[ri] + if rule.Labels == nil { + rule.Labels = map[string]string{} + } + rule.Labels[AlertSourceLabel] = source + for ai := range rule.Alerts { + if rule.Alerts[ai].Labels == nil { + rule.Alerts[ai].Labels = map[string]string{} + } + rule.Alerts[ai].Labels[AlertSourceLabel] = source + } + } + } +} + +func filterAlertsByState(alerts []PrometheusAlert, state string) []PrometheusAlert { + out := make([]PrometheusAlert, 0, len(alerts)) + for _, alert := range alerts { + if alert.State == state { + out = append(out, alert) + } + } + return out +} + +// enrichActiveAt replaces ActiveAt in Alertmanager-sourced alerts with the +// authoritative value from Prometheus. Alertmanager only exposes startsAt +// (when it received the alert), while Prometheus tracks the true activeAt +// (when the alert condition first became true). +func enrichActiveAt(amAlerts, promAlerts []PrometheusAlert) { + if len(promAlerts) == 0 { + return + } + + lookup := make(map[string]time.Time, len(promAlerts)) + for _, alert := range promAlerts { + fp := alertFingerprint(alert.Labels) + if !alert.ActiveAt.IsZero() { + lookup[fp] = alert.ActiveAt + } + } + + for i := range amAlerts { + fp := alertFingerprint(amAlerts[i].Labels) + if activeAt, ok := lookup[fp]; ok { + amAlerts[i].ActiveAt = activeAt + } + } +} + +// alertFingerprint builds a stable identity key from an alert's labels, +// excluding metadata labels injected by this plugin (source, backend). +// This matches the same alert *instance* across Alertmanager and Prometheus +// (which may differ only in injected metadata). It is distinct from the +// alert rule ID (GetAlertingRuleId) which identifies the *rule definition* +// and is computed from the rule spec (name, expr, duration, static labels). +func alertFingerprint(labels map[string]string) string { + keys := make([]string, 0, len(labels)) + for k := range labels { + if k == AlertSourceLabel || k == AlertBackendLabel { + continue + } + keys = append(keys, k) + } + sort.Strings(keys) + + var b strings.Builder + for i, k := range keys { + if i > 0 { + b.WriteByte('\xff') + } + b.WriteString(k) + b.WriteByte('\xfe') + b.WriteString(labels[k]) + } + return b.String() +} + +func mapAlertmanagerState(state string) string { + if state == "active" { + return "firing" + } + if state == "suppressed" { + return "silenced" + } + return "" +} + +func buildRouteURL(host string, routePath string, requestPath string) string { + basePath := strings.TrimSuffix(routePath, "/") + if basePath == "" { + return fmt.Sprintf("https://%s%s", host, requestPath) + } + if requestPath == basePath || strings.HasPrefix(requestPath, basePath+"/") { + return fmt.Sprintf("https://%s%s", host, requestPath) + } + return fmt.Sprintf("https://%s%s%s", host, basePath, requestPath) +} + +func namespaceFromLabels(labels map[string]string) string { + if labels == nil { + return "" + } + return strings.TrimSpace(labels["namespace"]) +} + +func (pa *prometheusAlerts) createHTTPClient() (*http.Client, error) { + tlsConfig, err := pa.buildTLSConfig() + if err != nil { + return nil, err + } + + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + }, nil +} + +func (pa *prometheusAlerts) buildTLSConfig() (*tls.Config, error) { + caCertPool, err := pa.loadCACertPool() + if err != nil { + return nil, err + } + + return &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: caCertPool, + }, nil +} + +func (pa *prometheusAlerts) loadCACertPool() (*x509.CertPool, error) { + caCertPool, err := x509.SystemCertPool() + if err != nil { + caCertPool = x509.NewCertPool() + } + + if len(pa.config.CAData) > 0 { + caCertPool.AppendCertsFromPEM(pa.config.CAData) + return caCertPool, nil + } + + if pa.config.CAFile != "" { + caCert, err := os.ReadFile(pa.config.CAFile) + if err != nil { + return nil, fmt.Errorf("read CA cert file: %w", err) + } + caCertPool.AppendCertsFromPEM(caCert) + } + + // OpenShift service CA bundle for in-cluster service certs. + if serviceCA, err := os.ReadFile(ServiceCAPath); err == nil { + caCertPool.AppendCertsFromPEM(serviceCA) + } + + return caCertPool, nil +} + +func copyStringSlice(in []string) []string { + if len(in) == 0 { + return []string{} + } + + out := make([]string, len(in)) + copy(out, in) + return out +} + +func (pa *prometheusAlerts) executeRequest(ctx context.Context, client *http.Client, url string) ([]byte, error) { + req, err := pa.createAuthenticatedRequest(ctx, url) + if err != nil { + return nil, err + } + + return pa.performRequest(client, req) +} + +func (pa *prometheusAlerts) createAuthenticatedRequest(ctx context.Context, url string) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + token := BearerTokenFromContext(ctx) + if token == "" { + var err error + token, err = pa.loadBearerToken() + if err != nil { + return nil, err + } + } + + req.Header.Set("Authorization", "Bearer "+token) + return req, nil +} + +func (pa *prometheusAlerts) loadBearerToken() (string, error) { + if pa.config.BearerToken != "" { + return pa.config.BearerToken, nil + } + + if pa.config.BearerTokenFile == "" { + return "", fmt.Errorf("no bearer token or token file configured") + } + + tokenBytes, err := os.ReadFile(pa.config.BearerTokenFile) + if err != nil { + return "", fmt.Errorf("load bearer token file: %w", err) + } + + return strings.TrimSpace(string(tokenBytes)), nil +} + +func (pa *prometheusAlerts) performRequest(client *http.Client, req *http.Request) ([]byte, error) { + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("execute request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) + } + + return body, nil +} + +func labelsMatch(req *GetAlertsRequest, alert *PrometheusAlert) bool { + for key, value := range req.Labels { + if alertValue, exists := alert.Labels[key]; !exists || alertValue != value { + return false + } + } + + return true +} diff --git a/pkg/k8s/prometheus_rules_types.go b/pkg/k8s/prometheus_rules_types.go index 3f5c289fb..c41ea4e89 100644 --- a/pkg/k8s/prometheus_rules_types.go +++ b/pkg/k8s/prometheus_rules_types.go @@ -5,6 +5,11 @@ import ( "time" ) +const ( + RuleTypeAlerting = "alerting" + RuleTypeRecording = "recording" +) + // GetRulesRequest holds parameters for filtering rules alerts. type GetRulesRequest struct { // Labels filters rules by exact label equality. The special key "namespace" diff --git a/pkg/k8s/relabeled_rules.go b/pkg/k8s/relabeled_rules.go index 02452c385..9ecec5325 100644 --- a/pkg/k8s/relabeled_rules.go +++ b/pkg/k8s/relabeled_rules.go @@ -9,7 +9,6 @@ import ( "sync" "time" - osmv1 "github.com/openshift/api/monitoring/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" monitoringv1client "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" "github.com/prometheus/common/model" @@ -149,7 +148,7 @@ func newRelabeledRulesManager(ctx context.Context, namespaceManager NamespaceInt return nil, fmt.Errorf("failed to sync RelabeledRulesConfig informer") } - if err := rrm.sync(ctx); err != nil { + if err := rrm.sync(ctx, "initial-sync"); err != nil { return nil, fmt.Errorf("initial relabeled rules sync failed: %w", err) } @@ -180,7 +179,7 @@ func (rrm *relabeledRulesManager) processNextWorkItem(ctx context.Context) bool defer rrm.queue.Done(key) - if err := rrm.sync(ctx); err != nil { + if err := rrm.sync(ctx, key); err != nil { log.Errorf("error syncing relabeled rules: %v", err) rrm.queue.AddRateLimited(key) return true @@ -191,7 +190,7 @@ func (rrm *relabeledRulesManager) processNextWorkItem(ctx context.Context) bool return true } -func (rrm *relabeledRulesManager) sync(ctx context.Context) error { +func (rrm *relabeledRulesManager) sync(ctx context.Context, key string) error { relabelConfigs, err := rrm.loadRelabelConfigs() if err != nil { return fmt.Errorf("failed to load relabel configs: %w", err) @@ -201,13 +200,20 @@ func (rrm *relabeledRulesManager) sync(ctx context.Context) error { rrm.relabelConfigs = relabelConfigs rrm.mu.Unlock() - alerts := rrm.collectAlerts(ctx, relabelConfigs) + alerts, allRuleIDs := rrm.collectAlerts(ctx, relabelConfigs) rrm.mu.Lock() rrm.relabeledRules = alerts rrm.mu.Unlock() log.Infof("Synced %d relabeled rules in memory", len(alerts)) + + // GC orphaned ARCs only when triggered by PrometheusRule events or + // initial sync — secret-only changes cannot create orphans. + if key == "prometheus-rule-sync" || key == "initial-sync" { + rrm.gcOrphanedARCs(ctx, allRuleIDs) + } + return nil } @@ -256,15 +262,10 @@ func (rrm *relabeledRulesManager) loadRelabelConfigs() ([]*relabel.Config, error return configs, nil } -func (rrm *relabeledRulesManager) collectAlerts(ctx context.Context, relabelConfigs []*relabel.Config) map[string]monitoringv1.Rule { +func (rrm *relabeledRulesManager) collectAlerts(ctx context.Context, relabelConfigs []*relabel.Config) (map[string]monitoringv1.Rule, map[string]struct{}) { alerts := make(map[string]monitoringv1.Rule) seenIDs := make(map[string]struct{}) - // Fetch all ARCs once from the informer cache (O(1) per-rule lookup below). - // This avoids O(n) live API server calls inside the per-rule loop that would - // cause exponential rate-limit backoff and stale cache data for new rules. - arcByName := rrm.arcsByName(ctx) - for _, obj := range rrm.prometheusRulesInformer.GetStore().List() { promRule, ok := obj.(*monitoringv1.PrometheusRule) if !ok { @@ -322,7 +323,7 @@ func (rrm *relabeledRulesManager) collectAlerts(ctx context.Context, relabelConf rule.Labels[managementlabels.AlertingRuleLabelName] = arName } - ruleManagedBy, relabelConfigManagedBy := rrm.determineManagedBy(promRule, alertRuleId, arcByName) + ruleManagedBy, relabelConfigManagedBy := rrm.determineManagedBy(ctx, promRule, alertRuleId) if ruleManagedBy != "" { rule.Labels[managementlabels.RuleManagedByLabel] = ruleManagedBy } @@ -336,7 +337,7 @@ func (rrm *relabeledRulesManager) collectAlerts(ctx context.Context, relabelConf } log.Debugf("Collected %d alerts", len(alerts)) - return alerts + return alerts, seenIDs } // alertingRuleOwner returns the name of the AlertingRule CR that generated @@ -390,28 +391,8 @@ func shortHash(id string, n int) string { return full[:n] } -// arcsByName builds a namespace/name → ARC map from the informer cache. -// Called once per sync cycle so that determineManagedBy can do O(1) lookups -// instead of one live API call per rule. -func (rrm *relabeledRulesManager) arcsByName(ctx context.Context) map[string]*osmv1.AlertRelabelConfig { - if rrm.alertRelabelConfigs == nil { - return nil - } - arcs, err := rrm.alertRelabelConfigs.List(ctx, "") - if err != nil { - log.Errorf("arcsByName: failed to list ARCs from cache: %v", err) - return nil - } - m := make(map[string]*osmv1.AlertRelabelConfig, len(arcs)) - for i := range arcs { - key := arcs[i].Namespace + "/" + arcs[i].Name - m[key] = &arcs[i] - } - return m -} - // determineManagedBy determines the openshift_io_rule_managed_by and openshift_io_relabel_config_managed_by label values -func (rrm *relabeledRulesManager) determineManagedBy(promRule *monitoringv1.PrometheusRule, alertRuleId string, arcByName map[string]*osmv1.AlertRelabelConfig) (string, string) { +func (rrm *relabeledRulesManager) determineManagedBy(ctx context.Context, promRule *monitoringv1.PrometheusRule, alertRuleId string) (string, string) { // Determine ruleManagedBy from PrometheusRule var ruleManagedBy string // If generated by AlertingRule CRD, do not mark as operator-managed; treat as user-via-platform @@ -425,14 +406,13 @@ func (rrm *relabeledRulesManager) determineManagedBy(promRule *monitoringv1.Prom } } - // Determine relabelConfigManagedBy only for platform rules using the - // pre-fetched cache map; no live API call is made here. + // Determine relabelConfigManagedBy only for platform rules isPlatform := rrm.namespaceManager.IsClusterMonitoringNamespace(promRule.Namespace) var relabelConfigManagedBy string - if isPlatform && arcByName != nil { + if isPlatform && rrm.alertRelabelConfigs != nil { arcName := GetAlertRelabelConfigName(promRule.Name, alertRuleId) - key := promRule.Namespace + "/" + arcName - if arc, found := arcByName[key]; found { + arc, found, err := rrm.alertRelabelConfigs.Get(ctx, promRule.Namespace, arcName) + if err == nil && found { if IsManagedByGitOps(arc.Annotations, arc.Labels) { relabelConfigManagedBy = managementlabels.ManagedByGitOps } @@ -442,27 +422,13 @@ func (rrm *relabeledRulesManager) determineManagedBy(promRule *monitoringv1.Prom return ruleManagedBy, relabelConfigManagedBy } -// DetermineManagedBy determines the managed-by labels for a single PrometheusRule -// alert rule. Callers that have a user-scoped context (e.g. tests) can pass a -// live AlertRelabelConfigInterface; a targeted Get is performed for that one rule. +// DetermineManagedBy determines the managed-by labels for a PrometheusRule alert rule. func DetermineManagedBy(ctx context.Context, alertRelabelConfigs AlertRelabelConfigInterface, namespaceManager NamespaceInterface, promRule *monitoringv1.PrometheusRule, alertRuleId string) (string, string) { - // Single-rule path: fetch only the specific ARC with RBAC enforcement on the - // caller's context, then build a one-entry map for determineManagedBy. - var arcByName map[string]*osmv1.AlertRelabelConfig - if alertRelabelConfigs != nil && namespaceManager.IsClusterMonitoringNamespace(promRule.Namespace) { - arcName := GetAlertRelabelConfigName(promRule.Name, alertRuleId) - arc, found, err := alertRelabelConfigs.Get(ctx, promRule.Namespace, arcName) - if err == nil && found { - arcByName = map[string]*osmv1.AlertRelabelConfig{ - promRule.Namespace + "/" + arcName: arc, - } - } - } rrm := &relabeledRulesManager{ alertRelabelConfigs: alertRelabelConfigs, namespaceManager: namespaceManager, } - return rrm.determineManagedBy(promRule, alertRuleId, arcByName) + return rrm.determineManagedBy(ctx, promRule, alertRuleId) } func (rrm *relabeledRulesManager) List(ctx context.Context) []monitoringv1.Rule { diff --git a/pkg/k8s/relabeled_rules_test.go b/pkg/k8s/relabeled_rules_test.go deleted file mode 100644 index 1d10ef48c..000000000 --- a/pkg/k8s/relabeled_rules_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package k8s - -import ( - "context" - "testing" - - osmv1 "github.com/openshift/api/monitoring/v1" - monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/openshift/monitoring-plugin/pkg/managementlabels" -) - -// arcGetPanicInterface implements AlertRelabelConfigInterface and panics if -// Get is called. It is used to verify that the sync path never calls Get. -type arcGetPanicInterface struct { - arcs []osmv1.AlertRelabelConfig -} - -func (m *arcGetPanicInterface) List(_ context.Context, namespace string) ([]osmv1.AlertRelabelConfig, error) { - if namespace == "" { - return m.arcs, nil - } - var filtered []osmv1.AlertRelabelConfig - for _, a := range m.arcs { - if a.Namespace == namespace { - filtered = append(filtered, a) - } - } - return filtered, nil -} - -func (m *arcGetPanicInterface) Get(_ context.Context, _, _ string) (*osmv1.AlertRelabelConfig, bool, error) { - panic("Get must not be called during sync; use the arcByName cache map instead") -} - -func (m *arcGetPanicInterface) Create(_ context.Context, arc osmv1.AlertRelabelConfig) (*osmv1.AlertRelabelConfig, error) { - return &arc, nil -} - -func (m *arcGetPanicInterface) Update(_ context.Context, _ osmv1.AlertRelabelConfig) error { - return nil -} - -func (m *arcGetPanicInterface) Delete(_ context.Context, _, _ string) error { - return nil -} - -// stubNamespaceManager implements NamespaceInterface for tests. -type stubNamespaceManager struct { - platformNamespaces map[string]bool -} - -func (s *stubNamespaceManager) IsClusterMonitoringNamespace(name string) bool { - return s.platformNamespaces[name] -} - -// TestDetermineManagedBy_NeverCallsGet verifies that determineManagedBy -// uses the pre-fetched arcByName map and never issues a live Get call, -// even for platform-namespace rules with a matching ARC. -func TestDetermineManagedBy_NeverCallsGet(t *testing.T) { - const ( - namespace = "openshift-monitoring" - promRuleName = "test-rule" - alertRuleID = "abc123" - ) - - arcName := GetAlertRelabelConfigName(promRuleName, alertRuleID) - arc := osmv1.AlertRelabelConfig{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: arcName, - Annotations: map[string]string{ - "argocd.argoproj.io/managed-by": "some-app", - }, - }, - } - - rrm := &relabeledRulesManager{ - // arcGetPanicInterface panics if Get is called — this is the guard. - alertRelabelConfigs: &arcGetPanicInterface{arcs: []osmv1.AlertRelabelConfig{arc}}, - namespaceManager: &stubNamespaceManager{ - platformNamespaces: map[string]bool{namespace: true}, - }, - } - - promRule := &monitoringv1.PrometheusRule{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: promRuleName, - }, - } - - // Build arcByName from List (no Get call). - arcByName := rrm.arcsByName(context.Background()) - - // This must not panic (i.e. must not call Get). - ruleManagedBy, relabelConfigManagedBy := rrm.determineManagedBy(promRule, alertRuleID, arcByName) - - if ruleManagedBy != "" { - t.Errorf("expected empty ruleManagedBy, got %q", ruleManagedBy) - } - if relabelConfigManagedBy != managementlabels.ManagedByGitOps { - t.Errorf("expected relabelConfigManagedBy=%q, got %q", managementlabels.ManagedByGitOps, relabelConfigManagedBy) - } -} - -// TestDetermineManagedBy_NoARCMatch verifies that a platform rule with no -// matching ARC in the cache produces empty relabelConfigManagedBy. -func TestDetermineManagedBy_NoARCMatch(t *testing.T) { - const namespace = "openshift-monitoring" - - rrm := &relabeledRulesManager{ - alertRelabelConfigs: &arcGetPanicInterface{arcs: nil}, - namespaceManager: &stubNamespaceManager{ - platformNamespaces: map[string]bool{namespace: true}, - }, - } - - promRule := &monitoringv1.PrometheusRule{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "some-rule", - }, - } - - arcByName := rrm.arcsByName(context.Background()) - _, relabelConfigManagedBy := rrm.determineManagedBy(promRule, "no-match-id", arcByName) - - if relabelConfigManagedBy != "" { - t.Errorf("expected empty relabelConfigManagedBy for no ARC match, got %q", relabelConfigManagedBy) - } -} - -// TestDetermineManagedBy_NonPlatformRuleSkipsARCLookup verifies that a -// user-workload rule (non-platform namespace) does not consult ARCs at all. -func TestDetermineManagedBy_NonPlatformRuleSkipsARCLookup(t *testing.T) { - rrm := &relabeledRulesManager{ - // Non-nil but panics on Get — confirms no lookup occurs. - alertRelabelConfigs: &arcGetPanicInterface{arcs: nil}, - namespaceManager: &stubNamespaceManager{platformNamespaces: map[string]bool{}}, - } - - promRule := &monitoringv1.PrometheusRule{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "user-namespace", - Name: "user-rule", - }, - } - - arcByName := rrm.arcsByName(context.Background()) - _, relabelConfigManagedBy := rrm.determineManagedBy(promRule, "some-id", arcByName) - - if relabelConfigManagedBy != "" { - t.Errorf("expected empty relabelConfigManagedBy for non-platform rule, got %q", relabelConfigManagedBy) - } -} diff --git a/pkg/k8s/rule_label_matchers.go b/pkg/k8s/rule_label_matchers.go new file mode 100644 index 000000000..cf8eb1f51 --- /dev/null +++ b/pkg/k8s/rule_label_matchers.go @@ -0,0 +1,91 @@ +package k8s + +import ( + "fmt" + "strings" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/promql/parser" +) + +const namespaceLabelKey = "namespace" + +func compileRuleLabelMatchers(req GetRulesRequest) ([]*labels.Matcher, error) { + var out []*labels.Matcher + + for k, v := range req.Labels { + if strings.TrimSpace(k) == "" { + continue + } + if k == namespaceLabelKey { + continue + } + m, err := labels.NewMatcher(labels.MatchEqual, k, v) + if err != nil { + return nil, fmt.Errorf("invalid label matcher %q=%q: %w", k, v, err) + } + out = append(out, m) + } + + for _, raw := range req.Matchers { + sel := strings.TrimSpace(raw) + if sel == "" { + continue + } + if !strings.HasPrefix(sel, "{") || !strings.HasSuffix(sel, "}") { + sel = "{" + sel + "}" + } + matchers, err := parser.ParseMetricSelector(sel) + if err != nil { + return nil, fmt.Errorf("invalid matcher %q: %w", raw, err) + } + out = append(out, matchers...) + } + + return out, nil +} + +func filterRuleGroupsByLabelMatchers(groups []PrometheusRuleGroup, matchers []*labels.Matcher) []PrometheusRuleGroup { + if len(matchers) == 0 || len(groups) == 0 { + return groups + } + + out := make([]PrometheusRuleGroup, 0, len(groups)) + for _, g := range groups { + kept := make([]PrometheusRule, 0, len(g.Rules)) + for _, r := range g.Rules { + if ruleMatchesLabelMatchers(r, matchers) { + kept = append(kept, r) + } + } + if len(kept) == 0 { + continue + } + g.Rules = kept + out = append(out, g) + } + + return out +} + +func ruleMatchesLabelMatchers(rule PrometheusRule, matchers []*labels.Matcher) bool { + if len(matchers) == 0 { + return true + } + + for _, m := range matchers { + val, ok := rule.Labels[m.Name] + if !ok { + // Prometheus semantics: negative matchers match missing labels. + if m.Type == labels.MatchNotEqual || m.Type == labels.MatchNotRegexp { + continue + } + return false + } + if !m.Matches(val) { + return false + } + } + + return true +} diff --git a/pkg/k8s/rule_label_matchers_test.go b/pkg/k8s/rule_label_matchers_test.go new file mode 100644 index 000000000..34169eaa7 --- /dev/null +++ b/pkg/k8s/rule_label_matchers_test.go @@ -0,0 +1,58 @@ +package k8s + +import "testing" + +func TestCompileRuleLabelMatchers_IgnoresNamespaceLabel(t *testing.T) { + matchers, err := compileRuleLabelMatchers(GetRulesRequest{ + Labels: map[string]string{ + "namespace": "ns-a", + "severity": "critical", + }, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if len(matchers) != 1 { + t.Fatalf("expected 1 matcher (severity), got %d", len(matchers)) + } + if matchers[0].Name != "severity" { + t.Fatalf("expected matcher for severity, got %q", matchers[0].Name) + } +} + +func TestRuleMatchesLabelMatchers_PrometheusMissingLabelSemantics(t *testing.T) { + neg, err := compileRuleLabelMatchers(GetRulesRequest{ + Matchers: []string{`missing!="x"`}, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if !ruleMatchesLabelMatchers(PrometheusRule{Labels: map[string]string{}}, neg) { + t.Fatalf("expected negative matcher to match missing label") + } + + pos, err := compileRuleLabelMatchers(GetRulesRequest{ + Matchers: []string{`missing="x"`}, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if ruleMatchesLabelMatchers(PrometheusRule{Labels: map[string]string{}}, pos) { + t.Fatalf("expected positive matcher not to match missing label") + } +} + +func TestCompileRuleLabelMatchers_AcceptsSelectorBody(t *testing.T) { + matchers, err := compileRuleLabelMatchers(GetRulesRequest{ + Matchers: []string{`severity=~"warning|critical"`}, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if len(matchers) != 1 { + t.Fatalf("expected 1 matcher, got %d", len(matchers)) + } + if matchers[0].Name != "severity" { + t.Fatalf("expected severity matcher, got %q", matchers[0].Name) + } +} diff --git a/pkg/k8s/types.go b/pkg/k8s/types.go index 102d5fccf..bf7b61b5b 100644 --- a/pkg/k8s/types.go +++ b/pkg/k8s/types.go @@ -21,6 +21,12 @@ type Client interface { // TestConnection tests the connection to the Kubernetes cluster TestConnection(ctx context.Context) error + // AlertingHealth returns alerting route and stack health details + AlertingHealth(ctx context.Context) (AlertingHealth, error) + + // PrometheusAlerts retrieves active Prometheus alerts + PrometheusAlerts() PrometheusAlertsInterface + // PrometheusRules returns the PrometheusRule interface PrometheusRules() PrometheusRuleInterface @@ -37,6 +43,14 @@ type Client interface { Namespace() NamespaceInterface } +// PrometheusAlertsInterface defines operations for managing PrometheusAlerts +type PrometheusAlertsInterface interface { + // GetAlerts retrieves Prometheus alerts with optional state filtering + GetAlerts(ctx context.Context, req GetAlertsRequest) ([]PrometheusAlert, error) + // GetRules retrieves Prometheus alerting rules and active alerts + GetRules(ctx context.Context, req GetRulesRequest) ([]PrometheusRuleGroup, error) +} + // PrometheusRuleInterface defines operations for managing PrometheusRules type PrometheusRuleInterface interface { // List lists all PrometheusRules from the informer cache @@ -104,6 +118,37 @@ type RelabeledRulesInterface interface { Config() []*relabel.Config } +// RouteStatus describes the availability state of a monitoring route. +type RouteStatus string + +const ( + RouteNotFound RouteStatus = "notFound" + RouteUnreachable RouteStatus = "unreachable" + RouteReachable RouteStatus = "reachable" +) + +// AlertingRouteHealth describes route availability and reachability. +type AlertingRouteHealth struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + Status RouteStatus `json:"status"` + FallbackReachable bool `json:"fallbackReachable,omitempty"` + Error string `json:"error,omitempty"` +} + +// AlertingStackHealth describes alerting health for a monitoring stack. +type AlertingStackHealth struct { + Prometheus AlertingRouteHealth `json:"prometheus"` + Alertmanager AlertingRouteHealth `json:"alertmanager"` +} + +// AlertingHealth provides alerting health details for platform and user workload stacks. +type AlertingHealth struct { + Platform *AlertingStackHealth `json:"platform"` + UserWorkloadEnabled bool `json:"userWorkloadEnabled"` + UserWorkload *AlertingStackHealth `json:"userWorkload"` +} + // NamespaceInterface defines operations for Namespaces type NamespaceInterface interface { // IsClusterMonitoringNamespace checks if a namespace has the openshift.io/cluster-monitoring=true label diff --git a/pkg/management/get_alerting_health.go b/pkg/management/get_alerting_health.go new file mode 100644 index 000000000..001d13f15 --- /dev/null +++ b/pkg/management/get_alerting_health.go @@ -0,0 +1,21 @@ +package management + +import ( + "context" + "time" + + "github.com/openshift/monitoring-plugin/pkg/k8s" +) + +const alertingHealthTimeout = 10 * time.Second + +// GetAlertingHealth retrieves alerting health details. +func (c *client) GetAlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) { + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + timeoutCtx, cancel := context.WithTimeout(ctx, alertingHealthTimeout) + defer cancel() + ctx = timeoutCtx + } + + return c.k8sClient.AlertingHealth(ctx) +} diff --git a/pkg/management/get_alerts.go b/pkg/management/get_alerts.go new file mode 100644 index 000000000..f7495227a --- /dev/null +++ b/pkg/management/get_alerts.go @@ -0,0 +1,308 @@ +package management + +import ( + "context" + "fmt" + "strings" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/relabel" + "k8s.io/apimachinery/pkg/types" + + alertrule "github.com/openshift/monitoring-plugin/pkg/alert_rule" + "github.com/openshift/monitoring-plugin/pkg/alertcomponent" + "github.com/openshift/monitoring-plugin/pkg/classification" + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +var cvoAlertNames = map[string]struct{}{ + "ClusterOperatorDown": {}, + "ClusterOperatorDegraded": {}, +} + +func (c *client) GetAlerts(ctx context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + alerts, err := c.k8sClient.PrometheusAlerts().GetAlerts(ctx, req) + if err != nil { + return nil, fmt.Errorf("failed to get prometheus alerts: %w", err) + } + + configs := c.k8sClient.RelabeledRules().Config() + rules := c.k8sClient.RelabeledRules().List(ctx) + + result := make([]k8s.PrometheusAlert, 0, len(alerts)) + for _, alert := range alerts { + // Only apply relabel configs for platform alerts. User workload alerts + // already come from their own stack and should not be relabeled here. + if alert.Labels[k8s.AlertSourceLabel] != k8s.AlertSourceUser { + relabels, keep := relabel.Process(labels.FromMap(alert.Labels), configs...) + if !keep { + continue + } + alert.Labels = relabels.Map() + } + + // Add calculated rule ID and source when not present (labels enrichment) + c.setRuleIDAndSourceIfMissing(ctx, &alert, rules) + + // correlate alert -> base alert rule via subset matching against relabeled rules + alertRuleId := alert.Labels[k8s.AlertRuleLabelId] + component := "" + layer := "" + + bestRule, corrId := correlateAlertToRule(alert.Labels, rules) + if corrId != "" { + alertRuleId = corrId + } + if bestRule == nil && alertRuleId != "" { + if rule, ok := c.k8sClient.RelabeledRules().Get(ctx, alertRuleId); ok { + bestRule = &rule + } + } + + if bestRule != nil { + if src := c.deriveAlertSource(bestRule.Labels); src != "" { + alert.Labels[k8s.AlertSourceLabel] = src + } + component, layer = classifyFromRule(bestRule) + } else { + component, layer = classifyFromAlertLabels(alert.Labels) + } + + if cvoComponent, cvoLayer, ok := classifyCvoAlert(alert.Labels); ok { + component = cvoComponent + layer = cvoLayer + } + + // Dynamic classification: _from labels on the rule point to alert labels + // whose runtime values become the classification. Takes precedence over + // static classification labels. + if bestRule != nil { + component, layer = ApplyDynamicClassification(bestRule.Labels, alert.Labels, component, layer) + } + + // keep label and optional enriched fields consistent + if alert.Labels[k8s.AlertRuleLabelId] == "" && alertRuleId != "" { + alert.Labels[k8s.AlertRuleLabelId] = alertRuleId + } + alert.AlertRuleId = alertRuleId + + alert.AlertComponent = component + alert.AlertLayer = layer + + delete(alert.Labels, managementlabels.ClassificationManagedByKey) + + result = append(result, alert) + } + + return result, nil +} + +func (c *client) setRuleIDAndSourceIfMissing(ctx context.Context, alert *k8s.PrometheusAlert, rules []monitoringv1.Rule) { + if alert.Labels[k8s.AlertRuleLabelId] == "" { + for _, existing := range rules { + if existing.Alert != alert.Labels[managementlabels.AlertNameLabel] { + continue + } + if !ruleMatchesAlert(existing.Labels, alert.Labels) { + continue + } + rid := alertrule.GetAlertingRuleId(&existing) + alert.Labels[k8s.AlertRuleLabelId] = rid + if alert.Labels[k8s.AlertSourceLabel] == "" { + if src := c.deriveAlertSource(existing.Labels); src != "" { + alert.Labels[k8s.AlertSourceLabel] = src + } + } + break + } + } + if alert.Labels[k8s.AlertSourceLabel] != "" { + return + } + if rid := alert.Labels[k8s.AlertRuleLabelId]; rid != "" { + if existing, ok := c.k8sClient.RelabeledRules().Get(ctx, rid); ok { + if src := c.deriveAlertSource(existing.Labels); src != "" { + alert.Labels[k8s.AlertSourceLabel] = src + } + } + } +} + +func ruleMatchesAlert(existingRuleLabels, alertLabels map[string]string) bool { + existingBusiness := filterBusinessLabels(existingRuleLabels) + for k, v := range existingBusiness { + lv, ok := alertLabels[k] + if !ok || lv != v { + return false + } + } + return true +} + +// correlateAlertToRule tries to find the base alert rule for the given alert labels +// by subset-matching against relabeled rules. +func correlateAlertToRule(alertLabels map[string]string, rules []monitoringv1.Rule) (*monitoringv1.Rule, string) { + // Determine best match: prefer rules with more labels (more specific) + var ( + bestId string + bestRule *monitoringv1.Rule + bestLabelCount int + ) + for i := range rules { + rule := &rules[i] + ruleLabels := sanitizeRuleLabels(rule.Labels) + if isSubset(ruleLabels, alertLabels) { + if len(ruleLabels) > bestLabelCount { + bestLabelCount = len(ruleLabels) + bestRule = rule + bestId = rule.Labels[k8s.AlertRuleLabelId] + } + } + } + if bestRule == nil { + return nil, "" + } + return bestRule, bestId +} + +// sanitizeRuleLabels removes meta labels that will not be present on alerts +func sanitizeRuleLabels(in map[string]string) map[string]string { + out := make(map[string]string, len(in)) + for k, v := range in { + if k == k8s.PrometheusRuleLabelNamespace || k == k8s.PrometheusRuleLabelName || k == k8s.AlertRuleLabelId { + continue + } + out[k] = v + } + return out +} + +// isSubset returns true if all key/value pairs in sub are present in sup +func isSubset(sub map[string]string, sup map[string]string) bool { + for k, v := range sub { + if sv, ok := sup[k]; !ok || sv != v { + return false + } + } + return true +} + +func (c *client) deriveAlertSource(ruleLabels map[string]string) string { + ns := ruleLabels[k8s.PrometheusRuleLabelNamespace] + name := ruleLabels[k8s.PrometheusRuleLabelName] + if ns == "" || name == "" { + return "" + } + if c.isPlatformManagedPrometheusRule(types.NamespacedName{Namespace: ns, Name: name}) { + return k8s.AlertSourcePlatform + } + return k8s.AlertSourceUser +} + +func classifyFromRule(rule *monitoringv1.Rule) (string, string) { + lbls := model.LabelSet{} + for k, v := range rule.Labels { + lbls[model.LabelName(k)] = model.LabelValue(v) + } + if _, ok := lbls["namespace"]; !ok { + if ns := rule.Labels[k8s.PrometheusRuleLabelNamespace]; ns != "" { + lbls["namespace"] = model.LabelValue(ns) + } + } + if rule.Alert != "" { + lbls[model.LabelName(managementlabels.AlertNameLabel)] = model.LabelValue(rule.Alert) + } + + layer, component := alertcomponent.DetermineComponent(lbls) + if component == "" || component == "Others" { + component = "other" + layer = deriveLayerFromSource(rule.Labels) + } + + component, layer = applyRuleScopedDefaults(rule.Labels, component, layer) + return component, layer +} + +func classifyFromAlertLabels(alertLabels map[string]string) (string, string) { + lbls := model.LabelSet{} + for k, v := range alertLabels { + lbls[model.LabelName(k)] = model.LabelValue(v) + } + layer, component := alertcomponent.DetermineComponent(lbls) + if component == "" || component == "Others" { + component = "other" + layer = deriveLayerFromSource(alertLabels) + } + component, layer = applyRuleScopedDefaults(alertLabels, component, layer) + return component, layer +} + +func deriveLayerFromSource(labels map[string]string) string { + if labels[k8s.AlertSourceLabel] == k8s.AlertSourcePlatform { + return "cluster" + } + if labels[k8s.PrometheusRuleLabelNamespace] == k8s.ClusterMonitoringNamespace { + return "cluster" + } + promSrc := labels["prometheus"] + if strings.HasPrefix(promSrc, "openshift-monitoring/") { + return "cluster" + } + return "namespace" +} + +// applyRuleScopedDefaults applies static classification labels from the rule. +func applyRuleScopedDefaults(ruleLabels map[string]string, component, layer string) (string, string) { + if ruleLabels == nil { + return component, layer + } + if v := strings.TrimSpace(ruleLabels[k8s.AlertRuleClassificationComponentKey]); v != "" { + if classification.ValidateComponent(v) { + component = v + } + } + if v := strings.TrimSpace(ruleLabels[k8s.AlertRuleClassificationLayerKey]); v != "" { + if classification.ValidateLayer(v) { + layer = strings.ToLower(strings.TrimSpace(v)) + } + } + return component, layer +} + +// applyDynamicClassification handles _from labels: the rule label points to an +// alert label whose runtime value becomes the classification. _from takes +// precedence over static classification labels. +func ApplyDynamicClassification(ruleLabels, alertLabels map[string]string, component, layer string) (string, string) { + if ruleLabels == nil { + return component, layer + } + if from := strings.TrimSpace(ruleLabels[k8s.AlertRuleClassificationComponentFromKey]); from != "" { + if classification.ValidatePromLabelName(from) { + if v := strings.TrimSpace(alertLabels[from]); v != "" && classification.ValidateComponent(v) { + component = v + } + } + } + if from := strings.TrimSpace(ruleLabels[k8s.AlertRuleClassificationLayerFromKey]); from != "" { + if classification.ValidatePromLabelName(from) { + if v := strings.ToLower(strings.TrimSpace(alertLabels[from])); classification.ValidateLayer(v) { + layer = v + } + } + } + return component, layer +} + +func classifyCvoAlert(alertLabels map[string]string) (string, string, bool) { + if _, ok := cvoAlertNames[alertLabels[managementlabels.AlertNameLabel]]; !ok { + return "", "", false + } + component := alertLabels["name"] + if component == "" { + component = "version" + } + return component, "cluster", true +} diff --git a/pkg/management/get_alerts_test.go b/pkg/management/get_alerts_test.go new file mode 100644 index 000000000..66b0e1902 --- /dev/null +++ b/pkg/management/get_alerts_test.go @@ -0,0 +1,465 @@ +package management_test + +import ( + "context" + "errors" + "strings" + "testing" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus/prometheus/model/relabel" + + alertrule "github.com/openshift/monitoring-plugin/pkg/alert_rule" + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management" + "github.com/openshift/monitoring-plugin/pkg/management/testutils" + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +func TestGetAlerts_ErrorPropagated(t *testing.T) { + ctx := context.Background() + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return nil, errors.New("failed to get alerts") + }, + } + }, + } + client := management.New(ctx, mockK8s) + + _, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "failed to get prometheus alerts") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestGetAlerts_ReturnsAllWithoutRelabelConfigs(t *testing.T) { + ctx := context.Background() + alert1 := k8s.PrometheusAlert{ + Labels: map[string]string{managementlabels.AlertNameLabel: "Alert1", "severity": "warning", "namespace": "default"}, + State: "firing", + } + alert2 := k8s.PrometheusAlert{ + Labels: map[string]string{managementlabels.AlertNameLabel: "Alert2", "severity": "critical", "namespace": "kube-system"}, + State: "pending", + } + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alert1, alert2}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 2 { + t.Fatalf("expected 2 alerts, got %d", len(alerts)) + } + if alerts[0].Labels[managementlabels.AlertNameLabel] != "Alert1" { + t.Errorf("alert[0] name mismatch") + } + if alerts[1].Labels[managementlabels.AlertNameLabel] != "Alert2" { + t.Errorf("alert[1] name mismatch") + } +} + +func TestGetAlerts_AppliesStaticClassificationFromRelabeledRule(t *testing.T) { + ctx := context.Background() + alert1 := k8s.PrometheusAlert{ + Labels: map[string]string{managementlabels.AlertNameLabel: "Alert1", "severity": "warning", "namespace": "default"}, + State: "firing", + } + + rule := monitoringv1.Rule{ + Alert: "Alert1", + Labels: map[string]string{ + "severity": "warning", + "namespace": "default", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "test-rule", + k8s.AlertRuleClassificationComponentKey: "networking", + k8s.AlertRuleClassificationLayerKey: "cluster", + }, + } + rule.Labels[k8s.AlertRuleLabelId] = alertrule.GetAlertingRuleId(&rule) + + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alert1}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{rule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == rule.Labels[k8s.AlertRuleLabelId] { + return rule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + NamespaceFunc: func() k8s.NamespaceInterface { + ns := &testutils.MockNamespaceInterface{} + ns.SetMonitoringNamespaces(map[string]bool{"openshift-monitoring": true}) + return ns + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].AlertComponent != "networking" { + t.Errorf("expected component=networking, got %q", alerts[0].AlertComponent) + } + if alerts[0].AlertLayer != "cluster" { + t.Errorf("expected layer=cluster, got %q", alerts[0].AlertLayer) + } +} + +func TestGetAlerts_DerivesComponentFromAlertLabel(t *testing.T) { + ctx := context.Background() + alertWithName := k8s.PrometheusAlert{ + Labels: map[string]string{ + managementlabels.AlertNameLabel: "Alert1", + "severity": "warning", + "namespace": "default", + "name": "kube_apiserver", + }, + State: "firing", + } + + rule := monitoringv1.Rule{ + Alert: "Alert1", + Labels: map[string]string{ + "severity": "warning", + "namespace": "default", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "test-rule", + k8s.AlertRuleClassificationComponentFromKey: "name", + k8s.AlertRuleClassificationLayerKey: "namespace", + }, + } + rule.Labels[k8s.AlertRuleLabelId] = alertrule.GetAlertingRuleId(&rule) + + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alertWithName}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{rule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == rule.Labels[k8s.AlertRuleLabelId] { + return rule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + NamespaceFunc: func() k8s.NamespaceInterface { + ns := &testutils.MockNamespaceInterface{} + ns.SetMonitoringNamespaces(map[string]bool{"openshift-monitoring": true}) + return ns + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].AlertComponent != "kube_apiserver" { + t.Errorf("expected component=kube_apiserver, got %q", alerts[0].AlertComponent) + } + if alerts[0].AlertLayer != "namespace" { + t.Errorf("expected layer=namespace, got %q", alerts[0].AlertLayer) + } +} + +func TestGetAlerts_DerivesLayerFromAlertLabel(t *testing.T) { + ctx := context.Background() + alertWithLayer := k8s.PrometheusAlert{ + Labels: map[string]string{ + managementlabels.AlertNameLabel: "Alert1", + "severity": "warning", + "namespace": "default", + "tier": "Cluster", + }, + State: "firing", + } + + rule := monitoringv1.Rule{ + Alert: "Alert1", + Labels: map[string]string{ + "severity": "warning", + "namespace": "default", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "test-rule", + k8s.AlertRuleClassificationComponentKey: "networking", + k8s.AlertRuleClassificationLayerFromKey: "tier", + }, + } + rule.Labels[k8s.AlertRuleLabelId] = alertrule.GetAlertingRuleId(&rule) + + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alertWithLayer}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{rule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == rule.Labels[k8s.AlertRuleLabelId] { + return rule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + NamespaceFunc: func() k8s.NamespaceInterface { + ns := &testutils.MockNamespaceInterface{} + ns.SetMonitoringNamespaces(map[string]bool{"openshift-monitoring": true}) + return ns + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].AlertComponent != "networking" { + t.Errorf("expected component=networking, got %q", alerts[0].AlertComponent) + } + // "Cluster" from alert label lowercased to "cluster" + if alerts[0].AlertLayer != "cluster" { + t.Errorf("expected layer=cluster, got %q", alerts[0].AlertLayer) + } +} + +func TestGetAlerts_UsesRuleLabelsAsDefaults(t *testing.T) { + ctx := context.Background() + alert := k8s.PrometheusAlert{ + Labels: map[string]string{ + "alertname": "AlertRuleDefaults", + "severity": "warning", + "namespace": "default", + k8s.AlertRuleClassificationComponentKey: "team_a", + k8s.AlertRuleClassificationLayerKey: "namespace", + }, + State: "firing", + } + + rule := monitoringv1.Rule{ + Alert: "AlertRuleDefaults", + Labels: map[string]string{ + "severity": "warning", + "namespace": "default", + k8s.AlertRuleClassificationComponentKey: "team_a", + k8s.AlertRuleClassificationLayerKey: "namespace", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "defaults-rule", + }, + } + rule.Labels[k8s.AlertRuleLabelId] = alertrule.GetAlertingRuleId(&rule) + + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alert}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{rule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == rule.Labels[k8s.AlertRuleLabelId] { + return rule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].AlertComponent != "team_a" { + t.Errorf("expected component=team_a, got %q", alerts[0].AlertComponent) + } + if alerts[0].AlertLayer != "namespace" { + t.Errorf("expected layer=namespace, got %q", alerts[0].AlertLayer) + } +} + +func TestGetAlerts_FallsBackToDefaultWhenNoMatchingRule(t *testing.T) { + ctx := context.Background() + alert1 := k8s.PrometheusAlert{ + Labels: map[string]string{managementlabels.AlertNameLabel: "Alert1", "severity": "warning", "namespace": "default"}, + State: "firing", + } + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alert1}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{} }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].AlertComponent != "other" { + t.Errorf("expected component=other, got %q", alerts[0].AlertComponent) + } + if alerts[0].AlertLayer != "namespace" { + t.Errorf("expected layer=namespace, got %q", alerts[0].AlertLayer) + } +} + +func TestGetAlerts_FallsBackToDefaultWithMatchingRuleNoLabels(t *testing.T) { + ctx := context.Background() + alert1 := k8s.PrometheusAlert{ + Labels: map[string]string{managementlabels.AlertNameLabel: "Alert1", "severity": "warning", "namespace": "default"}, + State: "firing", + } + + rule := monitoringv1.Rule{ + Alert: "Alert1", + Labels: map[string]string{ + "severity": "warning", + "namespace": "default", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "default-rule", + }, + } + rule.Labels[k8s.AlertRuleLabelId] = alertrule.GetAlertingRuleId(&rule) + + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{alert1}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return []monitoringv1.Rule{rule} }, + GetFunc: func(_ context.Context, id string) (monitoringv1.Rule, bool) { + if id == rule.Labels[k8s.AlertRuleLabelId] { + return rule, true + } + return monitoringv1.Rule{}, false + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].AlertComponent != "other" { + t.Errorf("expected component=other, got %q", alerts[0].AlertComponent) + } + if alerts[0].AlertLayer != "cluster" { + t.Errorf("expected layer=cluster, got %q", alerts[0].AlertLayer) + } +} + +func TestGetAlerts_ReturnsEmptyList(t *testing.T) { + ctx := context.Background() + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetAlertsFunc: func(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return []k8s.PrometheusAlert{}, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(ctx, mockK8s) + + alerts, err := client.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(alerts) != 0 { + t.Errorf("expected empty list, got %d", len(alerts)) + } +} diff --git a/pkg/management/get_rules.go b/pkg/management/get_rules.go new file mode 100644 index 000000000..f30822d35 --- /dev/null +++ b/pkg/management/get_rules.go @@ -0,0 +1,391 @@ +package management + +import ( + "context" + "fmt" + "math" + "sort" + "strings" + "time" + "unicode" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/relabel" + "github.com/prometheus/prometheus/promql/parser" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +func (c *client) GetRules(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + groups, err := c.k8sClient.PrometheusAlerts().GetRules(ctx, req) + if err != nil { + return nil, fmt.Errorf("failed to get prometheus rules: %w", err) + } + + configs := c.k8sClient.RelabeledRules().Config() + relabeledByAlert := indexRelabeledRules(c.k8sClient.RelabeledRules().List(ctx)) + applyFilters := req.State != "" || len(req.Labels) > 0 + + // Deduplicate rules that carry the same openshift_io_alert_rule_id across + // groups. This occurs when the same PrometheusRule group name is defined in + // multiple CRDs — Prometheus returns separate groups with identical rules + // that hash to the same ID after enrichment. + seenIDs := make(map[string]struct{}) + + filteredGroups := make([]k8s.PrometheusRuleGroup, 0, len(groups)) + for groupIdx := range groups { + group := groups[groupIdx] + filteredRules := make([]k8s.PrometheusRule, 0, len(group.Rules)) + + for ruleIdx := range group.Rules { + rule := group.Rules[ruleIdx] + if applyFilters && rule.Type != k8s.RuleTypeAlerting { + continue + } + applyRelabeledRuleLabels(&rule, relabeledByAlert) + + if ruleID := rule.Labels[k8s.AlertRuleLabelId]; ruleID != "" { + if _, seen := seenIDs[ruleID]; seen { + continue + } + seenIDs[ruleID] = struct{}{} + } + + if len(rule.Alerts) == 0 { + if applyFilters && rule.Type == k8s.RuleTypeAlerting { + continue + } + filteredRules = append(filteredRules, rule) + continue + } + + relabeledAlerts := make([]k8s.PrometheusRuleAlert, 0, len(rule.Alerts)) + for _, alert := range rule.Alerts { + if alert.State == "pending" || alert.State == "firing" { + if alert.Labels[k8s.AlertSourceLabel] != k8s.AlertSourceUser { + // Apply relabeling to the "real" alert labels only; preserve plugin meta labels. + src := alert.Labels[k8s.AlertSourceLabel] + in := make(map[string]string, len(alert.Labels)) + for k, v := range alert.Labels { + in[k] = v + } + delete(in, k8s.AlertSourceLabel) + + relabeledLabels, keep := relabel.Process(labels.FromMap(in), configs...) + if !keep { + continue + } + alert.Labels = relabeledLabels.Map() + if src != "" { + alert.Labels[k8s.AlertSourceLabel] = src + } + } + } + + if req.State != "" && alert.State != req.State { + continue + } + if !ruleAlertLabelsMatch(&req, &alert) { + continue + } + relabeledAlerts = append(relabeledAlerts, alert) + } + rule.Alerts = relabeledAlerts + + if applyFilters && rule.Type == k8s.RuleTypeAlerting && len(rule.Alerts) == 0 { + continue + } + + filteredRules = append(filteredRules, rule) + } + + group.Rules = filteredRules + if applyFilters && len(group.Rules) == 0 { + continue + } + filteredGroups = append(filteredGroups, group) + } + + return filteredGroups, nil +} + +func indexRelabeledRules(rules []monitoringv1.Rule) map[string][]monitoringv1.Rule { + byAlert := make(map[string][]monitoringv1.Rule, len(rules)) + for _, rule := range rules { + alertName := rule.Alert + if alertName == "" && rule.Labels != nil { + alertName = rule.Labels[managementlabels.AlertNameLabel] + } + if alertName == "" { + continue + } + byAlert[alertName] = append(byAlert[alertName], rule) + } + return byAlert +} + +func relabeledAlertName(rule *monitoringv1.Rule) string { + if rule == nil { + return "" + } + if rule.Alert != "" { + return rule.Alert + } + if rule.Labels != nil { + return rule.Labels[managementlabels.AlertNameLabel] + } + return "" +} + +func applyRelabeledRuleLabels(rule *k8s.PrometheusRule, relabeledByAlert map[string][]monitoringv1.Rule) { + if rule == nil || rule.Name == "" || rule.Type == k8s.RuleTypeRecording { + return + } + + // Preserve plugin meta labels added during API fetch. + source := "" + if rule.Labels != nil { + source = rule.Labels[k8s.AlertSourceLabel] + } + + match := findRelabeledMatch(rule, relabeledByAlert[rule.Name]) + if match == nil || match.Labels == nil { + return + } + + // Replace rule labels with the relabeled cache version so that actions which + // remove/rename labels (e.g. LabelDrop/LabelKeep/LabelMap) are faithfully reflected. + labelsOut := make(map[string]string, len(match.Labels)+1) + for k, v := range match.Labels { + labelsOut[k] = v + } + if source != "" { + labelsOut[k8s.AlertSourceLabel] = source + } + rule.Labels = labelsOut +} + +func findRelabeledMatch(rule *k8s.PrometheusRule, candidates []monitoringv1.Rule) *monitoringv1.Rule { + // Strict match first (preserves correctness when multiple rules share alertname). + for i := range candidates { + candidate := &candidates[i] + if promRuleMatchesRelabeled(rule, candidate) { + return candidate + } + } + + // If relabeling modified rule labels (e.g. severity), strict label matching may fail. + // Retry on a best-effort basis using (alertname, expr, for) only. If this is ambiguous, + // do not guess. + var relaxed *monitoringv1.Rule + for i := range candidates { + candidate := &candidates[i] + if rule == nil || candidate == nil { + continue + } + candidateName := relabeledAlertName(candidate) + if rule.Name == "" || candidateName == "" || rule.Name != candidateName { + continue + } + if canonicalizePromQL(rule.Query) != canonicalizePromQL(candidate.Expr.String()) { + continue + } + if !durationMatches(rule.Duration, candidate.For) { + continue + } + if relaxed != nil { + // ambiguous + relaxed = nil + break + } + relaxed = candidate + } + if relaxed != nil { + return relaxed + } + + // Fallback: if alertname is globally unique, avoid brittle PromQL/metadata matching. + // This helps when Prometheus stringifies PromQL differently than PrometheusRule YAML + // (e.g. label matcher ordering). + if len(candidates) == 1 { + return &candidates[0] + } + return nil +} + +func promRuleMatchesRelabeled(rule *k8s.PrometheusRule, candidate *monitoringv1.Rule) bool { + if rule == nil || candidate == nil { + return false + } + candidateName := relabeledAlertName(candidate) + if rule.Name == "" || candidateName == "" || rule.Name != candidateName { + return false + } + if canonicalizePromQL(rule.Query) != canonicalizePromQL(candidate.Expr.String()) { + return false + } + if !durationMatches(rule.Duration, candidate.For) { + return false + } + if !stringMapEqual(filterBusinessLabels(rule.Labels), filterBusinessLabels(candidate.Labels)) { + return false + } + return true +} + +func canonicalizePromQL(in string) string { + s := strings.TrimSpace(in) + if s == "" { + return "" + } + expr, err := parser.ParseExpr(s) + if err == nil && expr != nil { + parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error { + switch n := node.(type) { + case *parser.VectorSelector: + sort.Slice(n.LabelMatchers, func(i, j int) bool { + mi, mj := n.LabelMatchers[i], n.LabelMatchers[j] + if mi == nil || mj == nil { + return mi != nil + } + if mi.Name != mj.Name { + return mi.Name < mj.Name + } + if mi.Type != mj.Type { + return mi.Type < mj.Type + } + return mi.Value < mj.Value + }) + case *parser.AggregateExpr: + sort.Strings(n.Grouping) + case *parser.BinaryExpr: + if n.VectorMatching != nil { + sort.Strings(n.VectorMatching.MatchingLabels) + sort.Strings(n.VectorMatching.Include) + } + } + return nil + }) + + return expr.String() + } + return normalizeSpaceOutsideQuotes(s) +} + +func normalizeSpaceOutsideQuotes(in string) string { + if in == "" { + return "" + } + in = strings.TrimSpace(in) + + var b strings.Builder + b.Grow(len(in)) + + inQuote := false + escaped := false + pendingSpace := false + lastNoSpaceToken := false + + isNoSpaceToken := func(r rune) bool { + switch r { + case '(', ')', '{', '}', ',', '+', '-', '*', '/', '%', '^', '=', '!', '<', '>': + return true + default: + return false + } + } + + for _, r := range in { + if escaped { + if pendingSpace { + if !lastNoSpaceToken { + b.WriteByte(' ') + } + pendingSpace = false + } + b.WriteRune(r) + escaped = false + lastNoSpaceToken = false + continue + } + + if inQuote && r == '\\' { + if pendingSpace { + if !lastNoSpaceToken { + b.WriteByte(' ') + } + pendingSpace = false + } + b.WriteRune(r) + escaped = true + lastNoSpaceToken = false + continue + } + + if r == '"' { + if pendingSpace { + if !lastNoSpaceToken { + b.WriteByte(' ') + } + pendingSpace = false + } + inQuote = !inQuote + b.WriteRune(r) + lastNoSpaceToken = false + continue + } + + if !inQuote && unicode.IsSpace(r) { + pendingSpace = true + continue + } + + if pendingSpace && !lastNoSpaceToken && !isNoSpaceToken(r) { + b.WriteByte(' ') + } + pendingSpace = false + + b.WriteRune(r) + lastNoSpaceToken = !inQuote && isNoSpaceToken(r) + } + + return strings.TrimSpace(b.String()) +} + +func durationMatches(seconds float64, duration *monitoringv1.Duration) bool { + if duration == nil { + return seconds == 0 + } + parsed, err := time.ParseDuration(string(*duration)) + if err != nil { + return false + } + return math.Abs(parsed.Seconds()-seconds) < 0.001 +} + +func stringMapEqual(a, b map[string]string) bool { + if len(a) == 0 && len(b) == 0 { + return true + } + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +func ruleAlertLabelsMatch(req *k8s.GetRulesRequest, alert *k8s.PrometheusRuleAlert) bool { + for key, value := range req.Labels { + if alertValue, exists := alert.Labels[key]; !exists || alertValue != value { + return false + } + } + + return true +} diff --git a/pkg/management/get_rules_test.go b/pkg/management/get_rules_test.go new file mode 100644 index 000000000..56a5844fe --- /dev/null +++ b/pkg/management/get_rules_test.go @@ -0,0 +1,442 @@ +package management_test + +import ( + "context" + "testing" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/relabel" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management" + "github.com/openshift/monitoring-plugin/pkg/management/testutils" + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +// grFixture builds a management client with a PrometheusAlerts mock returning +// the given groups and a RelabeledRules mock returning the given configs/rules. +type grFixture struct { + groups []k8s.PrometheusRuleGroup + relabelRules []monitoringv1.Rule + relabelConfig []*relabel.Config +} + +func (f grFixture) client(t *testing.T) management.Client { + t.Helper() + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetRulesFunc: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return f.groups, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return f.relabelRules }, + ConfigFunc: func() []*relabel.Config { return f.relabelConfig }, + } + }, + } + return management.New(context.Background(), mockK8s) +} + +// threeAlertGroup returns a rule group containing one alerting rule with +// firing Alert1, pending Alert2, and inactive Alert3. +func threeAlertGroup() []k8s.PrometheusRuleGroup { + return []k8s.PrometheusRuleGroup{ + { + Name: "group-a", + Rules: []k8s.PrometheusRule{ + { + Name: "rule-a", + Type: k8s.RuleTypeAlerting, + Alerts: []k8s.PrometheusRuleAlert{ + {State: "firing", Labels: map[string]string{"alertname": "Alert1", "severity": "warning"}}, + {State: "pending", Labels: map[string]string{"alertname": "Alert2", "severity": "critical"}}, + {State: "inactive", Labels: map[string]string{"alertname": "Alert3", "severity": "warning"}}, + }, + }, + }, + }, + } +} + +func dropAlert2ReplaceAlert1Severity() []*relabel.Config { + return []*relabel.Config{ + { + SourceLabels: model.LabelNames{"alertname"}, + Regex: relabel.MustNewRegexp("Alert2"), + Action: relabel.Drop, + NameValidationScheme: model.UTF8Validation, + }, + { + SourceLabels: model.LabelNames{"alertname"}, + Regex: relabel.MustNewRegexp("Alert1"), + TargetLabel: "severity", + Replacement: "critical", + Action: relabel.Replace, + NameValidationScheme: model.UTF8Validation, + }, + } +} + +func TestGetRules_AppliesRelabelConfigsToPendingFiringOnly(t *testing.T) { + f := grFixture{ + groups: threeAlertGroup(), + relabelRules: []monitoringv1.Rule{}, + relabelConfig: dropAlert2ReplaceAlert1Severity(), + } + client := f.client(t) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 1 { + t.Fatalf("expected 1 group, got %d", len(groups)) + } + rules := groups[0].Rules + if len(rules) != 1 { + t.Fatalf("expected 1 rule, got %d", len(rules)) + } + alerts := rules[0].Alerts + if len(alerts) != 2 { + t.Fatalf("expected 2 alerts after drop, got %d", len(alerts)) + } + if alerts[0].Labels["alertname"] != "Alert1" || alerts[0].Labels["severity"] != "critical" { + t.Errorf("alert[0]: got alertname=%s severity=%s", alerts[0].Labels["alertname"], alerts[0].Labels["severity"]) + } + if alerts[1].Labels["alertname"] != "Alert3" || alerts[1].Labels["severity"] != "warning" { + t.Errorf("alert[1]: got alertname=%s severity=%s", alerts[1].Labels["alertname"], alerts[1].Labels["severity"]) + } +} + +func TestGetRules_FiltersByStateAndLabels(t *testing.T) { + f := grFixture{ + groups: threeAlertGroup(), + relabelRules: []monitoringv1.Rule{}, + relabelConfig: dropAlert2ReplaceAlert1Severity(), + } + client := f.client(t) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{ + State: "firing", + Labels: map[string]string{"severity": "critical"}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 1 { + t.Fatalf("expected 1 group, got %d", len(groups)) + } + alerts := groups[0].Rules[0].Alerts + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].Labels["alertname"] != "Alert1" || alerts[0].Labels["severity"] != "critical" { + t.Errorf("unexpected alert: %v", alerts[0].Labels) + } +} + +func TestGetRules_DropsNonMatchingRulesWhenFiltered(t *testing.T) { + f := grFixture{ + groups: threeAlertGroup(), + relabelRules: []monitoringv1.Rule{}, + relabelConfig: dropAlert2ReplaceAlert1Severity(), + } + client := f.client(t) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{ + State: "firing", + Labels: map[string]string{"severity": "does-not-exist"}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 0 { + t.Errorf("expected 0 groups, got %d", len(groups)) + } +} + +func TestGetRules_AddsManagedByLabelsFromRelabeledRules(t *testing.T) { + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetRulesFunc: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return []k8s.PrometheusRuleGroup{ + { + Name: "group-a", + Rules: []k8s.PrometheusRule{ + { + Name: "AlertWithManagedBy", + Type: "alerting", + Query: "up == 0", + Labels: map[string]string{"severity": "critical"}, + Annotations: map[string]string{"summary": "test alert"}, + }, + }, + }, + }, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { + return []monitoringv1.Rule{ + { + Alert: "AlertWithManagedBy", + Expr: intstr.FromString("up ==\n 0"), + Labels: map[string]string{ + "severity": "critical", + k8s.AlertRuleLabelId: "alert-id-1", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + managementlabels.RuleManagedByLabel: "operator", + managementlabels.RelabelConfigManagedByLabel: "gitops", + }, + Annotations: map[string]string{"summary": "test alert"}, + }, + } + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(context.Background(), mockK8s) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 1 || len(groups[0].Rules) != 1 { + t.Fatalf("expected 1 group with 1 rule") + } + rule := groups[0].Rules[0] + checks := map[string]string{ + k8s.AlertRuleLabelId: "alert-id-1", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + managementlabels.RuleManagedByLabel: "operator", + managementlabels.RelabelConfigManagedByLabel: "gitops", + } + for k, want := range checks { + if got := rule.Labels[k]; got != want { + t.Errorf("label[%s]: want %q, got %q", k, want, got) + } + } +} + +func TestGetRules_EnrichesWithAllLabelTypes(t *testing.T) { + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetRulesFunc: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return []k8s.PrometheusRuleGroup{ + { + Name: "group-a", + Rules: []k8s.PrometheusRule{ + { + Name: "ARCUpdatedRule", + Type: "alerting", + Query: "up == 0", + Labels: map[string]string{"severity": "warning", k8s.AlertSourceLabel: k8s.AlertSourcePlatform}, + }, + }, + }, + }, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { + return []monitoringv1.Rule{ + { + Alert: "ARCUpdatedRule", + Expr: intstr.FromString("up ==\n 0"), + Labels: map[string]string{ + "severity": "critical", + "team": "sre", + k8s.AlertRuleLabelId: "rid-arc-1", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + k8s.AlertRuleClassificationComponentKey: "compute", + k8s.AlertRuleClassificationLayerKey: "cluster", + managementlabels.RuleManagedByLabel: "operator", + managementlabels.RelabelConfigManagedByLabel: "gitops", + }, + }, + } + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(context.Background(), mockK8s) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 1 || len(groups[0].Rules) != 1 { + t.Fatalf("expected 1 group with 1 rule") + } + rule := groups[0].Rules[0] + checks := map[string]string{ + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.AlertRuleLabelId: "rid-arc-1", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + k8s.AlertRuleClassificationComponentKey: "compute", + k8s.AlertRuleClassificationLayerKey: "cluster", + "severity": "critical", + "team": "sre", + managementlabels.RuleManagedByLabel: "operator", + managementlabels.RelabelConfigManagedByLabel: "gitops", + } + for k, want := range checks { + if got := rule.Labels[k]; got != want { + t.Errorf("label[%s]: want %q, got %q", k, want, got) + } + } +} + +func TestGetRules_EnrichesWhenAlertFieldEmpty(t *testing.T) { + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetRulesFunc: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return []k8s.PrometheusRuleGroup{ + { + Name: "group-a", + Rules: []k8s.PrometheusRule{ + { + Name: "EmptyAlertFieldRule", + Type: "alerting", + Query: "up == 0", + Labels: map[string]string{"severity": "warning", k8s.AlertSourceLabel: k8s.AlertSourcePlatform}, + }, + }, + }, + }, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { + return []monitoringv1.Rule{ + { + Alert: "", + Expr: intstr.FromString("up ==\n 0"), + Labels: map[string]string{ + managementlabels.AlertNameLabel: "EmptyAlertFieldRule", + "severity": "critical", + k8s.AlertRuleLabelId: "rid-empty-alert-1", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + }, + }, + } + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(context.Background(), mockK8s) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 1 || len(groups[0].Rules) != 1 { + t.Fatalf("expected 1 group with 1 rule") + } + rule := groups[0].Rules[0] + checks := map[string]string{ + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.AlertRuleLabelId: "rid-empty-alert-1", + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + "severity": "critical", + } + for k, want := range checks { + if got := rule.Labels[k]; got != want { + t.Errorf("label[%s]: want %q, got %q", k, want, got) + } + } +} + +func TestGetRules_NoEnrichmentWhenMultipleCandidatesMatch(t *testing.T) { + mockK8s := &testutils.MockClient{ + PrometheusAlertsFunc: func() k8s.PrometheusAlertsInterface { + return &testutils.MockPrometheusAlertsInterface{ + GetRulesFunc: func(_ context.Context, _ k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + return []k8s.PrometheusRuleGroup{ + { + Name: "group-a", + Rules: []k8s.PrometheusRule{ + { + Name: "AmbiguousRule", + Type: "alerting", + Query: "up == 0", + Labels: map[string]string{"severity": "warning", k8s.AlertSourceLabel: k8s.AlertSourcePlatform}, + }, + }, + }, + }, nil + }, + } + }, + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { + return []monitoringv1.Rule{ + { + Alert: "", + Expr: intstr.FromString("up ==\n 0"), + Labels: map[string]string{ + managementlabels.AlertNameLabel: "AmbiguousRule", + "severity": "critical", + k8s.AlertRuleLabelId: "rid-amb-1", + }, + }, + { + Alert: "", + Expr: intstr.FromString("up==0"), + Labels: map[string]string{ + managementlabels.AlertNameLabel: "AmbiguousRule", + "severity": "critical", + k8s.AlertRuleLabelId: "rid-amb-2", + }, + }, + } + }, + ConfigFunc: func() []*relabel.Config { return []*relabel.Config{} }, + } + }, + } + client := management.New(context.Background(), mockK8s) + + groups, err := client.GetRules(context.Background(), k8s.GetRulesRequest{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(groups) != 1 || len(groups[0].Rules) != 1 { + t.Fatalf("expected 1 group with 1 rule") + } + rule := groups[0].Rules[0] + if rule.Labels[k8s.AlertSourceLabel] != k8s.AlertSourcePlatform { + t.Errorf("expected source=%s, got %s", k8s.AlertSourcePlatform, rule.Labels[k8s.AlertSourceLabel]) + } + if _, hasId := rule.Labels[k8s.AlertRuleLabelId]; hasId { + t.Errorf("expected no AlertRuleLabelId on ambiguous rule, but found: %s", rule.Labels[k8s.AlertRuleLabelId]) + } + if rule.Labels["severity"] != "warning" { + t.Errorf("expected severity=warning (from original), got %s", rule.Labels["severity"]) + } +} diff --git a/pkg/management/list_rules.go b/pkg/management/list_rules.go new file mode 100644 index 000000000..1b3d354eb --- /dev/null +++ b/pkg/management/list_rules.go @@ -0,0 +1,83 @@ +package management + +import ( + "context" + "sort" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + + "github.com/openshift/monitoring-plugin/pkg/k8s" +) + +func (c *client) ListRules(ctx context.Context, prOptions PrometheusRuleOptions, arOptions AlertRuleOptions, pgOptions PaginationOptions) (ListRulesResult, error) { + if prOptions.Name != "" && prOptions.Namespace == "" { + return ListRulesResult{}, &ValidationError{Message: "namespace is required when prometheusRuleName is specified"} + } + + allRules := c.k8sClient.RelabeledRules().List(ctx) + var filteredRules []monitoringv1.Rule + + for _, rule := range allRules { + if prOptions.Name != "" && prOptions.Namespace != "" { + namespace := rule.Labels[k8s.PrometheusRuleLabelNamespace] + name := rule.Labels[k8s.PrometheusRuleLabelName] + if namespace != prOptions.Namespace || name != prOptions.Name { + continue + } + } + + if !c.matchesAlertRuleFilters(rule, arOptions) { + continue + } + + filteredRules = append(filteredRules, rule) + } + + sort.Slice(filteredRules, func(i, j int) bool { + return filteredRules[i].Labels[k8s.AlertRuleLabelId] < filteredRules[j].Labels[k8s.AlertRuleLabelId] + }) + + if pgOptions.NextToken != "" { + idx := sort.Search(len(filteredRules), func(i int) bool { + return filteredRules[i].Labels[k8s.AlertRuleLabelId] > pgOptions.NextToken + }) + filteredRules = filteredRules[idx:] + } + + var nextToken string + if pgOptions.Limit > 0 && len(filteredRules) > pgOptions.Limit { + nextToken = filteredRules[pgOptions.Limit-1].Labels[k8s.AlertRuleLabelId] + filteredRules = filteredRules[:pgOptions.Limit] + } + + return ListRulesResult{Rules: filteredRules, NextToken: nextToken}, nil +} + +func (c *client) matchesAlertRuleFilters(rule monitoringv1.Rule, arOptions AlertRuleOptions) bool { + // Filter by alert name + if arOptions.Name != "" && string(rule.Alert) != arOptions.Name { + return false + } + + // Filter by source (platform) + if arOptions.Source == k8s.AlertSourcePlatform { + source, exists := rule.Labels[k8s.AlertSourceLabel] + if !exists { + return false + } + + return source == k8s.AlertSourcePlatform + } + + // Filter by labels + if len(arOptions.Labels) > 0 { + for key, value := range arOptions.Labels { + ruleValue, exists := rule.Labels[key] + if !exists || ruleValue != value { + return false + } + } + } + + return true +} diff --git a/pkg/management/list_rules_test.go b/pkg/management/list_rules_test.go new file mode 100644 index 000000000..d3564d382 --- /dev/null +++ b/pkg/management/list_rules_test.go @@ -0,0 +1,444 @@ +package management_test + +import ( + "context" + "errors" + "testing" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management" + "github.com/openshift/monitoring-plugin/pkg/management/testutils" +) + +var ( + lrRule1 = monitoringv1.Rule{ + Alert: "Alert1", + Expr: intstr.FromString("up == 0"), + Labels: map[string]string{ + "severity": "warning", + k8s.PrometheusRuleLabelNamespace: "namespace1", + k8s.PrometheusRuleLabelName: "rule1", + k8s.AlertRuleLabelId: "rid_aaa", + }, + } + + lrRule2 = monitoringv1.Rule{ + Alert: "Alert2", + Expr: intstr.FromString("up == 0"), + Labels: map[string]string{ + "severity": "critical", + k8s.PrometheusRuleLabelNamespace: "namespace1", + k8s.PrometheusRuleLabelName: "rule2", + k8s.AlertRuleLabelId: "rid_bbb", + }, + } + + lrRule3 = monitoringv1.Rule{ + Alert: "Alert3", + Expr: intstr.FromString("down == 1"), + Labels: map[string]string{ + "severity": "warning", + k8s.PrometheusRuleLabelNamespace: "namespace2", + k8s.PrometheusRuleLabelName: "rule3", + k8s.AlertRuleLabelId: "rid_ccc", + }, + } + + lrPlatformRule = monitoringv1.Rule{ + Alert: "PlatformAlert", + Expr: intstr.FromString("node_down == 1"), + Labels: map[string]string{ + "severity": "critical", + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.PrometheusRuleLabelNamespace: "openshift-monitoring", + k8s.PrometheusRuleLabelName: "platform-rule", + k8s.AlertRuleLabelId: "rid_ddd", + }, + } + + lrCustomLabelRule = monitoringv1.Rule{ + Alert: "CustomLabelAlert", + Expr: intstr.FromString("custom == 1"), + Labels: map[string]string{ + "severity": "info", + "team": "backend", + "env": "production", + k8s.PrometheusRuleLabelNamespace: "namespace1", + k8s.PrometheusRuleLabelName: "rule1", + k8s.AlertRuleLabelId: "rid_eee", + }, + } +) + +func newListRulesClient(t *testing.T, rules []monitoringv1.Rule) management.Client { + t.Helper() + mockK8s := &testutils.MockClient{ + RelabeledRulesFunc: func() k8s.RelabeledRulesInterface { + return &testutils.MockRelabeledRulesInterface{ + ListFunc: func(_ context.Context) []monitoringv1.Rule { return rules }, + } + }, + } + return management.New(context.Background(), mockK8s) +} + +var allLRRules = []monitoringv1.Rule{lrRule1, lrRule2, lrRule3, lrPlatformRule, lrCustomLabelRule} +var noPagination = management.PaginationOptions{} + +func TestListRules_MissingNamespaceReturnsValidationError(t *testing.T) { + client := newListRulesClient(t, allLRRules) + _, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{Name: "rule1"}, + management.AlertRuleOptions{}, + noPagination, + ) + if err == nil { + t.Fatal("expected error, got nil") + } + var ve *management.ValidationError + if !errors.As(err, &ve) { + t.Fatalf("expected ValidationError, got %T: %v", err, err) + } + if !containsSubstring(err.Error(), "namespace is required when prometheusRuleName is specified") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestListRules_NoFiltersReturnsAll(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 5 { + t.Errorf("expected 5 rules, got %d", len(result.Rules)) + } + if result.NextToken != "" { + t.Errorf("expected no next token, got %q", result.NextToken) + } +} + +func TestListRules_FilterByNameAndNamespace(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{Name: "rule1", Namespace: "namespace1"}, + management.AlertRuleOptions{}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 2 { + t.Fatalf("expected 2 rules, got %d", len(result.Rules)) + } + for _, r := range result.Rules { + if r.Alert != "Alert1" && r.Alert != "CustomLabelAlert" { + t.Errorf("unexpected rule: %s", r.Alert) + } + } +} + +func TestListRules_FilterByNameAndNamespace_NoMatch(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{Name: "nonexistent", Namespace: "namespace1"}, + management.AlertRuleOptions{}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 0 { + t.Errorf("expected 0 rules, got %d", len(result.Rules)) + } +} + +func TestListRules_FilterByAlertName(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{Name: "Alert1"}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 1 || result.Rules[0].Alert != "Alert1" { + t.Errorf("expected 1 rule Alert1, got %v", result.Rules) + } +} + +func TestListRules_FilterByAlertName_NoMatch(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{Name: "NonexistentAlert"}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 0 { + t.Errorf("expected 0 rules, got %d", len(result.Rules)) + } +} + +func TestListRules_FilterBySourcePlatform(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{Source: k8s.AlertSourcePlatform}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 1 { + t.Fatalf("expected 1 rule, got %d", len(result.Rules)) + } + if result.Rules[0].Alert != "PlatformAlert" { + t.Errorf("expected PlatformAlert, got %s", result.Rules[0].Alert) + } + if result.Rules[0].Labels[k8s.AlertSourceLabel] != k8s.AlertSourcePlatform { + t.Errorf("expected source=platform, got %s", result.Rules[0].Labels[k8s.AlertSourceLabel]) + } +} + +func TestListRules_FilterBySingleLabel(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{Labels: map[string]string{"severity": "warning"}}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 2 { + t.Errorf("expected 2 warning rules, got %d", len(result.Rules)) + } +} + +func TestListRules_FilterByMultipleLabels(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{Labels: map[string]string{"team": "backend", "env": "production"}}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 1 || result.Rules[0].Alert != "CustomLabelAlert" { + t.Errorf("expected 1 CustomLabelAlert, got %v", result.Rules) + } +} + +func TestListRules_FilterByLabels_NoMatch(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{Labels: map[string]string{"nonexistent": "value"}}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 0 { + t.Errorf("expected 0 rules, got %d", len(result.Rules)) + } +} + +func TestListRules_CombinedFilters(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{Name: "rule1", Namespace: "namespace1"}, + management.AlertRuleOptions{Labels: map[string]string{"severity": "warning"}}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 1 || result.Rules[0].Alert != "Alert1" { + t.Errorf("expected 1 Alert1, got %v", result.Rules) + } +} + +func TestListRules_CombinedFilters_NoMatch(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{Name: "rule1", Namespace: "namespace1"}, + management.AlertRuleOptions{Labels: map[string]string{"severity": "critical"}}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 0 { + t.Errorf("expected 0 rules, got %d", len(result.Rules)) + } +} + +func TestListRules_EmptyRelabeledRules(t *testing.T) { + client := newListRulesClient(t, []monitoringv1.Rule{}) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 0 { + t.Errorf("expected 0 rules, got %d", len(result.Rules)) + } +} + +func TestListRules_Pagination_FirstPage(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 2}, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 2 { + t.Fatalf("expected 2 rules, got %d", len(result.Rules)) + } + if result.NextToken == "" { + t.Error("expected next token, got empty") + } + if result.Rules[0].Labels[k8s.AlertRuleLabelId] != "rid_aaa" { + t.Errorf("expected rid_aaa, got %s", result.Rules[0].Labels[k8s.AlertRuleLabelId]) + } + if result.Rules[1].Labels[k8s.AlertRuleLabelId] != "rid_bbb" { + t.Errorf("expected rid_bbb, got %s", result.Rules[1].Labels[k8s.AlertRuleLabelId]) + } +} + +func TestListRules_Pagination_SecondPage(t *testing.T) { + client := newListRulesClient(t, allLRRules) + first, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 2}, + ) + if err != nil { + t.Fatalf("unexpected error on first page: %v", err) + } + + second, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 2, NextToken: first.NextToken}, + ) + if err != nil { + t.Fatalf("unexpected error on second page: %v", err) + } + if len(second.Rules) != 2 { + t.Fatalf("expected 2 rules on second page, got %d", len(second.Rules)) + } + if second.Rules[0].Labels[k8s.AlertRuleLabelId] != "rid_ccc" { + t.Errorf("expected rid_ccc, got %s", second.Rules[0].Labels[k8s.AlertRuleLabelId]) + } + if second.Rules[1].Labels[k8s.AlertRuleLabelId] != "rid_ddd" { + t.Errorf("expected rid_ddd, got %s", second.Rules[1].Labels[k8s.AlertRuleLabelId]) + } + if second.NextToken == "" { + t.Error("expected next token for third page") + } +} + +func TestListRules_Pagination_LastPageNoNextToken(t *testing.T) { + client := newListRulesClient(t, allLRRules) + first, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 2}, + ) + if err != nil { + t.Fatalf("page 1 error: %v", err) + } + second, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 2, NextToken: first.NextToken}, + ) + if err != nil { + t.Fatalf("page 2 error: %v", err) + } + third, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 2, NextToken: second.NextToken}, + ) + if err != nil { + t.Fatalf("page 3 error: %v", err) + } + if len(third.Rules) != 1 { + t.Errorf("expected 1 rule on last page, got %d", len(third.Rules)) + } + if third.NextToken != "" { + t.Errorf("expected no next token on last page, got %q", third.NextToken) + } +} + +func TestListRules_Pagination_LimitExceedsTotal(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + management.PaginationOptions{Limit: 100}, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Rules) != 5 { + t.Errorf("expected 5 rules, got %d", len(result.Rules)) + } + if result.NextToken != "" { + t.Errorf("expected no next token, got %q", result.NextToken) + } +} + +func TestListRules_Pagination_SortedByRuleId(t *testing.T) { + client := newListRulesClient(t, allLRRules) + result, err := client.ListRules(context.Background(), + management.PrometheusRuleOptions{}, + management.AlertRuleOptions{}, + noPagination, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for i := 1; i < len(result.Rules); i++ { + prev := result.Rules[i-1].Labels[k8s.AlertRuleLabelId] + curr := result.Rules[i].Labels[k8s.AlertRuleLabelId] + if prev >= curr { + t.Errorf("rules not sorted: %s >= %s at index %d", prev, curr, i) + } + } +} + +// containsSubstring is a local helper to avoid importing strings in test files +// that don't otherwise need it. +func containsSubstring(s, sub string) bool { + if len(sub) == 0 { + return true + } + for i := 0; i <= len(s)-len(sub); i++ { + if s[i:i+len(sub)] == sub { + return true + } + } + return false +} diff --git a/pkg/management/management.go b/pkg/management/management.go index 652ac14de..124a98785 100644 --- a/pkg/management/management.go +++ b/pkg/management/management.go @@ -1,9 +1,14 @@ package management import ( + "context" + "net/http" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management/metrics" ) type client struct { @@ -20,3 +25,7 @@ type client struct { func (c *client) isPlatformManagedPrometheusRule(nn types.NamespacedName) bool { return c.k8sClient.Namespace().IsClusterMonitoringNamespace(nn.Namespace) } + +func (c *client) MetricsHandler(ctx context.Context, kubeConfig *rest.Config) (http.Handler, error) { + return metrics.NewHandler(ctx, c, kubeConfig) +} diff --git a/pkg/management/management_suite_test.go b/pkg/management/management_suite_test.go new file mode 100644 index 000000000..d5fa1082b --- /dev/null +++ b/pkg/management/management_suite_test.go @@ -0,0 +1,15 @@ +package management_test + +import ( + "os" + "testing" + + "github.com/prometheus/common/model" +) + +func TestMain(m *testing.M) { + // LegacyValidation is required for tests that construct relabel configs + // containing label names with special characters (e.g. slashes). + model.NameValidationScheme = model.LegacyValidation //nolint:staticcheck + os.Exit(m.Run()) +} diff --git a/pkg/management/metrics/alerts_collector.go b/pkg/management/metrics/alerts_collector.go new file mode 100644 index 000000000..fd9dd9bed --- /dev/null +++ b/pkg/management/metrics/alerts_collector.go @@ -0,0 +1,223 @@ +package metrics + +import ( + "context" + "fmt" + "net/http" + "sort" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" + "k8s.io/client-go/rest" + + "github.com/openshift/monitoring-plugin/pkg/k8s" +) + +var metricsLog = logrus.WithField("module", "metrics") + +const ( + MetricName = "alerts_effective_active_at_timestamp_seconds" + metricHelp = "The activeAt timestamp of effective (post-ARC) alerts. " + + "Value is the Unix timestamp when the alert became active." + + DefaultSyncInterval = 30 * time.Second + + labelAlertState = "alertstate" +) + +// AlertsFetcher retrieves enriched alerts for the metric. The management.Client +// satisfies this interface — it applies ARC relabeling and computes +// classification (AlertComponent / AlertLayer) on every alert. +type AlertsFetcher interface { + GetAlerts(ctx context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) +} + +// alertMetric holds a single alert's pre-built metric data. +// The prometheus.Desc is created once during sync, not on every scrape. +type alertMetric struct { + desc *prometheus.Desc + labelValues []string + activeAtSec float64 +} + +// AlertsCollector implements prometheus.Collector. It periodically fetches +// alerts via the management client's GetAlerts (which applies ARC relabeling +// and computes classification) and exposes them as the +// alerts_effective_active_at_timestamp_seconds gauge. +// +// Only the leader pod (determined via Lease-based leader election) runs the +// sync loop and exposes metrics. Follower pods return nothing on Collect, +// ensuring each alert appears exactly once in Prometheus. +// +// Each alert produces one time series whose value is the alert's activeAt +// Unix timestamp. Labels are the alert's enriched labels (post-ARC, source, +// backend, component, layer) plus "alertstate". Thanos-sourced alerts are +// filtered out to avoid duplicates. Annotations are excluded because they +// are available from the alert rule definition. +type AlertsCollector struct { + fetcher AlertsFetcher + syncInterval time.Duration + isLeader func() bool + + mu sync.RWMutex + metrics []alertMetric + + sentinelDesc *prometheus.Desc +} + +// NewHandler creates a metrics HTTP handler that exposes the alerts effective +// metric. It sets up Lease-based leader election internally so that only one +// replica produces metrics, then wires the collector, registry and promhttp +// handler. Callers receive a ready-to-use http.Handler. +func NewHandler(ctx context.Context, fetcher AlertsFetcher, kubeConfig *rest.Config) (http.Handler, error) { + isLeader, err := startLeaderElection(ctx, kubeConfig, k8s.ClusterMonitoringNamespace) + if err != nil { + return nil, fmt.Errorf("start metrics leader election: %w", err) + } + + collector := NewAlertsCollector(ctx, fetcher, DefaultSyncInterval, isLeader) + registry := prometheus.NewRegistry() + registry.MustRegister(collector) + return promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), nil +} + +// NewAlertsCollector creates a collector that periodically syncs alerts and +// exposes them as Prometheus metrics. The isLeader callback controls whether +// this replica actively syncs and exposes metrics (follower pods return nothing). +func NewAlertsCollector(ctx context.Context, fetcher AlertsFetcher, syncInterval time.Duration, isLeader func() bool) *AlertsCollector { + c := &AlertsCollector{ + fetcher: fetcher, + syncInterval: syncInterval, + isLeader: isLeader, + sentinelDesc: prometheus.NewDesc(MetricName, metricHelp, nil, nil), + } + go c.syncLoop(ctx) + return c +} + +// Describe sends a sentinel descriptor to satisfy the Collector contract. +func (c *AlertsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.sentinelDesc +} + +// Collect emits the current set of alert metrics using pre-built Descs. +// Returns nothing if this replica is not the leader. +func (c *AlertsCollector) Collect(ch chan<- prometheus.Metric) { + if !c.isLeader() { + return + } + + c.mu.RLock() + defer c.mu.RUnlock() + + for i := range c.metrics { + m := &c.metrics[i] + metric, err := prometheus.NewConstMetric(m.desc, prometheus.GaugeValue, m.activeAtSec, m.labelValues...) + if err != nil { + metricsLog.WithError(err).Warn("failed to create metric") + continue + } + ch <- metric + } +} + +func (c *AlertsCollector) syncLoop(ctx context.Context) { + c.sync(ctx) + + ticker := time.NewTicker(c.syncInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.sync(ctx) + } + } +} + +func (c *AlertsCollector) sync(ctx context.Context) { + if !c.isLeader() { + return + } + + alerts, err := c.fetcher.GetAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + metricsLog.WithError(err).Warn("failed to fetch alerts for effective metric") + return + } + + built := make([]alertMetric, 0, len(alerts)) + for i := range alerts { + alert := &alerts[i] + + // Drop Thanos-sourced alerts: they duplicate what Alertmanager and + // Prometheus already provide and would inflate the metric cardinality. + if alert.Labels[k8s.AlertBackendLabel] == k8s.AlertBackendThanos { + continue + } + + enrichClassificationLabels(alert) + + m := buildAlertMetric(alert) + if m != nil { + built = append(built, *m) + } + } + + c.mu.Lock() + c.metrics = built + c.mu.Unlock() + + metricsLog.Debugf("synced %d alerts for effective metric", len(built)) +} + +// enrichClassificationLabels copies the management-computed AlertComponent and +// AlertLayer into the alert's Labels map so they appear on the metric. Labels +// already set (e.g. via ARC) take precedence. +func enrichClassificationLabels(alert *k8s.PrometheusAlert) { + if alert.AlertComponent != "" { + if _, exists := alert.Labels[k8s.AlertRuleClassificationComponentKey]; !exists { + alert.Labels[k8s.AlertRuleClassificationComponentKey] = alert.AlertComponent + } + } + if alert.AlertLayer != "" { + if _, exists := alert.Labels[k8s.AlertRuleClassificationLayerKey]; !exists { + alert.Labels[k8s.AlertRuleClassificationLayerKey] = alert.AlertLayer + } + } +} + +// buildAlertMetric converts a PrometheusAlert into an alertMetric with a +// pre-built prometheus.Desc. Uses the alert's labels plus the alertstate label. +func buildAlertMetric(alert *k8s.PrometheusAlert) *alertMetric { + if alert.ActiveAt.IsZero() { + return nil + } + + labelNames := make([]string, 0, len(alert.Labels)+1) + for k := range alert.Labels { + labelNames = append(labelNames, k) + } + sort.Strings(labelNames) + labelNames = append(labelNames, labelAlertState) + + labelValues := make([]string, 0, len(labelNames)) + for _, name := range labelNames { + if name == labelAlertState { + labelValues = append(labelValues, alert.State) + } else { + labelValues = append(labelValues, alert.Labels[name]) + } + } + + return &alertMetric{ + desc: prometheus.NewDesc(MetricName, metricHelp, labelNames, nil), + labelValues: labelValues, + activeAtSec: float64(alert.ActiveAt.Unix()), + } +} diff --git a/pkg/management/metrics/alerts_collector_test.go b/pkg/management/metrics/alerts_collector_test.go new file mode 100644 index 000000000..5ce053834 --- /dev/null +++ b/pkg/management/metrics/alerts_collector_test.go @@ -0,0 +1,354 @@ +package metrics_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management/metrics" +) + +type mockAlertsFetcher struct { + alerts []k8s.PrometheusAlert + err error +} + +func (m *mockAlertsFetcher) GetAlerts(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + return m.alerts, m.err +} + +func collectMetrics(t *testing.T, collector prometheus.Collector) []*dto.MetricFamily { + t.Helper() + reg := prometheus.NewRegistry() + reg.MustRegister(collector) + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather metrics: %v", err) + } + return families +} + +func findFamily(families []*dto.MetricFamily, name string) *dto.MetricFamily { + for _, f := range families { + if f.GetName() == name { + return f + } + } + return nil +} + +func labelValue(m *dto.Metric, name string) string { + for _, lp := range m.GetLabel() { + if lp.GetName() == name { + return lp.GetValue() + } + } + return "" +} + +func newCollector(t *testing.T, mock *mockAlertsFetcher) (prometheus.Collector, context.CancelFunc) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + collector := metrics.NewAlertsCollector(ctx, mock, 1*time.Hour, func() bool { return true }) + time.Sleep(100 * time.Millisecond) + t.Cleanup(cancel) + return collector, cancel +} + +func TestAlertsCollector_FiringAndSilenced(t *testing.T) { + activeAt := time.Date(2026, 3, 5, 10, 0, 0, 0, time.UTC) + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{"alertname": "HighCPU", "severity": "critical", "namespace": "production"}, + State: "firing", + ActiveAt: activeAt, + }, + { + Labels: map[string]string{"alertname": "DiskFull", "severity": "warning", "namespace": "storage"}, + State: "silenced", + ActiveAt: activeAt.Add(-1 * time.Hour), + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil { + t.Fatal("expected metric family, got nil") + } + if len(family.GetMetric()) != 2 { + t.Fatalf("expected 2 metrics, got %d", len(family.GetMetric())) + } + + var firing, silenced *dto.Metric + for _, m := range family.GetMetric() { + switch labelValue(m, "alertname") { + case "HighCPU": + firing = m + case "DiskFull": + silenced = m + } + } + + if firing == nil { + t.Fatal("expected HighCPU metric") + } + if labelValue(firing, "alertstate") != "firing" { + t.Errorf("alertstate: want firing, got %q", labelValue(firing, "alertstate")) + } + if labelValue(firing, "severity") != "critical" { + t.Errorf("severity: want critical, got %q", labelValue(firing, "severity")) + } + if labelValue(firing, "namespace") != "production" { + t.Errorf("namespace: want production, got %q", labelValue(firing, "namespace")) + } + if firing.GetGauge().GetValue() != float64(activeAt.Unix()) { + t.Errorf("gauge value: want %v, got %v", float64(activeAt.Unix()), firing.GetGauge().GetValue()) + } + + if silenced == nil { + t.Fatal("expected DiskFull metric") + } + if labelValue(silenced, "alertstate") != "silenced" { + t.Errorf("alertstate: want silenced, got %q", labelValue(silenced, "alertstate")) + } + if silenced.GetGauge().GetValue() != float64(activeAt.Add(-1*time.Hour).Unix()) { + t.Errorf("silenced gauge value mismatch") + } +} + +func TestAlertsCollector_NoAnnotationLabels(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "TestAlert"}, State: "firing", ActiveAt: time.Now()}, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil { + t.Fatal("expected metric family") + } + if len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric, got %d", len(family.GetMetric())) + } + for _, lp := range family.GetMetric()[0].GetLabel() { + switch lp.GetName() { + case "summary", "description", "runbook_url": + t.Errorf("unexpected annotation label: %s", lp.GetName()) + } + } +} + +func TestAlertsCollector_SkipsZeroActiveAt(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "NoActiveAt"}, State: "firing", ActiveAt: time.Time{}}, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family != nil && len(family.GetMetric()) != 0 { + t.Errorf("expected no metrics for zero ActiveAt, got %d", len(family.GetMetric())) + } +} + +func TestAlertsCollector_EmptyAlerts(t *testing.T) { + mock := &mockAlertsFetcher{alerts: []k8s.PrometheusAlert{}} + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family != nil && len(family.GetMetric()) != 0 { + t.Errorf("expected no metrics for empty alerts, got %d", len(family.GetMetric())) + } +} + +func TestAlertsCollector_FetcherErrorProducesNoMetrics(t *testing.T) { + mock := &mockAlertsFetcher{err: errors.New("connection refused")} + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family != nil && len(family.GetMetric()) != 0 { + t.Errorf("expected no metrics on initial failure, got %d", len(family.GetMetric())) + } +} + +func TestAlertsCollector_ClassificationLabels(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{ + "alertname": "KubePodCrashLooping", + "severity": "warning", + "namespace": "kube-system", + k8s.AlertRuleLabelId: "abc123", + k8s.AlertRuleClassificationComponentKey: "kube-controller-manager", + k8s.AlertRuleClassificationLayerKey: "cluster", + }, + State: "firing", + ActiveAt: time.Now(), + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric, got family=%v", family) + } + m := family.GetMetric()[0] + checks := map[string]string{ + k8s.AlertRuleLabelId: "abc123", + k8s.AlertRuleClassificationComponentKey: "kube-controller-manager", + k8s.AlertRuleClassificationLayerKey: "cluster", + "alertstate": "firing", + } + for k, want := range checks { + if got := labelValue(m, k); got != want { + t.Errorf("label[%s]: want %q, got %q", k, want, got) + } + } +} + +func TestAlertsCollector_IncludesPendingAlerts(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "Firing"}, State: "firing", ActiveAt: time.Now()}, + {Labels: map[string]string{"alertname": "Silenced"}, State: "silenced", ActiveAt: time.Now()}, + {Labels: map[string]string{"alertname": "Pending"}, State: "pending", ActiveAt: time.Now()}, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 3 { + t.Fatalf("expected 3 metrics, got %v", family) + } + states := map[string]bool{} + for _, m := range family.GetMetric() { + states[labelValue(m, "alertstate")] = true + } + for _, s := range []string{"firing", "silenced", "pending"} { + if !states[s] { + t.Errorf("expected state %q in metrics", s) + } + } +} + +func TestAlertsCollector_SourceAndBackendLabels(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{ + "alertname": "HighCPU", + "severity": "critical", + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.AlertBackendLabel: k8s.AlertBackendAM, + }, + State: "firing", + ActiveAt: time.Now(), + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric") + } + m := family.GetMetric()[0] + if got := labelValue(m, k8s.AlertSourceLabel); got != k8s.AlertSourcePlatform { + t.Errorf("source: want %q, got %q", k8s.AlertSourcePlatform, got) + } + if got := labelValue(m, k8s.AlertBackendLabel); got != k8s.AlertBackendAM { + t.Errorf("backend: want %q, got %q", k8s.AlertBackendAM, got) + } +} + +func TestAlertsCollector_FiltersThanosBackend(t *testing.T) { + now := time.Now() + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "HighCPU", k8s.AlertBackendLabel: k8s.AlertBackendAM, k8s.AlertSourceLabel: k8s.AlertSourcePlatform}, State: "firing", ActiveAt: now}, + {Labels: map[string]string{"alertname": "HighCPU", k8s.AlertBackendLabel: k8s.AlertBackendThanos, k8s.AlertSourceLabel: k8s.AlertSourceUser}, State: "firing", ActiveAt: now}, + {Labels: map[string]string{"alertname": "PendingAlert", k8s.AlertBackendLabel: k8s.AlertBackendProm, k8s.AlertSourceLabel: k8s.AlertSourcePlatform}, State: "pending", ActiveAt: now}, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 2 { + t.Fatalf("expected 2 metrics (thanos filtered), got %v", family) + } + for _, m := range family.GetMetric() { + if labelValue(m, k8s.AlertBackendLabel) == k8s.AlertBackendThanos { + t.Error("thanos duplicate should be filtered out") + } + } +} + +func TestAlertsCollector_InjectsClassificationFromFields(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{"alertname": "TestAlert", k8s.AlertBackendLabel: k8s.AlertBackendAM}, + State: "firing", + ActiveAt: time.Now(), + AlertComponent: "networking", + AlertLayer: "cluster", + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric") + } + m := family.GetMetric()[0] + if got := labelValue(m, k8s.AlertRuleClassificationComponentKey); got != "networking" { + t.Errorf("component: want networking, got %q", got) + } + if got := labelValue(m, k8s.AlertRuleClassificationLayerKey); got != "cluster" { + t.Errorf("layer: want cluster, got %q", got) + } +} + +func TestAlertsCollector_DoesNotOverwriteARCLabels(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{ + "alertname": "TestAlert", + k8s.AlertBackendLabel: k8s.AlertBackendAM, + k8s.AlertRuleClassificationComponentKey: "arc-component", + k8s.AlertRuleClassificationLayerKey: "namespace", + }, + State: "firing", + ActiveAt: time.Now(), + AlertComponent: "default-component", + AlertLayer: "cluster", + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric") + } + m := family.GetMetric()[0] + if got := labelValue(m, k8s.AlertRuleClassificationComponentKey); got != "arc-component" { + t.Errorf("component: want arc-component, got %q", got) + } + if got := labelValue(m, k8s.AlertRuleClassificationLayerKey); got != "namespace" { + t.Errorf("layer: want namespace, got %q", got) + } +} diff --git a/pkg/management/metrics/leader_election.go b/pkg/management/metrics/leader_election.go new file mode 100644 index 000000000..2a71f9231 --- /dev/null +++ b/pkg/management/metrics/leader_election.go @@ -0,0 +1,87 @@ +package metrics + +import ( + "context" + "fmt" + "os" + "sync" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" +) + +const ( + leaseName = "monitoring-plugin-metrics" + leaseDuration = 15 * time.Second + leaseRenew = 10 * time.Second + leaseRetry = 2 * time.Second +) + +// startLeaderElection sets up Lease-based leader election for the alerts +// effective metric. Returns a thread-safe isLeader callback. +func startLeaderElection(ctx context.Context, kubeConfig *rest.Config, namespace string) (func() bool, error) { + coordClient, err := coordinationv1client.NewForConfig(kubeConfig) + if err != nil { + return nil, fmt.Errorf("create coordination client: %w", err) + } + + identity, err := os.Hostname() + if err != nil { + return nil, fmt.Errorf("get hostname: %w", err) + } + + lock := &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{ + Name: leaseName, + Namespace: namespace, + }, + Client: coordClient, + LockConfig: resourcelock.ResourceLockConfig{ + Identity: identity, + }, + } + + var mu sync.Mutex + isLeading := false + + isLeader := func() bool { + mu.Lock() + defer mu.Unlock() + return isLeading + } + + le, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ + Lock: lock, + LeaseDuration: leaseDuration, + RenewDeadline: leaseRenew, + RetryPeriod: leaseRetry, + ReleaseOnCancel: true, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(_ context.Context) { + mu.Lock() + isLeading = true + mu.Unlock() + metricsLog.Info("became leader for alert management metrics") + }, + OnStoppedLeading: func() { + mu.Lock() + isLeading = false + mu.Unlock() + metricsLog.Info("lost leadership for alert management metrics") + }, + OnNewLeader: func(identity string) { + metricsLog.Infof("new leader for alert management metrics: %s", identity) + }, + }, + }) + if err != nil { + return nil, fmt.Errorf("create leader elector: %w", err) + } + + go le.Run(ctx) + return isLeader, nil +} diff --git a/pkg/management/testutils/k8s_client_mock.go b/pkg/management/testutils/k8s_client_mock.go index 0b9adbdb2..370125966 100644 --- a/pkg/management/testutils/k8s_client_mock.go +++ b/pkg/management/testutils/k8s_client_mock.go @@ -17,6 +17,8 @@ import ( // by AlertingRules().Get) hit the same store. type MockClient struct { TestConnectionFunc func(ctx context.Context) error + AlertingHealthFunc func(ctx context.Context) (k8s.AlertingHealth, error) + PrometheusAlertsFunc func() k8s.PrometheusAlertsInterface PrometheusRulesFunc func() k8s.PrometheusRuleInterface AlertRelabelConfigsFunc func() k8s.AlertRelabelConfigInterface AlertingRulesFunc func() k8s.AlertingRuleInterface @@ -38,6 +40,20 @@ func (m *MockClient) TestConnection(ctx context.Context) error { return nil } +func (m *MockClient) AlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) { + if m.AlertingHealthFunc != nil { + return m.AlertingHealthFunc(ctx) + } + return k8s.AlertingHealth{}, nil +} + +func (m *MockClient) PrometheusAlerts() k8s.PrometheusAlertsInterface { + if m.PrometheusAlertsFunc != nil { + return m.PrometheusAlertsFunc() + } + return &MockPrometheusAlertsInterface{} +} + func (m *MockClient) PrometheusRules() k8s.PrometheusRuleInterface { if m.PrometheusRulesFunc != nil { return m.PrometheusRulesFunc() @@ -88,7 +104,42 @@ func (m *MockClient) Namespace() k8s.NamespaceInterface { return m.namespace } -// MockPrometheusRuleInterface is a mock implementation of k8s.PrometheusRuleInterface +type MockPrometheusAlertsInterface struct { + GetAlertsFunc func(ctx context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) + GetRulesFunc func(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) + + ActiveAlerts []k8s.PrometheusAlert + RuleGroups []k8s.PrometheusRuleGroup +} + +func (m *MockPrometheusAlertsInterface) SetActiveAlerts(alerts []k8s.PrometheusAlert) { + m.ActiveAlerts = alerts +} + +func (m *MockPrometheusAlertsInterface) SetRuleGroups(groups []k8s.PrometheusRuleGroup) { + m.RuleGroups = groups +} + +func (m *MockPrometheusAlertsInterface) GetAlerts(ctx context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) { + if m.GetAlertsFunc != nil { + return m.GetAlertsFunc(ctx, req) + } + if m.ActiveAlerts != nil { + return m.ActiveAlerts, nil + } + return []k8s.PrometheusAlert{}, nil +} + +func (m *MockPrometheusAlertsInterface) GetRules(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) { + if m.GetRulesFunc != nil { + return m.GetRulesFunc(ctx, req) + } + if m.RuleGroups != nil { + return m.RuleGroups, nil + } + return []k8s.PrometheusRuleGroup{}, nil +} + type MockPrometheusRuleInterface struct { ListFunc func() ([]monitoringv1.PrometheusRule, error) GetFunc func(ctx context.Context, namespace string, name string) (*monitoringv1.PrometheusRule, bool, error) diff --git a/pkg/management/types.go b/pkg/management/types.go index e150832c8..64edd0810 100644 --- a/pkg/management/types.go +++ b/pkg/management/types.go @@ -2,8 +2,12 @@ package management import ( "context" + "net/http" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/client-go/rest" + + "github.com/openshift/monitoring-plugin/pkg/k8s" ) // Client is the interface for managing alert rules @@ -46,6 +50,21 @@ type Client interface { UpdateAlertRuleClassification(ctx context.Context, req UpdateRuleClassificationRequest) error // BulkUpdateAlertRuleClassification updates classification for multiple rule ids BulkUpdateAlertRuleClassification(ctx context.Context, items []UpdateRuleClassificationRequest) []error + + // ListRules lists alert rules, optionally paginated via cursor-based pagination + ListRules(ctx context.Context, prOptions PrometheusRuleOptions, arOptions AlertRuleOptions, pgOptions PaginationOptions) (ListRulesResult, error) + + // GetAlerts retrieves Prometheus alerts + GetAlerts(ctx context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, error) + // GetRules retrieves Prometheus alerting rules and active alerts + GetRules(ctx context.Context, req k8s.GetRulesRequest) ([]k8s.PrometheusRuleGroup, error) + + // GetAlertingHealth retrieves alerting health details + GetAlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) + + // MetricsHandler returns an HTTP handler that exposes alert management metrics. + // It handles leader election internally using the provided kubeConfig. + MetricsHandler(ctx context.Context, kubeConfig *rest.Config) (http.Handler, error) } // PrometheusRuleOptions specifies options for selecting PrometheusRule resources and groups @@ -59,3 +78,30 @@ type PrometheusRuleOptions struct { // GroupName of the RuleGroup within the PrometheusRule resource GroupName string `json:"groupName"` } + +type AlertRuleOptions struct { + // Name filters alert rules by alert name + Name string `json:"name,omitempty"` + + // Source filters alert rules by source type (platform or user-defined) + Source string `json:"source,omitempty"` + + // Labels filters alert rules by arbitrary label key-value pairs + Labels map[string]string `json:"labels,omitempty"` +} + +// PaginationOptions controls cursor-based pagination for list endpoints. +type PaginationOptions struct { + // Limit is the maximum number of results to return. Zero means no limit. + Limit int + + // NextToken is an opaque cursor returned by a previous call; results will + // start after the rule identified by this token. + NextToken string +} + +// ListRulesResult holds a page of rules and an optional cursor for the next page. +type ListRulesResult struct { + Rules []monitoringv1.Rule `json:"rules"` + NextToken string `json:"nextToken,omitempty"` +} diff --git a/pkg/management/update_classification.go b/pkg/management/update_classification.go index 777fc8846..fa76ec913 100644 --- a/pkg/management/update_classification.go +++ b/pkg/management/update_classification.go @@ -434,28 +434,3 @@ func getOriginalPlatformRuleFromPR(pr *monitoringv1.PrometheusRule, namespace st AdditionalInfo: fmt.Sprintf("in PrometheusRule %s/%s", namespace, name), } } - -// ApplyDynamicClassification resolves the effective component and layer for an -// alert by applying _from indirection. If a rule carries a component_from or -// layer_from label, the corresponding alert label value is used instead of the -// static default. Unresolvable or empty lookups fall back to the supplied -// defaults. -func ApplyDynamicClassification(ruleLabels, alertLabels map[string]string, defaultComponent, defaultLayer string) (string, string) { - component := defaultComponent - layer := defaultLayer - - if ruleLabels != nil { - if fromKey := ruleLabels[k8s.AlertRuleClassificationComponentFromKey]; fromKey != "" { - if v, ok := alertLabels[fromKey]; ok && v != "" { - component = v - } - } - if fromKey := ruleLabels[k8s.AlertRuleClassificationLayerFromKey]; fromKey != "" { - if v, ok := alertLabels[fromKey]; ok && v != "" { - layer = strings.ToLower(v) - } - } - } - - return component, layer -} diff --git a/pkg/server/server.go b/pkg/server/server.go index 5c5c49800..c048d455d 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -184,7 +184,10 @@ func createHTTPServer(ctx context.Context, cfg *Config) (*http.Server, error) { log.Info("alert management API enabled") } - router, pluginConfig := setupRoutes(cfg, managementClient) + router, pluginConfig, err := setupRoutes(ctx, cfg, managementClient, k8sconfig) + if err != nil { + return nil, fmt.Errorf("failed to set up routes: %w", err) + } router.Use(corsHeaderMiddleware()) tlsConfig := &tls.Config{} @@ -275,7 +278,7 @@ func createHTTPServer(ctx context.Context, cfg *Config) (*http.Server, error) { return httpServer, nil } -func setupRoutes(cfg *Config, managementClient management.Client) (*mux.Router, *PluginConfig) { +func setupRoutes(ctx context.Context, cfg *Config, managementClient management.Client, k8sconfig *rest.Config) (*mux.Router, *PluginConfig, error) { configHandlerFunc, pluginConfig := configHandler(cfg) router := mux.NewRouter() @@ -290,11 +293,18 @@ func setupRoutes(cfg *Config, managementClient management.Client) (*mux.Router, if managementClient != nil { managementRouter := managementrouter.New(managementClient) router.PathPrefix("/api/v1/alerting").Handler(managementRouter) + + metricsHandler, err := managementClient.MetricsHandler(ctx, k8sconfig) + if err != nil { + return nil, nil, fmt.Errorf("failed to start alert management metrics: %w", err) + } + router.Path("/metrics").Handler(metricsHandler) + log.Info("alert management metrics started") } router.PathPrefix("/").Handler(filesHandler(http.Dir(cfg.StaticPath))) - return router, pluginConfig + return router, pluginConfig, nil } func setupProxyRoutes(cfg *Config, k8sclient *dynamic.DynamicClient, kind monitoring.KindType) *mux.Router { @@ -366,7 +376,6 @@ func corsHeaderMiddleware() func(next http.Handler) http.Handler { func featuresHandler(cfg *Config) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { jsonFeatures, err := json.Marshal(cfg.Features) - if err != nil { log.WithError(err).Errorf("cannot marshall, features were: %v", string(jsonFeatures)) http.Error(w, err.Error(), http.StatusInternalServerError) @@ -380,7 +389,6 @@ func featuresHandler(cfg *Config) http.HandlerFunc { func configHandler(cfg *Config) (http.HandlerFunc, *PluginConfig) { pluginConfData, err := os.ReadFile(cfg.PluginConfigPath) - if err != nil { log.WithError(err).Warnf("cannot read config file, serving plugin with default configuration, tried %s", cfg.PluginConfigPath) @@ -392,7 +400,6 @@ func configHandler(cfg *Config) (http.HandlerFunc, *PluginConfig) { var pluginConfig PluginConfig err = yaml.Unmarshal(pluginConfData, &pluginConfig) - if err != nil { log.WithError(err).Error("unable to unmarshall config data") return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -401,7 +408,6 @@ func configHandler(cfg *Config) (http.HandlerFunc, *PluginConfig) { } jsonPluginConfig, err := pluginConfig.MarshalJSON() - if err != nil { log.WithError(err).Errorf("unable to marshall, config data: %v", pluginConfig) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/test/e2e/alerts_effective_metric_test.go b/test/e2e/alerts_effective_metric_test.go new file mode 100644 index 000000000..614655f8c --- /dev/null +++ b/test/e2e/alerts_effective_metric_test.go @@ -0,0 +1,315 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management/metrics" + "github.com/openshift/monitoring-plugin/test/e2e/framework" +) + +func fetchMetrics(f *framework.Framework) (string, error) { + resp, err := f.HTTPClient().Get(f.PluginURL + "/metrics") + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(body), nil +} + +func parseMetricLines(body string) []string { + var lines []string + for _, line := range strings.Split(body, "\n") { + if strings.HasPrefix(line, metrics.MetricName+"{") { + lines = append(lines, line) + } + } + return lines +} + +func extractLabel(metricLine, labelName string) string { + key := labelName + `="` + idx := strings.Index(metricLine, key) + if idx < 0 { + return "" + } + start := idx + len(key) + end := strings.Index(metricLine[start:], `"`) + if end < 0 { + return "" + } + return metricLine[start : start+end] +} + +// TestMetricEndpointExposesEffectiveMetric +// Verifies that the /metrics endpoint exposes alerts_effective_active_at_timestamp_seconds. +func TestMetricEndpointExposesEffectiveMetric(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + var metricBody string + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + body, err := fetchMetrics(f) + if err != nil { + t.Logf("Failed to fetch metrics: %v", err) + return false, nil + } + + if !strings.Contains(body, metrics.MetricName) { + t.Logf("Metric %s not found yet (leader election may be in progress)", metrics.MetricName) + return false, nil + } + + metricBody = body + return true, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for metric to appear: %v", err) + } + + if !strings.Contains(metricBody, "# HELP "+metrics.MetricName) { + t.Error("Missing HELP line for metric") + } + if !strings.Contains(metricBody, "# TYPE "+metrics.MetricName+" gauge") { + t.Error("Missing or incorrect TYPE line for metric (expected gauge)") + } + + lines := parseMetricLines(metricBody) + if len(lines) == 0 { + t.Fatal("Expected at least one metric series, got none") + } + + t.Logf("Found %d metric series for %s", len(lines), metrics.MetricName) +} + +// TestMetricSeriesHaveRequiredLabels +// Verifies every metric series has alertname, alertstate, openshift_io_alert_source, +// openshift_io_alert_backend, and a valid timestamp value. +func TestMetricSeriesHaveRequiredLabels(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + var lines []string + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + body, err := fetchMetrics(f) + if err != nil { + t.Logf("Failed to fetch metrics: %v", err) + return false, nil + } + lines = parseMetricLines(body) + return len(lines) > 0, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for metric series: %v", err) + } + + requiredLabels := []string{ + "alertname", + "alertstate", + k8s.AlertSourceLabel, + k8s.AlertBackendLabel, + } + + for i, line := range lines { + for _, label := range requiredLabels { + val := extractLabel(line, label) + if val == "" { + t.Errorf("Series %d missing required label %q: %s", i, label, line) + } + } + + state := extractLabel(line, "alertstate") + validStates := map[string]bool{"firing": true, "pending": true, "silenced": true, "suppressed": true} + if !validStates[state] { + t.Errorf("Series %d has unexpected alertstate=%q: %s", i, state, line) + } + + parts := strings.Split(line, " ") + if len(parts) < 2 { + t.Errorf("Series %d has no value: %s", i, line) + continue + } + var ts float64 + if _, err := fmt.Sscanf(parts[len(parts)-1], "%g", &ts); err != nil { + t.Errorf("Series %d has unparseable value %q: %v", i, parts[len(parts)-1], err) + continue + } + if ts < 9.46e+08 { + t.Errorf("Series %d has suspiciously low timestamp value: %g (before year 2000)", i, ts) + } + } + + t.Logf("All %d series have required labels and valid values", len(lines)) +} + +// TestMetricIncludesClassificationLabels +// Verifies that all metric series have classification labels +// (openshift_io_alert_rule_component and openshift_io_alert_rule_layer). +func TestMetricIncludesClassificationLabels(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + var lines []string + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + body, err := fetchMetrics(f) + if err != nil { + return false, nil + } + lines = parseMetricLines(body) + return len(lines) > 0, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for metric series: %v", err) + } + + for i, line := range lines { + if extractLabel(line, k8s.AlertRuleClassificationComponentKey) == "" { + t.Errorf("Series %d missing %s label: %s", i, k8s.AlertRuleClassificationComponentKey, line) + } + if extractLabel(line, k8s.AlertRuleClassificationLayerKey) == "" { + t.Errorf("Series %d missing %s label: %s", i, k8s.AlertRuleClassificationLayerKey, line) + } + } + + t.Logf("All %d series have classification labels (component + layer)", len(lines)) +} + +// TestMetricExcludesAnnotations +// Verifies that annotations (summary, description, runbook_url) are not +// included as metric labels. +func TestMetricExcludesAnnotations(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + var lines []string + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + body, err := fetchMetrics(f) + if err != nil { + return false, nil + } + lines = parseMetricLines(body) + return len(lines) > 0, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for metric series: %v", err) + } + + annotationLabels := []string{"summary", "description", "runbook_url"} + + for i, line := range lines { + for _, annLabel := range annotationLabels { + if extractLabel(line, annLabel) != "" { + t.Errorf("Series %d contains annotation label %q (annotations should be excluded): %s", + i, annLabel, line) + } + } + } + + t.Logf("Verified %d series - none contain annotation labels", len(lines)) +} + +// TestMetricActiveAtTimestampsAreReasonable +// Verifies that activeAt timestamps are not too recent. +func TestMetricActiveAtTimestampsAreReasonable(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + var lines []string + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + body, err := fetchMetrics(f) + if err != nil { + return false, nil + } + lines = parseMetricLines(body) + return len(lines) > 0, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for metric series: %v", err) + } + + now := float64(time.Now().Unix()) + fiveMinutesAgo := now - 300 + + recentCount := 0 + for _, line := range lines { + alertname := extractLabel(line, "alertname") + if alertname == "Watchdog" { + continue + } + + parts := strings.Split(line, " ") + if len(parts) < 2 { + continue + } + valueStr := parts[len(parts)-1] + + var ts float64 + if _, err := fmt.Sscanf(valueStr, "%e", &ts); err != nil { + if _, err := fmt.Sscanf(valueStr, "%f", &ts); err != nil { + continue + } + } + + if ts > fiveMinutesAgo { + recentCount++ + t.Logf("WARN: %s has activeAt within last 5 minutes (ts=%.0f, now=%.0f)", alertname, ts, now) + } + } + + totalNonWatchdog := 0 + for _, line := range lines { + if extractLabel(line, "alertname") != "Watchdog" { + totalNonWatchdog++ + } + } + + if totalNonWatchdog > 0 { + recentPct := float64(recentCount) / float64(totalNonWatchdog) * 100 + if recentPct > 80 { + t.Errorf("%.0f%% of alerts (%d/%d) have activeAt within last 5 minutes — "+ + "likely using Alertmanager startsAt instead of Prometheus activeAt", + recentPct, recentCount, totalNonWatchdog) + } + } + + t.Logf("Timestamp check: %d/%d non-Watchdog alerts have recent activeAt", recentCount, totalNonWatchdog) +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index eb181d521..9357ed5a6 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -332,6 +332,64 @@ func (f *Framework) CreateScopedUser(ctx context.Context, name, namespace, apiGr return &ScopedUser{Token: token, Cleanup: func() error { rollback(); return nil }}, nil } +// CreateUserWithClusterRole creates a ServiceAccount in the given namespace and +// binds an existing ClusterRole to it via a namespaced RoleBinding. Use this +// for OpenShift built-in roles such as monitoring-rules-view (Thanos tenancy +// on port 9093). API calls are retried to tolerate transient failures. +func (f *Framework) CreateUserWithClusterRole(ctx context.Context, name, namespace, clusterRoleName string) (*ScopedUser, error) { + rollback := func() { + _ = f.Clientset.RbacV1().RoleBindings(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + _ = f.Clientset.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + } + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + } + err := retry(3, func() error { + _, err := f.Clientset.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err + }) + if err != nil { + return nil, fmt.Errorf("creating service account %s/%s: %w", namespace, name, err) + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Subjects: []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, + Name: name, + Namespace: namespace, + }}, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: clusterRoleName, + }, + } + err = retry(3, func() error { + _, err := f.Clientset.RbacV1().RoleBindings(namespace).Create(ctx, rb, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err + }) + if err != nil { + rollback() + return nil, fmt.Errorf("creating role binding %s/%s for cluster role %s: %w", namespace, name, clusterRoleName, err) + } + + token, err := f.requestServiceAccountToken(ctx, namespace, name) + if err != nil { + rollback() + return nil, err + } + + return &ScopedUser{Token: token, Cleanup: func() error { rollback(); return nil }}, nil +} + // CreateAnonymousUser creates a ServiceAccount with no RBAC permissions. // The whole setup is retried to tolerate transient API failures. func (f *Framework) CreateAnonymousUser(ctx context.Context, name, namespace string) (*ScopedUser, error) { diff --git a/test/e2e/get_alerts_test.go b/test/e2e/get_alerts_test.go new file mode 100644 index 000000000..49fbbe334 --- /dev/null +++ b/test/e2e/get_alerts_test.go @@ -0,0 +1,297 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "testing" + "time" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/test/e2e/framework" +) + +func TestGetAlerts(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + testNamespace, cleanup, err := f.CreateUserNamespace(ctx, "test-get-alerts") + if err != nil { + t.Fatalf("Failed to create test namespace: %v", err) + } + defer cleanup() + + forDuration := monitoringv1.Duration("1s") + alertName := "E2EGetAlertsTest" + + promRule := &monitoringv1.PrometheusRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-get-alerts-rule", + Namespace: testNamespace, + }, + Spec: monitoringv1.PrometheusRuleSpec{ + Groups: []monitoringv1.RuleGroup{ + { + Name: "e2e-test-group", + Rules: []monitoringv1.Rule{ + { + Alert: alertName, + Expr: intstr.FromString("vector(1)"), + For: &forDuration, + Labels: map[string]string{ + "severity": "none", + "team": "e2e", + }, + Annotations: map[string]string{ + "summary": "E2E test alert for GET /alerts", + }, + }, + }, + }, + }, + }, + } + + _, err = f.Monitoringv1clientset.MonitoringV1().PrometheusRules(testNamespace).Create( + ctx, promRule, metav1.CreateOptions{}, + ) + if err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + httpClient := f.HTTPClient() + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + alertsURL := f.PluginURL + "/api/v1/alerting/alerts" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, alertsURL, nil) + if err != nil { + return false, err + } + if f.BearerToken != "" { + req.Header.Set("Authorization", "Bearer "+f.BearerToken) + } + + resp, err := httpClient.Do(req) + if err != nil { + t.Logf("Failed to query alerts: %v", err) + return false, nil + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Logf("GET /alerts returned status %d, retrying", resp.StatusCode) + return false, nil + } + + var alertsResp struct { + Data struct { + Alerts []k8s.PrometheusAlert `json:"alerts"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&alertsResp); err != nil { + t.Logf("Failed to decode alerts response: %v", err) + return false, nil + } + + for _, alert := range alertsResp.Data.Alerts { + if alert.Labels["alertname"] == alertName { + if alert.State != "firing" && alert.State != "pending" { + t.Logf("Found alert %s but state is %q, waiting for firing/pending", alertName, alert.State) + return false, nil + } + if alert.Labels["severity"] != "none" { + t.Errorf("Expected severity=none, got %q", alert.Labels["severity"]) + } + t.Logf("Found alert %s in state %q", alertName, alert.State) + return true, nil + } + } + + t.Logf("Alert %s not found yet (got %d alerts total)", alertName, len(alertsResp.Data.Alerts)) + return false, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for alert to appear: %v", err) + } + + t.Log("GET /alerts e2e test passed successfully") +} + +// TestRBAC_GetAlerts verifies Thanos-tenancy RBAC for GET /alerts. +// +// With ?namespace=: User A (no perms) gets HTTP 200 without the UWM alert in +// ns Y; User B (monitoring-rules-view in Y) sees Y but not Z; cluster-admin +// sees Y. +// +// Without ?namespace=: fan-out must not leak the alert to unprivileged users +// and must still return it for namespace-scoped viewers. +func TestRBAC_GetAlerts(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + nsY, cleanupY, err := f.CreateUserNamespace(ctx, "test-rbac-get-alerts-y") + if err != nil { + t.Fatalf("Failed to create namespace Y: %v", err) + } + defer func() { _ = cleanupY() }() + + nsZ, cleanupZ, err := f.CreateUserNamespace(ctx, "test-rbac-get-alerts-z") + if err != nil { + t.Fatalf("Failed to create namespace Z: %v", err) + } + defer func() { _ = cleanupZ() }() + + userA, err := f.CreateAnonymousUser(ctx, "e2e-rbac-get-a", "default") + if err != nil { + t.Fatalf("Failed to create unprivileged user A: %v", err) + } + defer func() { _ = userA.Cleanup() }() + + userB, err := f.CreateUserWithClusterRole(ctx, "e2e-rbac-get-b", nsY, "monitoring-rules-view") + if err != nil { + t.Fatalf("Failed to create scoped user B: %v", err) + } + defer func() { _ = userB.Cleanup() }() + + alertName := "E2ERBACGetAlertsTest" + if err := createFiringPrometheusRule(ctx, f, nsY, "e2e-rbac-get-alerts-rule", alertName); err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + alerts, status, err := getAlertsWithToken(f, ctx, f.BearerToken, nsY) + if err != nil { + t.Logf("Admin GET /alerts failed: %v", err) + return false, nil + } + if status != http.StatusOK { + t.Logf("Admin GET /alerts returned status %d, retrying", status) + return false, nil + } + if containsAlert(alerts, alertName) { + t.Logf("Admin sees alert %s", alertName) + return true, nil + } + t.Logf("Waiting for alert %s (admin sees %d alerts)", alertName, len(alerts)) + return false, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for admin to see alert: %v", err) + } + + cases := []struct { + name string + token string + namespace string + wantAlert bool + }{ + {"UserA_NoPerms_NamespaceY", userA.Token, nsY, false}, + {"UserA_NoPerms_NoNamespace", userA.Token, "", false}, + {"UserB_RulesView_NamespaceY", userB.Token, nsY, true}, + {"UserB_RulesView_NamespaceZ", userB.Token, nsZ, false}, + {"UserB_RulesView_NoNamespace", userB.Token, "", true}, + {"UserC_ClusterAdmin_NamespaceY", f.BearerToken, nsY, true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + alerts, status, err := getAlertsWithToken(f, ctx, tc.token, tc.namespace) + if err != nil { + t.Fatalf("GET /alerts request failed: %v", err) + } + if status != http.StatusOK { + t.Fatalf("Expected status %d, got %d", http.StatusOK, status) + } + got := containsAlert(alerts, alertName) + if got != tc.wantAlert { + t.Fatalf("Alert %s visibility: want %v, got %v (%d alerts returned)", alertName, tc.wantAlert, got, len(alerts)) + } + }) + } +} + +func createFiringPrometheusRule(ctx context.Context, f *framework.Framework, namespace, name, alertName string) error { + forDuration := monitoringv1.Duration("1s") + promRule := &monitoringv1.PrometheusRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: monitoringv1.PrometheusRuleSpec{ + Groups: []monitoringv1.RuleGroup{{ + Name: "e2e-rbac-group", + Rules: []monitoringv1.Rule{{ + Alert: alertName, + Expr: intstr.FromString("vector(1)"), + For: &forDuration, + Labels: map[string]string{ + "severity": "none", + "e2e_test": "rbac_get", + }, + }}, + }}, + }, + } + _, err := f.Monitoringv1clientset.MonitoringV1().PrometheusRules(namespace).Create(ctx, promRule, metav1.CreateOptions{}) + return err +} + +func containsAlert(alerts []k8s.PrometheusAlert, alertName string) bool { + for _, a := range alerts { + if a.Labels["alertname"] == alertName { + return true + } + } + return false +} + +// getAlertsWithToken calls GET /alerts with an optional namespace query param. +// It returns the decoded alerts and HTTP status. A non-OK status is not an +// error — callers must assert on status explicitly. +func getAlertsWithToken(f *framework.Framework, ctx context.Context, token, namespace string) ([]k8s.PrometheusAlert, int, error) { + alertsURL := f.PluginURL + "/api/v1/alerting/alerts" + if namespace != "" { + alertsURL += "?" + url.Values{"namespace": {namespace}}.Encode() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, alertsURL, nil) + if err != nil { + return nil, 0, fmt.Errorf("creating request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+token) + + resp, err := f.HTTPClient().Do(req) + if err != nil { + return nil, 0, fmt.Errorf("executing request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, resp.StatusCode, nil + } + + var alertsResp struct { + Data struct { + Alerts []k8s.PrometheusAlert `json:"alerts"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&alertsResp); err != nil { + return nil, resp.StatusCode, fmt.Errorf("decoding response: %w", err) + } + return alertsResp.Data.Alerts, resp.StatusCode, nil +} diff --git a/test/e2e/health_test.go b/test/e2e/health_test.go new file mode 100644 index 000000000..d0485293e --- /dev/null +++ b/test/e2e/health_test.go @@ -0,0 +1,57 @@ +package e2e + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/test/e2e/framework" +) + +func TestGetHealth(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + healthURL := f.PluginURL + "/api/v1/alerting/health" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthURL, nil) + if err != nil { + t.Fatalf("Failed to create HTTP request: %v", err) + } + if f.BearerToken != "" { + req.Header.Set("Authorization", "Bearer "+f.BearerToken) + } + + resp, err := f.HTTPClient().Do(req) + if err != nil { + t.Fatalf("Failed to make health request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("Expected status 200, got %d", resp.StatusCode) + } + + var healthResp struct { + Alerting *k8s.AlertingHealth `json:"alerting"` + } + if err := json.NewDecoder(resp.Body).Decode(&healthResp); err != nil { + t.Fatalf("Failed to decode health response: %v", err) + } + + if healthResp.Alerting == nil { + t.Fatal("Expected 'alerting' field in health response") + } + + if healthResp.Alerting.Platform == nil { + t.Error("Expected 'platform' field in alerting health") + } + + t.Logf("Health response: userWorkloadEnabled=%v", healthResp.Alerting.UserWorkloadEnabled) + t.Log("GET /health e2e test passed successfully") +} diff --git a/test/e2e/relabeled_rules_test.go b/test/e2e/relabeled_rules_test.go new file mode 100644 index 000000000..d5087ed38 --- /dev/null +++ b/test/e2e/relabeled_rules_test.go @@ -0,0 +1,443 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "testing" + "time" + + osmv1 "github.com/openshift/api/monitoring/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/test/e2e/framework" +) + +type listRulesRuleGroup struct { + Name string `json:"name"` + Rules []monitoringv1.Rule `json:"rules"` +} + +type listRulesResponse struct { + Data struct { + Groups []listRulesRuleGroup `json:"groups"` + } `json:"data"` +} + +func listRules(ctx context.Context, f *framework.Framework) ([]monitoringv1.Rule, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, f.PluginURL+"/api/v1/alerting/rules", nil) + if err != nil { + return nil, err + } + if f.BearerToken != "" { + req.Header.Set("Authorization", "Bearer "+f.BearerToken) + } + + resp, err := f.HTTPClient().Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + var listResp listRulesResponse + if err := json.NewDecoder(resp.Body).Decode(&listResp); err != nil { + return nil, err + } + + var allRules []monitoringv1.Rule + for _, group := range listResp.Data.Groups { + allRules = append(allRules, group.Rules...) + } + return allRules, nil +} + +func TestPrometheusRuleAppearsInMemory(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + testNamespace, cleanup, err := f.CreateUserNamespace(ctx, "test-prometheus-rule") + if err != nil { + t.Fatalf("Failed to create test namespace: %v", err) + } + defer cleanup() + + testAlertName := "TestAlert" + forDuration := monitoringv1.Duration("5m") + testRule := monitoringv1.Rule{ + Alert: testAlertName, + Expr: intstr.FromString("up == 0"), + For: &forDuration, + Labels: map[string]string{ + "severity": "warning", + }, + Annotations: map[string]string{ + "description": "Test alert for e2e testing", + "summary": "Test alert", + }, + } + + _, err = createPrometheusRule(ctx, f, testNamespace, testRule) + if err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + rules, err := listRules(ctx, f) + if err != nil { + t.Logf("Failed to list rules: %v", err) + return false, nil + } + + for _, rule := range rules { + if rule.Alert == testAlertName { + expectedLabels := map[string]string{ + k8s.PrometheusRuleLabelNamespace: testNamespace, + k8s.PrometheusRuleLabelName: "test-prometheus-rule", + } + + if err := compareRuleLabels(t, testAlertName, rule.Labels, expectedLabels); err != nil { + return false, err + } + + if _, ok := rule.Labels[k8s.AlertRuleLabelId]; !ok { + t.Errorf("Alert %s missing openshift_io_alert_rule_id label", testAlertName) + return false, fmt.Errorf("alert missing openshift_io_alert_rule_id label") + } + + t.Logf("Found alert %s in memory with all expected labels", testAlertName) + return true, nil + } + } + + t.Logf("Alert %s not found in memory yet (found %d rules)", testAlertName, len(rules)) + return false, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for alert to appear in memory: %v", err) + } +} + +func TestRelabelAlert(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + testNamespace, cleanup, err := f.CreatePlatformNamespace(ctx, "test-relabel-alert") + if err != nil { + t.Fatalf("Failed to create test namespace: %v", err) + } + defer cleanup() + + forDuration := monitoringv1.Duration("5m") + + criticalRule := monitoringv1.Rule{ + Alert: "TestRelabelAlert", + Expr: intstr.FromString("up == 0"), + For: &forDuration, + Labels: map[string]string{ + "severity": "critical", + "team": "web", + }, + Annotations: map[string]string{ + "description": "Critical alert for relabel testing", + "summary": "Critical test alert", + }, + } + + warningRule := monitoringv1.Rule{ + Alert: "TestRelabelAlert", + Expr: intstr.FromString("up == 1"), + For: &forDuration, + Labels: map[string]string{ + "severity": "warning", + "team": "web", + }, + Annotations: map[string]string{ + "description": "Warning alert for relabel testing", + "summary": "Warning test alert", + }, + } + + _, err = createPrometheusRule(ctx, f, testNamespace, criticalRule, warningRule) + if err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + relabelConfigName := "change-critical-team" + arc := &osmv1.AlertRelabelConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: relabelConfigName, + Namespace: k8s.ClusterMonitoringNamespace, + }, + Spec: osmv1.AlertRelabelConfigSpec{ + Configs: []osmv1.RelabelConfig{ + { + SourceLabels: []osmv1.LabelName{"alertname", "severity"}, + Regex: "TestRelabelAlert;critical", + Separator: ";", + TargetLabel: "team", + Replacement: "ops", + Action: "Replace", + }, + }, + }, + } + + _, err = f.Osmv1clientset.MonitoringV1().AlertRelabelConfigs(k8s.ClusterMonitoringNamespace).Create( + ctx, arc, metav1.CreateOptions{}, + ) + if err != nil { + t.Fatalf("Failed to create AlertRelabelConfig: %v", err) + } + defer func() { + err = f.Osmv1clientset.MonitoringV1().AlertRelabelConfigs(k8s.ClusterMonitoringNamespace).Delete(ctx, relabelConfigName, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Failed to delete AlertRelabelConfig: %v", err) + } + }() + + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + rules, err := listRules(ctx, f) + if err != nil { + t.Logf("Failed to list rules: %v", err) + return false, nil + } + + foundCriticalWithOps := false + + for _, rule := range rules { + if rule.Alert == "TestRelabelAlert" { + if rule.Labels["team"] == "ops" && rule.Labels["severity"] == "critical" { + t.Logf("Found critical alert with team=ops (relabeling successful)") + foundCriticalWithOps = true + } + } + } + + if foundCriticalWithOps { + t.Logf("Relabeling verified: critical alert has team=ops") + return true, nil + } + + t.Logf("Waiting for relabeling to take effect") + return false, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for relabeling to take effect: %v", err) + } +} + +func createPrometheusRule(ctx context.Context, f *framework.Framework, namespace string, rules ...monitoringv1.Rule) (*monitoringv1.PrometheusRule, error) { + interval := monitoringv1.Duration("30s") + prometheusRule := &monitoringv1.PrometheusRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-prometheus-rule", + Namespace: namespace, + }, + Spec: monitoringv1.PrometheusRuleSpec{ + Groups: []monitoringv1.RuleGroup{ + { + Name: "test-group", + Interval: &interval, + Rules: rules, + }, + }, + }, + } + + return f.Monitoringv1clientset.MonitoringV1().PrometheusRules(namespace).Create( + ctx, prometheusRule, metav1.CreateOptions{}, + ) +} + +func compareRuleLabels(t *testing.T, alertName string, foundLabels map[string]string, wantedLabels map[string]string) error { + t.Helper() + if foundLabels == nil { + t.Errorf("Alert %s has no labels", alertName) + return fmt.Errorf("alert has no labels") + } + + for key, wantValue := range wantedLabels { + if gotValue, ok := foundLabels[key]; !ok { + t.Errorf("Alert %s missing %s label", alertName, key) + return fmt.Errorf("alert missing %s label", key) + } else if gotValue != wantValue { + t.Errorf("Alert %s has wrong %s label. Expected %s, got %s", + alertName, key, wantValue, gotValue) + return fmt.Errorf("alert has wrong %s label", key) + } + } + + return nil +} + +// TestRBAC_GetRules verifies Thanos-tenancy RBAC for GET /rules. +// +// With ?namespace=: User A (no perms) gets HTTP 200 without the UWM rule in +// ns Y; User B (monitoring-rules-view in Y) sees Y but not Z; cluster-admin +// sees Y. +// +// Without ?namespace=: fan-out must not leak the rule to unprivileged users +// and must still return it for namespace-scoped viewers. +func TestRBAC_GetRules(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + nsY, cleanupY, err := f.CreateUserNamespace(ctx, "test-rbac-get-rules-y") + if err != nil { + t.Fatalf("Failed to create namespace Y: %v", err) + } + defer func() { _ = cleanupY() }() + + nsZ, cleanupZ, err := f.CreateUserNamespace(ctx, "test-rbac-get-rules-z") + if err != nil { + t.Fatalf("Failed to create namespace Z: %v", err) + } + defer func() { _ = cleanupZ() }() + + userA, err := f.CreateAnonymousUser(ctx, "e2e-rbac-rules-a", "default") + if err != nil { + t.Fatalf("Failed to create unprivileged user A: %v", err) + } + defer func() { _ = userA.Cleanup() }() + + userB, err := f.CreateUserWithClusterRole(ctx, "e2e-rbac-rules-b", nsY, "monitoring-rules-view") + if err != nil { + t.Fatalf("Failed to create scoped user B: %v", err) + } + defer func() { _ = userB.Cleanup() }() + + alertName := "E2ERBACGetRulesTest" + forDuration := monitoringv1.Duration("5m") + testRule := monitoringv1.Rule{ + Alert: alertName, + Expr: intstr.FromString("vector(1)"), + For: &forDuration, + Labels: map[string]string{ + "severity": "none", + "e2e_test": "rbac_get_rules", + }, + } + + _, err = createPrometheusRule(ctx, f, nsY, testRule) + if err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + rules, status, err := listRulesWithToken(ctx, f, f.BearerToken, nsY) + if err != nil { + t.Logf("Admin GET /rules failed: %v", err) + return false, nil + } + if status != http.StatusOK { + t.Logf("Admin GET /rules returned status %d, retrying", status) + return false, nil + } + if containsRule(rules, alertName) { + return true, nil + } + t.Logf("Waiting for rule %s (admin sees %d rules)", alertName, len(rules)) + return false, nil + }) + if err != nil { + t.Fatalf("Timeout waiting for admin to see rule: %v", err) + } + + cases := []struct { + name string + token string + namespace string + wantRule bool + }{ + {"UserA_NoPerms_NamespaceY", userA.Token, nsY, false}, + {"UserA_NoPerms_NoNamespace", userA.Token, "", false}, + {"UserB_RulesView_NamespaceY", userB.Token, nsY, true}, + {"UserB_RulesView_NamespaceZ", userB.Token, nsZ, false}, + {"UserB_RulesView_NoNamespace", userB.Token, "", true}, + {"UserC_ClusterAdmin_NamespaceY", f.BearerToken, nsY, true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rules, status, err := listRulesWithToken(ctx, f, tc.token, tc.namespace) + if err != nil { + t.Fatalf("GET /rules request failed: %v", err) + } + if status != http.StatusOK { + t.Fatalf("Expected status %d, got %d", http.StatusOK, status) + } + got := containsRule(rules, alertName) + if got != tc.wantRule { + t.Fatalf("Rule %s visibility: want %v, got %v (%d rules returned)", alertName, tc.wantRule, got, len(rules)) + } + }) + } +} + +func containsRule(rules []monitoringv1.Rule, alertName string) bool { + for _, r := range rules { + if r.Alert == alertName { + return true + } + } + return false +} + +// listRulesWithToken calls GET /rules with an optional namespace query param. +// A non-OK status is not an error — callers must assert on status explicitly. +func listRulesWithToken(ctx context.Context, f *framework.Framework, token, namespace string) ([]monitoringv1.Rule, int, error) { + rulesURL := f.PluginURL + "/api/v1/alerting/rules" + if namespace != "" { + rulesURL += "?" + url.Values{"namespace": {namespace}}.Encode() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rulesURL, nil) + if err != nil { + return nil, 0, err + } + req.Header.Set("Authorization", "Bearer "+token) + + resp, err := f.HTTPClient().Do(req) + if err != nil { + return nil, 0, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, resp.StatusCode, nil + } + + var listResp listRulesResponse + if err := json.NewDecoder(resp.Body).Decode(&listResp); err != nil { + return nil, resp.StatusCode, err + } + + var allRules []monitoringv1.Rule + for _, group := range listResp.Data.Groups { + allRules = append(allRules, group.Rules...) + } + return allRules, resp.StatusCode, nil +}