From c30ef937e920fad14e8637d4faa49bb259244853 Mon Sep 17 00:00:00 2001 From: Yamunadevi Shanmugam Date: Thu, 18 Jun 2026 13:42:07 +0530 Subject: [PATCH 1/4] test(audit): add OTE tests for API Migrate audit tests to OTE --- test/extended/cluster/audit.go | 166 +++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/test/extended/cluster/audit.go b/test/extended/cluster/audit.go index de51025c4810..bdc454d3cf0a 100644 --- a/test/extended/cluster/audit.go +++ b/test/extended/cluster/audit.go @@ -3,6 +3,7 @@ package cluster import ( "bufio" "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -10,6 +11,10 @@ import ( g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" + ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" + + configv1 "github.com/openshift/api/config/v1" + exutil "github.com/openshift/origin/test/extended/util" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -152,3 +157,164 @@ func parseAuditLine(line string) (auditEvent, error) { } return event, nil } + +// validAuditProfiles is the set of profile names accepted by the APIServer CR. +var validAuditProfiles = map[configv1.AuditProfileType]struct{}{ + configv1.DefaultAuditProfileType: {}, + configv1.WriteRequestBodiesAuditProfileType: {}, + configv1.AllRequestBodiesAuditProfileType: {}, + configv1.NoneAuditProfileType: {}, + configv1.AuditProfileType(""): {}, // unset is treated as Default +} + +var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Audit", func() { + oc := exutil.NewCLIWithoutNamespace("apiserver-audit") + + // Read current audit profile and assert it is one of the four known valid values. + g.It("[OTP] should report the current audit profile as a known valid value [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + apiServer, err := oc.AdminConfigClient().ConfigV1().APIServers().Get( + ctx, "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + _, ok := validAuditProfiles[apiServer.Spec.Audit.Profile] + o.Expect(ok).To(o.BeTrue(), + "unexpected audit profile %q, expected one of Default, WriteRequestBodies, AllRequestBodies, None", + apiServer.Spec.Audit.Profile) + }) + + // API server must reject an unrecognised audit profile name at admission time. + g.It("[OTP] should reject an invalid audit profile name in the APIServer configuration [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", + "--type=merge", + `--patch={"spec":{"audit":{"profile":"InvalidAuditProfile"}}}`, + "--dry-run=server", + ).Output() + o.Expect(err).To(o.HaveOccurred(), + "expected server-side validation to reject an invalid audit profile name") + }) + + // Custom audit rule must specify a non-empty group name; an empty value should be + // rejected at admission time without altering the live configuration. + g.It("[OTP] should reject a custom audit rule with an empty group name [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", + "--type=merge", + `--patch={"spec":{"audit":{"customRules":[{"group":"","profile":"Default"}]}}}`, + "--dry-run=server", + ).Output() + o.Expect(err).To(o.HaveOccurred(), + "expected server-side validation to reject a custom audit rule with an empty group name") + }) + + // Custom audit rule that references an unknown profile name must be rejected at + // admission time without altering the live configuration. + g.It("[OTP] should reject a custom audit rule with an invalid profile name [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", + "--type=merge", + `--patch={"spec":{"audit":{"customRules":[{"group":"system:authenticated","profile":"BadProfile"}]}}}`, + "--dry-run=server", + ).Output() + o.Expect(err).To(o.HaveOccurred(), + "expected server-side validation to reject a custom audit rule with an invalid profile name") + }) + + // Every documented profile name (Default, WriteRequestBodies, AllRequestBodies, None) + // must be accepted by the API server. A server-side dry-run is used so no rollout is triggered. + g.It("[OTP] should accept all valid audit profile names via server-side dry-run [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + for _, profile := range []configv1.AuditProfileType{ + configv1.DefaultAuditProfileType, + configv1.WriteRequestBodiesAuditProfileType, + configv1.AllRequestBodiesAuditProfileType, + configv1.NoneAuditProfileType, + } { + g.By(fmt.Sprintf("verifying profile %q is accepted by the API server", profile)) + patch := fmt.Sprintf(`{"spec":{"audit":{"profile":%q}}}`, profile) + _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", + "--type=merge", + "--patch="+patch, + "--dry-run=server", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred(), + "expected profile %q to be accepted by the API server", profile) + } + }) + + // Audit log files must be present under /var/log/kube-apiserver/ on every master node. + // Skipped on HyperShift where the control plane is hosted externally and master nodes are not + // directly accessible through node-logs. + g.It("[OTP] should have audit log files present on master nodes [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + if ok, _ := exutil.IsHypershift(ctx, oc.AdminConfigClient()); ok { + g.Skip("HyperShift hosts the control plane externally; master node logs are not accessible via node-logs") + } + + masters, err := oc.AsAdmin().KubeClient().CoreV1().Nodes().List( + ctx, metav1.ListOptions{LabelSelector: "node-role.kubernetes.io/master"}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(masters.Items).NotTo(o.BeEmpty(), "expected at least one master node") + + for _, master := range masters.Items { + g.By(fmt.Sprintf("checking for audit log files on master node %s", master.Name)) + output, err := oc.AsAdmin().WithoutNamespace().Run("adm").Args( + "node-logs", master.Name, "--path=kube-apiserver/", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred(), + "failed to list kube-apiserver log files on master node %s", master.Name) + o.Expect(output).To(o.MatchRegexp(`audit`), + "expected audit log files to be present on master node %s", master.Name) + } + }) + + // Each line in the kube-apiserver audit log must be valid JSON and contain the + // mandatory fields defined by the audit.k8s.io/v1 event schema + // (kind, apiVersion, level, requestURI, verb, user, stage). + // Skipped on HyperShift where the control plane is hosted externally and master nodes are not + // directly accessible through node-logs. + g.It("[OTP] should write audit log entries in valid JSON format with required fields [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + if ok, _ := exutil.IsHypershift(ctx, oc.AdminConfigClient()); ok { + g.Skip("HyperShift hosts the control plane externally; master node logs are not accessible via node-logs") + } + + masters, err := oc.AsAdmin().KubeClient().CoreV1().Nodes().List( + ctx, metav1.ListOptions{LabelSelector: "node-role.kubernetes.io/master"}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(masters.Items).NotTo(o.BeEmpty(), "expected at least one master node") + + // Inspect the most recent audit log entries from the first master only to + // keep the test fast while still validating the format is correct. + master := masters.Items[0] + g.By(fmt.Sprintf("reading recent audit log entries from master node %s", master.Name)) + output, err := oc.AsAdmin().WithoutNamespace().Run("adm").Args( + "node-logs", master.Name, "--path=kube-apiserver/audit.log", "--tail=20", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred(), + "failed to read kube-apiserver audit log from master node %s", master.Name) + + checkedLines := 0 + for _, line := range strings.Split(strings.TrimSpace(output), "\n") { + if strings.TrimSpace(line) == "" { + continue + } + var entry map[string]interface{} + o.Expect(json.Unmarshal([]byte(line), &entry)).NotTo(o.HaveOccurred(), + "audit log entry is not valid JSON on node %s: %s", master.Name, line) + + for _, field := range []string{"kind", "apiVersion", "level", "requestURI", "verb", "user", "stage"} { + o.Expect(entry).To(o.HaveKey(field), + "audit log entry on node %s is missing required field %q: %s", master.Name, field, line) + } + checkedLines++ + } + o.Expect(checkedLines).To(o.BeNumerically(">", 0), + "expected at least one non-empty audit log line on node %s", master.Name) + }) +}) From 018a553cee35a6398e0d2077d6c7b3365feaa7d6 Mon Sep 17 00:00:00 2001 From: Yamunadevi Shanmugam Date: Thu, 18 Jun 2026 18:26:30 +0530 Subject: [PATCH 2/4] test(prometheus): add OTE tests for API Migrate prometheus tests to OTE --- test/extended/prometheus/prometheus.go | 160 +++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/test/extended/prometheus/prometheus.go b/test/extended/prometheus/prometheus.go index 19a6ba5b8837..652399afaf3d 100644 --- a/test/extended/prometheus/prometheus.go +++ b/test/extended/prometheus/prometheus.go @@ -19,6 +19,7 @@ import ( g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" + ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" promv1 "github.com/prometheus/client_golang/api/prometheus/v1" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" @@ -1196,3 +1197,162 @@ func SkipOperatorHubMetricsCheck(oc *exutil.CLI) bool { } return stdout == "true" } + +// Verify that specific kube-apiserver and audit alert rules are present +// in the cluster's Prometheus instance and are correctly configured. +var _ = g.Describe("[sig-instrumentation] [Jira:apiserver-auth] Prometheus Alerts", func() { + oc := exutil.NewCLIWithoutNamespace("apiserver-alerts") + + g.BeforeEach(func() { + kubeClient, err := kubernetes.NewForConfig(oc.AdminConfig()) + o.Expect(err).NotTo(o.HaveOccurred()) + nsExist, err := exutil.IsNamespaceExist(kubeClient, "openshift-monitoring") + o.Expect(err).NotTo(o.HaveOccurred()) + if !nsExist { + g.Skip("openshift-monitoring namespace does not exist") + } + }) + + // ExtremelyHighIndividualControlPlaneCPU alert rule must be defined + // and contain the mandatory summary and description annotations. + g.It("[OTP] should have ExtremelyHighIndividualControlPlaneCPU alert rule defined with required annotations [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) + + alertName := "ExtremelyHighIndividualControlPlaneCPU" + g.By(fmt.Sprintf("looking for alert rule %q", alertName)) + rule := lookupAlertingRule(prometheusURL, bearerToken, alertName) + o.Expect(rule).NotTo(o.BeNil(), + "alert rule %q was not found in Prometheus", alertName) + + g.By("verifying required annotations are present") + o.Expect(rule.Annotations).To(o.HaveKey(model.LabelName("summary")), + "alert rule %q is missing 'summary' annotation", alertName) + o.Expect(rule.Annotations).To(o.HaveKey(model.LabelName("description")), + "alert rule %q is missing 'description' annotation", alertName) + }) + + // KubeAPIErrorBudgetBurn alert rule must be defined and + // carry a non-empty severity label. + g.It("[OTP] should have KubeAPIErrorBudgetBurn alert rule defined with a severity label [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) + + alertName := "KubeAPIErrorBudgetBurn" + g.By(fmt.Sprintf("looking for alert rule %q", alertName)) + rule := lookupAlertingRule(prometheusURL, bearerToken, alertName) + o.Expect(rule).NotTo(o.BeNil(), + "alert rule %q was not found in Prometheus", alertName) + + g.By("verifying severity label is present and non-empty") + severity, ok := rule.Labels["severity"] + o.Expect(ok).To(o.BeTrue(), + "alert rule %q is missing 'severity' label", alertName) + o.Expect(string(severity)).NotTo(o.BeEmpty(), + "alert rule %q has an empty 'severity' label", alertName) + }) + + // AuditLogError alert rule must be defined, its PromQL + // expression must reference audit-related metrics, and it must carry a summary annotation. + g.It("[OTP] should have AuditLogError alert rule defined and referencing audit metrics [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) + + alertName := "AuditLogError" + g.By(fmt.Sprintf("looking for alert rule %q", alertName)) + rule := lookupAlertingRule(prometheusURL, bearerToken, alertName) + o.Expect(rule).NotTo(o.BeNil(), + "alert rule %q was not found in Prometheus", alertName) + + g.By("verifying the PromQL expression references audit-related metrics") + o.Expect(string(rule.Query)).To(o.MatchRegexp(`audit`), + "alert rule %q query %q does not reference audit metrics", alertName, rule.Query) + + g.By("verifying required annotations are present") + o.Expect(rule.Annotations).To(o.HaveKey(model.LabelName("summary")), + "alert rule %q is missing 'summary' annotation", alertName) + }) + + // KubeAggregatedAPIErrors alert rule must be defined and + // carry a severity label. + g.It("[OTP] should have KubeAggregatedAPIErrors alert rule defined with severity label [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) + + alertName := "KubeAggregatedAPIErrors" + g.By(fmt.Sprintf("looking for alert rule %q", alertName)) + rule := lookupAlertingRule(prometheusURL, bearerToken, alertName) + o.Expect(rule).NotTo(o.BeNil(), + "alert rule %q was not found in Prometheus", alertName) + + g.By("verifying severity label is present") + _, ok := rule.Labels["severity"] + o.Expect(ok).To(o.BeTrue(), + "alert rule %q is missing 'severity' label", alertName) + }) + + // KubeAPIDown alert must be defined with severity=critical + // and must not be currently in a firing state. + g.It("[OTP] should have KubeAPIDown alert rule defined and not currently firing [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) + + alertName := "KubeAPIDown" + g.By(fmt.Sprintf("looking for alert rule %q", alertName)) + rule := lookupAlertingRule(prometheusURL, bearerToken, alertName) + o.Expect(rule).NotTo(o.BeNil(), + "alert rule %q was not found in Prometheus", alertName) + + g.By("verifying severity=critical label") + severity, ok := rule.Labels["severity"] + o.Expect(ok).To(o.BeTrue(), + "alert rule %q is missing 'severity' label", alertName) + o.Expect(string(severity)).To(o.Equal("critical"), + "alert rule %q expected severity=critical, got %q", alertName, severity) + + g.By(fmt.Sprintf("verifying %q is not currently firing", alertName)) + prometheusClient := oc.NewPrometheusClient(ctx) + err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 2*time.Minute, true, + func(ctx context.Context) (bool, error) { + resp, queryErr := helper.RunQuery(ctx, prometheusClient, + fmt.Sprintf(`ALERTS{alertname=%q,alertstate="firing"}`, alertName)) + if queryErr != nil { + e2e.Logf("prometheus query error (will retry): %v", queryErr) + return false, nil + } + o.Expect(resp.Data.Result).To(o.BeEmpty(), + "alert %q is unexpectedly firing", alertName) + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), + "timed out waiting to confirm %q is not firing", alertName) + }) +}) + +// lookupAlertingRule searches all Prometheus rule groups for a rule with the given +// name and returns it, or nil if not found. +func lookupAlertingRule(prometheusURL, bearerToken, name string) *promv1.AlertingRule { + rules, err := helper.FetchAlertingRules(prometheusURL, bearerToken) + if err != nil { + e2e.Logf("failed to fetch alerting rules: %v", err) + return nil + } + for _, group := range rules { + for i := range group { + if group[i].Name == name { + return &group[i] + } + } + } + return nil +} + +// mustGetPrometheusURLAndToken is a test helper that fetches the Prometheus route URL +// and a service-account bearer token, failing the test immediately on error. +func mustGetPrometheusURLAndToken(ctx g.SpecContext, oc *exutil.CLI) (string, string) { + prometheusURL, err := helper.PrometheusRouteURL(ctx, oc) + o.Expect(err).NotTo(o.HaveOccurred(), "get public URL of prometheus") + bearerToken, err := helper.RequestPrometheusServiceAccountAPIToken(ctx, oc) + o.Expect(err).NotTo(o.HaveOccurred(), "request prometheus service account API token") + return prometheusURL, bearerToken +} From 59439049b1411af967eb7e1474842b2caf0883c6 Mon Sep 17 00:00:00 2001 From: Yamunadevi Shanmugam Date: Thu, 18 Jun 2026 22:49:32 +0530 Subject: [PATCH 3/4] test(etcd-encryption): add OTE tests for API Migrate etcd encryption tests to OTE --- test/extended/apiserver/tls.go | 944 ++++++++++++++++++++++++++++++++- 1 file changed, 943 insertions(+), 1 deletion(-) diff --git a/test/extended/apiserver/tls.go b/test/extended/apiserver/tls.go index 58ac08e1524d..cc99f6b34ddb 100644 --- a/test/extended/apiserver/tls.go +++ b/test/extended/apiserver/tls.go @@ -3,16 +3,27 @@ package apiserver import ( "context" "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" "fmt" "io" "math/rand" + "net/url" + "os" "os/exec" + "path/filepath" + "regexp" + "strconv" "strings" "time" g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" + ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" e2e "k8s.io/kubernetes/test/e2e/framework" configv1 "github.com/openshift/api/config/v1" @@ -22,6 +33,15 @@ import ( const ( namespace = "apiserver-tls-test" + + // Logging frequency constants for polling loops to reduce noise + logEveryNAttemptsKeyRotation = 12 // Every 12 attempts * 5s = once per minute + logEveryNAttemptsSecretRecreate = 20 // Every 20 attempts * 3s = once per minute + logEveryNAttemptsKeyAppear = 10 // Every 10 attempts * 6s = once per minute + + // Encryption key secret name prefixes + encryptionKeyOASPrefix = "encryption-key-openshift-apiserver-" + encryptionKeyKASPrefix = "encryption-key-openshift-kube-apiserver-" ) // This test only checks whether components are serving the proper TLS version based @@ -210,7 +230,8 @@ func forwardPortAndExecute(serviceName, namespace, remotePort string, toExecute defer stderr.Close() defer e2e.TryKill(cmd) - e2e.Logf("oc port-forward output: %s", readPartialFrom(stdout, 1024)) + // Read and discard port-forward output to avoid logging sensitive cluster metadata + _ = readPartialFrom(stdout, 1024) return toExecute(localPort) }(); err == nil { return nil @@ -252,3 +273,924 @@ func checkTLSConnection(port int, tlsShouldWork, tlsShouldNotWork *tls.Config) e } return nil } + +var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Operators / Certs", func() { + defer g.GinkgoRecover() + + var oc = exutil.NewCLIWithoutNamespace("apiserver-certs") + + // Verify TLS secrets in openshift-kube-apiserver and CA bundles in + // openshift-config-managed are all valid (non-expired, not-yet-valid) PEM certificates. + g.It("[OTP] should have valid TLS certificates for authentication and encryption between API server components [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + g.By("1) check TLS secrets in openshift-kube-apiserver") + ns := "openshift-kube-apiserver" + e2e.Logf("==================================== OpenShift TLS Secrets Verification ====================================") + + secretsJSON, err := oc.AsAdmin().WithoutNamespace(). + Run("get").Args("secret", "-n", ns, "-ojson").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + var secretList struct { + Items []struct { + Type string `json:"type"` + Data map[string]string `json:"data"` + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + } `json:"items"` + } + o.Expect(json.Unmarshal([]byte(secretsJSON), &secretList)).NotTo(o.HaveOccurred(), + "failed to parse oc secret JSON output") + + for _, s := range secretList.Items { + if s.Type != "kubernetes.io/tls" { + continue + } + name := s.Metadata.Name + rawcrt, ok := s.Data["tls.crt"] + if !ok || rawcrt == "" { + e2e.Logf(" %s/%s — no tls.crt found, skipping", ns, name) + continue + } + decoded, decErr := base64.StdEncoding.DecodeString(rawcrt) + if decErr != nil { + decoded, decErr = base64.RawStdEncoding.DecodeString(rawcrt) + o.Expect(decErr).NotTo(o.HaveOccurred(), + "failed to base64-decode tls.crt in secret %s/%s", ns, name) + } + e2e.Logf(" checking secret %s/%s", ns, name) + parseAndCheckPEMs(decoded, fmt.Sprintf("%s/%s", ns, name)) + } + + g.By("2) check CA bundle configmaps in openshift-config-managed") + e2e.Logf("==================================== OpenShift CA Bundles Verification ====================================") + ns = "openshift-config-managed" + cms := []struct{ name, key string }{ + {"kube-apiserver-server-ca", "ca-bundle.crt"}, + {"kube-apiserver-client-ca", "ca-bundle.crt"}, + {"kube-root-ca.crt", "ca.crt"}, + {"trusted-ca-bundle", "ca-bundle.crt"}, + {"service-ca", "ca-bundle.crt"}, + } + for _, cm := range cms { + cmJSON, cmErr := oc.AsAdmin().WithoutNamespace(). + Run("get").Args("cm", cm.name, "-n", ns, "-ojson").Output() + o.Expect(cmErr).NotTo(o.HaveOccurred(), + "failed to get configmap %s/%s", ns, cm.name) + + var cmObj struct { + Data map[string]string `json:"data"` + } + err := json.Unmarshal([]byte(cmJSON), &cmObj) + o.Expect(err).NotTo(o.HaveOccurred(), + "failed to parse configmap JSON for %s/%s", ns, cm.name) + + val, ok := cmObj.Data[cm.key] + if !ok || val == "" { + // fallback: search any key containing "ca" or "crt" + found := false + for k, v := range cmObj.Data { + if kl := strings.ToLower(k); (strings.Contains(kl, "ca") || strings.Contains(kl, "crt")) && v != "" { + e2e.Logf(" checking configmap %s/%s (key=%s)", ns, cm.name, k) + parseAndCheckPEMs([]byte(v), fmt.Sprintf("%s/%s(%s)", ns, cm.name, k)) + found = true + break + } + } + o.Expect(found).To(o.BeTrue(), + "no CA/crt key found in configmap %s/%s", ns, cm.name) + continue + } + e2e.Logf(" checking configmap %s/%s (key=%s)", ns, cm.name, cm.key) + parseAndCheckPEMs([]byte(val), fmt.Sprintf("%s/%s(%s)", ns, cm.name, cm.key)) + } + e2e.Logf("Certificate verification complete.") + }) + + // Add a custom TLS certificate to the cluster API server, verify it is served, + // then restore the original configuration. + g.It("[OTP] should support adding a custom TLS certificate for the cluster API [Disruptive][Slow][apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + isHyperShift, err := exutil.IsHypershift(ctx, oc.AdminConfigClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isHyperShift { + g.Skip("custom serving certificates for the API server are managed by HyperShift — skipping") + } + + tmpdir := g.GinkgoT().TempDir() + // Use unique secret name to avoid conflicts with pre-existing secrets + testSecretName := fmt.Sprintf("custom-api-cert-test-%d", time.Now().Unix()) + + var ( + originKubeconfig = os.Getenv("KUBECONFIG") + originKubeconfigBkp = filepath.Join(tmpdir, "kubeconfig.origin.bkp") + originCA = filepath.Join(tmpdir, "certificate-authority-data-origin.crt") + newCA = filepath.Join(tmpdir, "certificate-authority-data-origin-new.crt") + cnBase = "kas-test-cert" + caKeypem = filepath.Join(tmpdir, "caKey.pem") + caCertpem = filepath.Join(tmpdir, "caCert.pem") + serverKeypem = filepath.Join(tmpdir, "serverKey.pem") + serverconf = filepath.Join(tmpdir, "server.conf") + serverWithSANcsr = filepath.Join(tmpdir, "serverWithSAN.csr") + serverCertWithSAN = filepath.Join(tmpdir, "serverCertWithSAN.pem") + ) + + // Snapshot original apiserver configuration before modification + g.By("0. snapshot original apiserver configuration") + origAPIServer, err := oc.AdminConfigClient().ConfigV1().APIServers().Get( + ctx, "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get original apiserver config") + + origNamedCerts := origAPIServer.Spec.ServingCerts.NamedCertificates + var patchToRecover string + if origNamedCerts == nil || len(origNamedCerts) == 0 { + patchToRecover = `{"spec":{"servingCerts": {"namedCertificates": null}}}` + } else { + // Marshal original namedCertificates to restore exact state + certsJSON, err := json.Marshal(origNamedCerts) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to marshal original namedCertificates") + patchToRecover = fmt.Sprintf(`{"spec":{"servingCerts": {"namedCertificates": %s}}}`, string(certsJSON)) + } + + defer func() { + g.By("restoring cluster to original state") + _, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "apiserver/cluster", "--type=merge", "-p", patchToRecover).Output() + if _, cpErr := exec.Command("bash", "-c", + fmt.Sprintf("cp %s %s", originKubeconfigBkp, originKubeconfig)).Output(); cpErr != nil { + e2e.Logf("warning: failed to restore kubeconfig: %v", cpErr) + } + err := oc.AsAdmin().WithoutNamespace().Run("adm").Args("wait-for-stable-cluster").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + // Only delete the test-specific secret, not any pre-existing secrets + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args( + "secret", testSecretName, "-n", "openshift-config", "--ignore-not-found").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + }() + + fqdnName, port := getAPIServerFQDNAndPort(ctx, oc) + + g.By("1. take a backup of the original kubeconfig") + origData, err := os.ReadFile(originKubeconfig) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(os.WriteFile(originKubeconfigBkp, origData, 0600)).NotTo(o.HaveOccurred()) + + g.By("2. extract the original CA certificate from kubeconfig") + kubeconfigData, err := os.ReadFile(originKubeconfig) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to read kubeconfig") + + // Parse kubeconfig to extract certificate-authority-data + var caDataBase64 string + for _, line := range strings.Split(string(kubeconfigData), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "certificate-authority-data:") { + caDataBase64 = strings.TrimSpace(strings.TrimPrefix(line, "certificate-authority-data:")) + break + } + } + o.Expect(caDataBase64).NotTo(o.BeEmpty(), "certificate-authority-data not found in kubeconfig") + + // Decode base64 and write to file + caDataDecoded, err := base64.StdEncoding.DecodeString(caDataBase64) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to decode certificate-authority-data") + err = os.WriteFile(originCA, caDataDecoded, 0600) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to write CA file") + + g.By("3. generate a new CA key, CA cert, server key, and server cert with SAN") + // Generate CA private key + out, err := exec.Command("openssl", "genrsa", "-out", caKeypem, "2048").CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "openssl genrsa CA failed: %s", out) + + // Generate CA certificate + out, err = exec.Command("openssl", "req", "-x509", "-new", "-nodes", + "-key", caKeypem, "-days", "100000", "-out", caCertpem, + "-subj", fmt.Sprintf("/CN=%s_ca", cnBase)).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "openssl req CA failed: %s", out) + + // Generate server private key + out, err = exec.Command("openssl", "genrsa", "-out", serverKeypem, "2048").CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "openssl genrsa server failed: %s", out) + serverconfContent := fmt.Sprintf(`[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = %s`, fqdnName) + o.Expect(os.WriteFile(serverconf, []byte(serverconfContent), 0644)).NotTo(o.HaveOccurred()) + + // Generate server CSR with SAN + out, err = exec.Command("openssl", "req", "-new", + "-key", serverKeypem, "-out", serverWithSANcsr, + "-subj", fmt.Sprintf("/CN=%s_server", cnBase), + "-config", serverconf).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "openssl req server CSR failed: %s", out) + + // Sign server certificate with CA + out, err = exec.Command("openssl", "x509", "-req", + "-in", serverWithSANcsr, "-CA", caCertpem, "-CAkey", caKeypem, + "-CAcreateserial", "-out", serverCertWithSAN, + "-days", "100000", "-extensions", "v3_req", "-extfile", serverconf).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "openssl x509 sign failed: %s", out) + + g.By("4. create a TLS secret for the custom API certificate") + err = oc.AsAdmin().WithoutNamespace().Run("create").Args( + "secret", "tls", testSecretName, + "--cert="+serverCertWithSAN, "--key="+serverKeypem, + "-n", "openshift-config").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("5. patch apiserver/cluster to use the new named certificate") + patchCmd := fmt.Sprintf( + `{"spec":{"servingCerts": {"namedCertificates": [{"names": ["%s"], "servingCertificate": {"name": "%s"}}]}}}`, + fqdnName, testSecretName) + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "apiserver/cluster", "--type=merge", "-p", patchCmd).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("6. update kubeconfig to include the new CA alongside the original CA") + caCertData, err := os.ReadFile(caCertpem) + o.Expect(err).NotTo(o.HaveOccurred()) + originCAData, err := os.ReadFile(originCA) + o.Expect(err).NotTo(o.HaveOccurred()) + concatenated := append(caCertData, originCAData...) + o.Expect(os.WriteFile(newCA, concatenated, 0644)).NotTo(o.HaveOccurred()) + b64Cert := base64.StdEncoding.EncodeToString(concatenated) + updateKubeconfCmd := fmt.Sprintf( + `sed -i "s/certificate-authority-data: .*/certificate-authority-data: %s/" %s`, + b64Cert, originKubeconfig) + _, err = exec.Command("bash", "-c", updateKubeconfCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("7. wait for kube-apiserver operator to start progressing (≤300s)") + err = waitCoBecomes(ctx, oc, "kube-apiserver", 300, map[string]string{"Progressing": "True"}) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not start progressing within 300s") + + e2e.Logf("waiting for kube-apiserver operator to become stable (≤1500s)") + err = waitCoBecomes(ctx, oc, "kube-apiserver", 1500, map[string]string{ + "Available": "True", + "Progressing": "False", + "Degraded": "False", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not stabilise within 1500s") + + g.By("8. validate that the custom certificate is now served by the API server") + certDetails, err := getServerCertInfo(fqdnName, port, caCertpem) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(certDetails.Subject).To(o.ContainSubstring("CN=kas-test-cert_server")) + o.Expect(certDetails.Issuer).To(o.ContainSubstring("CN=kas-test-cert_ca")) + + g.By("9. validate the original CA no longer verifies the new certificate") + _, err = getServerCertInfo(fqdnName, port, originCA) + o.Expect(err).To(o.HaveOccurred(), "original CA should not verify the new custom certificate") + }) + + // Force encryption key rotation for the etcd datastore by patching both + // openshiftapiserver and kubeapiserver with an unsupportedConfigOverride, then verify + // that new encryption keys are generated and the resources are re-encrypted. + g.It("[OTP] should force etcd encryption key rotation and verify resources are re-encrypted [Disruptive][Slow][apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + g.By("1. check that the cluster has etcd encryption enabled") + encryptionType, err := oc.WithoutNamespace().Run("get").Args( + "apiserver/cluster", "-o=jsonpath={.spec.encryption.type}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + encryptionWasEnabled := (encryptionType == "aescbc" || encryptionType == "aesgcm") + if !encryptionWasEnabled { + e2e.Logf("etcd encryption is not enabled (current type: %s), enabling aescbc encryption", encryptionType) + + // Enable encryption + g.By("1a. enabling etcd encryption with aescbc") + err = oc.WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", "--type=merge", + "-p", `{"spec":{"encryption":{"type":"aescbc"}}}`).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + // Wait for encryption to be enabled and migration to complete + g.By("1b. waiting for kube-apiserver operator to process encryption enablement (≤300s)") + err = waitCoBecomes(ctx, oc, "kube-apiserver", 300, map[string]string{"Progressing": "True"}) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not start progressing within 300s") + + e2e.Logf("waiting for kube-apiserver operator to stabilize (≤1800s)") + err = waitCoBecomes(ctx, oc, "kube-apiserver", 1800, map[string]string{ + "Available": "True", + "Progressing": "False", + "Degraded": "False", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not stabilize within 1800s") + + // Wait for openshift-apiserver as well + g.By("1c. waiting for openshift-apiserver operator to stabilize (≤1800s)") + err = waitCoBecomes(ctx, oc, "openshift-apiserver", 1800, map[string]string{ + "Available": "True", + "Progressing": "False", + "Degraded": "False", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "openshift-apiserver operator did not stabilize within 1800s") + + encryptionType = "aescbc" + e2e.Logf("etcd encryption successfully enabled with type: %s", encryptionType) + + // Cleanup: restore to identity (no encryption) at the end + defer func() { + e2e.Logf("restoring encryption to identity (disabled)") + _ = oc.WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", "--type=merge", + "-p", `{"spec":{"encryption":{"type":"identity"}}}`).Execute() + }() + } else { + e2e.Logf("etcd encryption type: %s", encryptionType) + } + + g.By("2. record current encryption prefixes") + oasEncValPrefix1, err := getEncryptionPrefix(ctx, oc, "/openshift.io/routes") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get encryption prefix for routes") + e2e.Logf("openshift-apiserver encryption prefix recorded before rotation") + + kasEncValPrefix1, err := getEncryptionPrefix(ctx, oc, "/kubernetes.io/secrets") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get encryption prefix for secrets") + e2e.Logf("kube-apiserver encryption prefix recorded before rotation") + + // Record current highest key numbers before rotation + oasEncNumberBefore, err := getEncryptionKeyNumber(oc, `encryption-key-openshift-apiserver-[^ ]*`) + o.Expect(err).NotTo(o.HaveOccurred()) + kasEncNumberBefore, err := getEncryptionKeyNumber(oc, `encryption-key-openshift-kube-apiserver-[^ ]*`) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("encryption keys before rotation: openshift-apiserver=%d, kube-apiserver=%d", + oasEncNumberBefore, kasEncNumberBefore) + + t := time.Now().Format(time.RFC3339) + patchYamlToRestore := `[{"op":"replace","path":"/spec/unsupportedConfigOverrides","value":null}]` + patchYaml := ` +spec: + unsupportedConfigOverrides: + encryption: + reason: force OAS rotation ` + t + + for i, kind := range []string{"openshiftapiserver", "kubeapiserver"} { + defer func(k string) { + e2e.Logf("restoring %s/cluster unsupportedConfigOverrides", k) + _ = oc.WithoutNamespace().Run("patch").Args( + k, "cluster", "--type=json", "-p", patchYamlToRestore).Execute() + }(kind) + g.By(fmt.Sprintf("3.%d) force %s encryption key rotation", i+1, kind)) + err := oc.WithoutNamespace().Run("patch").Args( + kind, "cluster", "--type=merge", "-p", patchYaml).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + g.By("4. wait for new encryption key secrets to appear (up to 15 minutes)") + // Use an explicit timeout context derived from the spec context so Ginkgo cancellation still works + // Increased to 15 minutes because kube-apiserver operator can be throttled and take 12+ minutes + pollCtx, cancel := context.WithTimeout(ctx, 15*time.Minute) + defer cancel() + + // Pre-compile regexes outside the polling loop for efficiency + oasPattern := regexp.MustCompile(`encryption-key-openshift-apiserver-[^ ]*`) + kasPattern := regexp.MustCompile(`encryption-key-openshift-kube-apiserver-[^ ]*`) + + var newOASEncNumber, newKASEncNumber int + retryCount := 0 + errKey := wait.PollUntilContextTimeout(pollCtx, 5*time.Second, 15*time.Minute, false, + func(pollCtx context.Context) (bool, error) { + // Dynamically check for new keys instead of pre-calculating expected numbers + currentOAS, err := getEncryptionKeyNumberWithRegex(oc, oasPattern) + if err != nil { + return false, nil + } + currentKAS, err := getEncryptionKeyNumberWithRegex(oc, kasPattern) + if err != nil { + return false, nil + } + + // Check if both operators created new keys (number increased) + if currentOAS > oasEncNumberBefore && currentKAS > kasEncNumberBefore { + newOASEncNumber = currentOAS + newKASEncNumber = currentKAS + e2e.Logf("new encryption keys detected after %d attempts: openshift-apiserver=%d, kube-apiserver=%d", + retryCount, newOASEncNumber, newKASEncNumber) + return true, nil + } + + retryCount++ + // Only log every 12th attempt (once per minute) to reduce noise + if retryCount%12 == 1 { + e2e.Logf("waiting for new encryption keys (attempt %d): openshift-apiserver=%d (want >%d), kube-apiserver=%d (want >%d)", + retryCount, currentOAS, oasEncNumberBefore, currentKAS, kasEncNumberBefore) + } + return false, nil + }) + + o.Expect(errKey).NotTo(o.HaveOccurred(), + "new encryption keys not created after 15 minutes (openshift-apiserver: want >%d, kube-apiserver: want >%d)", + oasEncNumberBefore, kasEncNumberBefore) + + g.By("5. wait for kube-apiserver encryption migration to complete") + newKASEncSecretName := "encryption-key-openshift-kube-apiserver-" + strconv.Itoa(newKASEncNumber) + completed, err := waitEncryptionKeyMigration(ctx, oc, newKASEncSecretName) + o.Expect(err).NotTo(o.HaveOccurred(), + "encryption key migration did not complete for %s", newKASEncSecretName) + o.Expect(completed).To(o.BeTrue()) + + g.By("6. verify encryption prefixes changed after rotation") + oasEncValPrefix2, err := getEncryptionPrefix(ctx, oc, "/openshift.io/routes") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get encryption prefix for routes after rotation") + e2e.Logf("openshift-apiserver encryption prefix verified after rotation") + + kasEncValPrefix2, err := getEncryptionPrefix(ctx, oc, "/kubernetes.io/secrets") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get encryption prefix for secrets after rotation") + e2e.Logf("kube-apiserver encryption prefix verified after rotation") + + o.Expect(oasEncValPrefix2).To(o.ContainSubstring(fmt.Sprintf("k8s:enc:%s:v1", encryptionType))) + o.Expect(kasEncValPrefix2).To(o.ContainSubstring(fmt.Sprintf("k8s:enc:%s:v1", encryptionType))) + o.Expect(oasEncValPrefix2).NotTo(o.Equal(oasEncValPrefix1), + "encryption prefix for routes did not change after key rotation") + o.Expect(kasEncValPrefix2).NotTo(o.Equal(kasEncValPrefix1), + "encryption prefix for secrets did not change after key rotation") + }) + + // Delete etcd encryption config and key secrets, then verify the cluster + // self-heals by recreating them and completing re-encryption. + g.It("[OTP] should self-recover when etcd encryption configuration secrets are deleted [Disruptive][Slow][apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + g.By("1. check that the cluster has etcd encryption enabled") + encryptionType, err := oc.WithoutNamespace().Run("get").Args( + "apiserver/cluster", "-o=jsonpath={.spec.encryption.type}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + encryptionWasEnabled := (encryptionType == "aescbc" || encryptionType == "aesgcm") + if !encryptionWasEnabled { + e2e.Logf("etcd encryption is not enabled (current type: %s), enabling aescbc encryption", encryptionType) + + // Enable encryption + g.By("1a. enabling etcd encryption with aescbc") + err = oc.WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", "--type=merge", + "-p", `{"spec":{"encryption":{"type":"aescbc"}}}`).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + // Wait for encryption to be enabled and migration to complete + g.By("1b. waiting for kube-apiserver operator to process encryption enablement (≤300s)") + err = waitCoBecomes(ctx, oc, "kube-apiserver", 300, map[string]string{"Progressing": "True"}) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not start progressing within 300s") + + e2e.Logf("waiting for kube-apiserver operator to stabilize (≤1800s)") + err = waitCoBecomes(ctx, oc, "kube-apiserver", 1800, map[string]string{ + "Available": "True", + "Progressing": "False", + "Degraded": "False", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not stabilize within 1800s") + + // Wait for openshift-apiserver as well + g.By("1c. waiting for openshift-apiserver operator to stabilize (≤1800s)") + err = waitCoBecomes(ctx, oc, "openshift-apiserver", 1800, map[string]string{ + "Available": "True", + "Progressing": "False", + "Degraded": "False", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "openshift-apiserver operator did not stabilize within 1800s") + + encryptionType = "aescbc" + e2e.Logf("etcd encryption successfully enabled with type: %s", encryptionType) + + // Cleanup: restore to identity (no encryption) at the end + defer func() { + e2e.Logf("restoring encryption to identity (disabled)") + _ = oc.WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", "--type=merge", + "-p", `{"spec":{"encryption":{"type":"identity"}}}`).Execute() + }() + } else { + e2e.Logf("etcd encryption type: %s", encryptionType) + } + + uidsOld, err := oc.WithoutNamespace().Run("get").Args( + "secret", + "encryption-config-openshift-apiserver", + "encryption-config-openshift-kube-apiserver", + "-n", "openshift-config-managed", + "-o=jsonpath={.items[*].metadata.uid}", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("2. delete encryption-config-* secrets from openshift-config-managed") + for _, item := range []string{ + "encryption-config-openshift-apiserver", + "encryption-config-openshift-kube-apiserver", + } { + e2e.Logf("removing finalizers from secret %s", item) + err := oc.WithoutNamespace().Run("patch").Args( + "secret", item, "-n", "openshift-config-managed", + `-p={"metadata":{"finalizers":null}}`).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + e2e.Logf("deleting secret %s", item) + err = oc.WithoutNamespace().Run("delete").Args( + "secret", item, "-n", "openshift-config-managed").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + uidsOldSlice := strings.Split(uidsOld, " ") + e2e.Logf("original secret count: %d", len(uidsOldSlice)) + + // Use an explicit timeout context derived from the spec context so Ginkgo cancellation still works + pollCtx1, cancel1 := context.WithTimeout(ctx, 5*time.Minute) + defer cancel1() + + retryCount1 := 0 + errSecret := wait.PollUntilContextTimeout(pollCtx1, 3*time.Second, 5*time.Minute, false, + func(pollCtx context.Context) (bool, error) { + uidsNew, err := oc.WithoutNamespace().Run("get").Args( + "secret", + "encryption-config-openshift-apiserver", + "encryption-config-openshift-kube-apiserver", + "-n", "openshift-config-managed", + "-o=jsonpath={.items[*].metadata.uid}", + ).Output() + if err != nil { + retryCount1++ + // Only log every 20th attempt (once per minute) to reduce noise + if retryCount1%20 == 1 { + e2e.Logf("waiting for encryption-config-* secrets to be recreated (attempt %d)", retryCount1) + } + return false, nil + } + uidsNewSlice := strings.Split(uidsNew, " ") + if len(uidsNewSlice) >= 2 && uidsNewSlice[0] != uidsOldSlice[0] && uidsNewSlice[1] != uidsOldSlice[1] { + e2e.Logf("encryption-config-* secrets recreated after %d attempts (UIDs changed)", retryCount1) + return true, nil + } + return false, nil + }) + o.Expect(errSecret).NotTo(o.HaveOccurred(), + "encryption-config-* secrets were not recreated within 60s") + + oasEncNumber, err := getEncryptionKeyNumber(oc, `encryption-key-openshift-apiserver-[^ ]*`) + o.Expect(err).NotTo(o.HaveOccurred()) + kasEncNumber, err := getEncryptionKeyNumber(oc, `encryption-key-openshift-kube-apiserver-[^ ]*`) + o.Expect(err).NotTo(o.HaveOccurred()) + + oldOASEncSecretName := "encryption-key-openshift-apiserver-" + strconv.Itoa(oasEncNumber) + oldKASEncSecretName := "encryption-key-openshift-kube-apiserver-" + strconv.Itoa(kasEncNumber) + + g.By("3. delete current encryption-key-* secrets from openshift-config-managed") + for _, item := range []string{oldOASEncSecretName, oldKASEncSecretName} { + e2e.Logf("removing finalizers from secret %s", item) + err := oc.WithoutNamespace().Run("patch").Args( + "secret", item, "-n", "openshift-config-managed", + `-p={"metadata":{"finalizers":null}}`).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + e2e.Logf("deleting secret %s", item) + err = oc.WithoutNamespace().Run("delete").Args( + "secret", item, "-n", "openshift-config-managed").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + newOASEncSecretName := "encryption-key-openshift-apiserver-" + strconv.Itoa(oasEncNumber+1) + newKASEncSecretName := "encryption-key-openshift-kube-apiserver-" + strconv.Itoa(kasEncNumber+1) + + g.By("4. wait for new encryption-key-* secrets to appear (up to 10 minutes)") + // Use an explicit timeout context derived from the spec context so Ginkgo cancellation still works + pollCtx2, cancel2 := context.WithTimeout(ctx, 10*time.Minute) + defer cancel2() + + retryCount2 := 0 + errKey := wait.PollUntilContextTimeout(pollCtx2, 6*time.Second, 10*time.Minute, false, + func(pollCtx context.Context) (bool, error) { + _, err := oc.WithoutNamespace().Run("get").Args( + "secrets", newOASEncSecretName, newKASEncSecretName, + "-n", "openshift-config-managed").Output() + if err != nil { + retryCount2++ + // Only log every 10th attempt (once per minute) to reduce noise + if retryCount2%10 == 1 { + e2e.Logf("waiting for new encryption-key-* secrets (attempt %d)", retryCount2) + } + return false, nil + } + e2e.Logf("new encryption-key-* secrets found after %d attempts", retryCount2) + return true, nil + }) + o.Expect(errKey).NotTo(o.HaveOccurred(), + "new encryption key secrets %s, %s not found after 10 minutes", newOASEncSecretName, newKASEncSecretName) + + g.By("5. wait for encryption migration to complete for both components") + completedOAS, errOAS := waitEncryptionKeyMigration(ctx, oc, newOASEncSecretName) + o.Expect(errOAS).NotTo(o.HaveOccurred(), + "encryption key migration did not complete for %s", newOASEncSecretName) + o.Expect(completedOAS).To(o.BeTrue()) + + completedKAS, errKAS := waitEncryptionKeyMigration(ctx, oc, newKASEncSecretName) + o.Expect(errKAS).NotTo(o.HaveOccurred(), + "encryption key migration did not complete for %s", newKASEncSecretName) + o.Expect(completedKAS).To(o.BeTrue()) + }) +}) + +// ---- helper functions for Operators / Certs tests ---- + +// parseAndCheckPEMs decodes all PEM-encoded certificates in data and asserts each one is +// currently valid (NotBefore in the past, NotAfter in the future). +func parseAndCheckPEMs(data []byte, source string) { + rest := data + count := 0 + for { + block, r := pem.Decode(rest) + if block == nil { + break + } + rest = r + if block.Type != "CERTIFICATE" { + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + e2e.Logf(" [%s] failed to parse certificate block: %v", source, err) + continue + } + count++ + now := time.Now() + e2e.Logf(" [%s] cert #%d NotBefore=%s NotAfter=%s", + source, count, + cert.NotBefore.Format(time.RFC3339), cert.NotAfter.Format(time.RFC3339)) + o.Expect(cert.NotAfter.After(now)).To(o.BeTrue(), + "certificate %q from %s has expired (NotAfter=%s)", cert.Subject.CommonName, source, cert.NotAfter) + o.Expect(cert.NotBefore.Before(now)).To(o.BeTrue(), + "certificate %q from %s is not yet valid (NotBefore=%s)", cert.Subject.CommonName, source, cert.NotBefore) + } + e2e.Logf(" [%s] checked %d certificates", source, count) +} + +// certInfo holds the Subject and Issuer DN strings from a TLS leaf certificate. +type certInfo struct { + Subject string + Issuer string +} + +// getServerCertInfo opens a TLS connection to fqdn:port using caPath as the trusted root CA +// and returns the leaf certificate's Subject and Issuer DNs. +func getServerCertInfo(fqdn, port, caPath string) (*certInfo, error) { + caData, err := os.ReadFile(caPath) + if err != nil { + return nil, fmt.Errorf("failed to read CA file %s: %w", caPath, err) + } + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(caData) + + conn, err := tls.Dial("tcp", fmt.Sprintf("%s:%s", fqdn, port), + &tls.Config{RootCAs: pool, ServerName: fqdn}) + if err != nil { + return nil, err + } + defer conn.Close() + + certs := conn.ConnectionState().PeerCertificates + if len(certs) == 0 { + return nil, fmt.Errorf("no peer certificates returned by %s:%s", fqdn, port) + } + return &certInfo{ + Subject: certs[0].Subject.String(), + Issuer: certs[0].Issuer.String(), + }, nil +} + +// getAPIServerFQDNAndPort returns the external API server hostname and port from the +// cluster's infrastructure config. +func getAPIServerFQDNAndPort(ctx context.Context, oc *exutil.CLI) (string, string) { + infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get( + ctx, "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + rawURL := infra.Status.APIServerURL + u, err := url.Parse(rawURL) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to parse API server URL %q", rawURL) + + host := u.Hostname() + port := u.Port() + if port == "" { + port = "443" + } + return host, port +} + +// waitCoBecomes polls the named ClusterOperator until all entries in conditions match +// (e.g. {"Available": "True", "Progressing": "False"}) or the timeout elapses. +func waitCoBecomes(ctx context.Context, oc *exutil.CLI, coName string, timeoutSec int, conditions map[string]string) error { + lastLogTime := time.Time{} + logInterval := 30 * time.Second + return wait.PollUntilContextTimeout( + ctx, 5*time.Second, time.Duration(timeoutSec)*time.Second, false, + func(pollCtx context.Context) (bool, error) { + co, err := oc.AdminConfigClient().ConfigV1().ClusterOperators().Get( + pollCtx, coName, metav1.GetOptions{}) + if err != nil { + e2e.Logf("failed to get ClusterOperator %s: %v — retrying", coName, err) + return false, nil + } + + // Build a map of observed conditions + observed := make(map[string]string) + for _, cond := range co.Status.Conditions { + observed[string(cond.Type)] = string(cond.Status) + } + + // Verify all requested conditions are present and match + for condType, want := range conditions { + got, found := observed[condType] + if !found || got != want { + // Only log every 30 seconds to reduce verbosity + now := time.Now() + if now.Sub(lastLogTime) >= logInterval { + if !found { + e2e.Logf("ClusterOperator %s: condition %s not found (want %s)", coName, condType, want) + } else { + e2e.Logf("ClusterOperator %s: condition %s=%s (want %s)", coName, condType, got, want) + } + lastLogTime = now + } + return false, nil + } + } + return true, nil + }) +} + +// getEncryptionPrefix returns the first 30 bytes (as a string) of the etcd value stored at +// etcdPath. For an encrypted cluster this prefix will contain the encryption scheme identifier, +// e.g. "k8s:enc:aescbc:v1:1:". +func getEncryptionPrefix(ctx context.Context, oc *exutil.CLI, etcdPath string) (string, error) { + pods, err := oc.AsAdmin().KubeClient().CoreV1().Pods("openshift-etcd").List( + ctx, metav1.ListOptions{LabelSelector: "app=etcd"}) + if err != nil { + return "", fmt.Errorf("failed to list etcd pods: %w", err) + } + if len(pods.Items) == 0 { + return "", fmt.Errorf("no etcd pods found in openshift-etcd") + } + podName := pods.Items[0].Name + + out, err := oc.AsAdmin().Run("exec").Args( + "-n", "openshift-etcd", podName, "-c", "etcdctl", "--", + "etcdctl", "get", etcdPath, "--prefix", "--limit=1", "--print-value-only", + ).Output() + if err != nil { + return "", fmt.Errorf("etcdctl get %s failed: %w", etcdPath, err) + } + // Trim to the ASCII encryption prefix (at most 30 chars) so the output is safe to compare. + if len(out) > 30 { + out = out[:30] + } + return out, nil +} + +// getEncryptionKeyNumber lists secrets in openshift-config-managed whose names match +// pattern and returns the highest numeric suffix found. For example, if secrets +// "encryption-key-openshift-apiserver-3" and "-4" exist, it returns 4. +func getEncryptionKeyNumber(oc *exutil.CLI, pattern string) (int, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return 0, fmt.Errorf("invalid pattern %q: %w", pattern, err) + } + return getEncryptionKeyNumberWithRegex(oc, re) +} + +// getEncryptionKeyNumberWithRegex is like getEncryptionKeyNumber but accepts a pre-compiled +// regex for efficiency when called repeatedly in polling loops. +func getEncryptionKeyNumberWithRegex(oc *exutil.CLI, re *regexp.Regexp) (int, error) { + out, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "secrets", "-n", "openshift-config-managed", + "-o=jsonpath={.items[*].metadata.name}", + ).Output() + if err != nil { + return 0, fmt.Errorf("failed to list secrets in openshift-config-managed: %w", err) + } + + maxNum := 0 + found := false + for _, name := range strings.Fields(out) { + if !re.MatchString(name) { + continue + } + parts := strings.Split(name, "-") + if len(parts) == 0 { + continue + } + n, err := strconv.Atoi(parts[len(parts)-1]) + if err == nil { + found = true + if n > maxNum { + maxNum = n + } + } + } + if !found { + return 0, fmt.Errorf("no encryption key secrets matching pattern %q found", re.String()) + } + return maxNum, nil +} + +// waitEncryptionKeyMigration polls the named secret in openshift-config-managed until the +// "encryption.apiserver.operator.openshift.io/migrated-resources" annotation is non-empty, +// indicating the encryption key migration has completed (up to 30 minutes). +func waitEncryptionKeyMigration(ctx context.Context, oc *exutil.CLI, secretName string) (bool, error) { + const migrationAnnotation = "encryption.apiserver.operator.openshift.io/migrated-resources" + err := wait.PollUntilContextTimeout( + ctx, 30*time.Second, 30*time.Minute, false, + func(pollCtx context.Context) (bool, error) { + secret, err := oc.AsAdmin().KubeClient().CoreV1().Secrets("openshift-config-managed").Get( + pollCtx, secretName, metav1.GetOptions{}) + if err != nil { + e2e.Logf("failed to get secret %s: %v — retrying", secretName, err) + return false, nil + } + if v := secret.Annotations[migrationAnnotation]; v != "" { + e2e.Logf("encryption migration complete for %s: %s", secretName, v) + return true, nil + } + e2e.Logf("waiting for migration to complete for secret %s", secretName) + return false, nil + }) + if err != nil { + return false, err + } + return true, nil +} + +// ensureEncryptionEnabled checks if etcd encryption is enabled, and if not, enables it with aescbc. +// Returns the encryption type and a cleanup function that should be deferred. +// The cleanup function will restore encryption to identity (disabled) if it was originally disabled. +func ensureEncryptionEnabled(ctx context.Context, oc *exutil.CLI) (encryptionType string, cleanup func(), err error) { + currentType, err := oc.WithoutNamespace().Run("get").Args( + "apiserver/cluster", "-o=jsonpath={.spec.encryption.type}").Output() + if err != nil { + return "", nil, fmt.Errorf("failed to get encryption type: %w", err) + } + + wasEnabled := (currentType == "aescbc" || currentType == "aesgcm") + if wasEnabled { + e2e.Logf("etcd encryption already enabled with type: %s", currentType) + return currentType, func() {}, nil + } + + // Encryption not enabled - enable it + e2e.Logf("etcd encryption is not enabled (current type: %s), enabling aescbc encryption", currentType) + + err = oc.WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", "--type=merge", + "-p", `{"spec":{"encryption":{"type":"aescbc"}}}`).Execute() + if err != nil { + return "", nil, fmt.Errorf("failed to enable encryption: %w", err) + } + + // Wait for kube-apiserver operator to start progressing + err = waitCoBecomes(ctx, oc, "kube-apiserver", 300, map[string]string{"Progressing": "True"}) + if err != nil { + return "", nil, fmt.Errorf("kube-apiserver operator did not start progressing within 300s: %w", err) + } + + // Wait for kube-apiserver to stabilize + e2e.Logf("waiting for kube-apiserver operator to stabilize (≤1800s)") + err = waitCoBecomes(ctx, oc, "kube-apiserver", 1800, map[string]string{ + "Available": "True", + "Progressing": "False", + "Degraded": "False", + }) + if err != nil { + return "", nil, fmt.Errorf("kube-apiserver operator did not stabilize within 1800s: %w", err) + } + + // Wait for openshift-apiserver to stabilize + e2e.Logf("waiting for openshift-apiserver operator to stabilize (≤1800s)") + err = waitCoBecomes(ctx, oc, "openshift-apiserver", 1800, map[string]string{ + "Available": "True", + "Progressing": "False", + "Degraded": "False", + }) + if err != nil { + return "", nil, fmt.Errorf("openshift-apiserver operator did not stabilize within 1800s: %w", err) + } + + e2e.Logf("etcd encryption successfully enabled with type: aescbc") + + // Return cleanup function that restores to identity + cleanupFunc := func() { + e2e.Logf("restoring encryption to identity (disabled)") + _ = oc.WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", "--type=merge", + "-p", `{"spec":{"encryption":{"type":"identity"}}}`).Execute() + } + + return "aescbc", cleanupFunc, nil +} + +// buildEncryptionKeySecretName constructs the encryption key secret name from prefix and number. +func buildEncryptionKeySecretName(prefix string, keyNumber int) string { + return prefix + strconv.Itoa(keyNumber) +} From 3e93804a03f84c83231554d7d732eb08465bc387 Mon Sep 17 00:00:00 2001 From: Yamunadevi Shanmugam Date: Thu, 2 Jul 2026 13:19:38 +0530 Subject: [PATCH 4/4] refactor(tests): improve code quality Extract helpers, fix security, prevent panics Signed-off-by: Yamunadevi Shanmugam --- test/extended/apiserver/tls.go | 822 +++---------------------- test/extended/cluster/audit.go | 23 +- test/extended/operators/certs.go | 69 +++ test/extended/prometheus/prometheus.go | 10 +- test/extended/util/tls.go | 374 +++++++++++ 5 files changed, 556 insertions(+), 742 deletions(-) create mode 100644 test/extended/util/tls.go diff --git a/test/extended/apiserver/tls.go b/test/extended/apiserver/tls.go index cc99f6b34ddb..15c34df576b5 100644 --- a/test/extended/apiserver/tls.go +++ b/test/extended/apiserver/tls.go @@ -3,19 +3,12 @@ package apiserver import ( "context" "crypto/tls" - "crypto/x509" "encoding/base64" "encoding/json" - "encoding/pem" "fmt" - "io" - "math/rand" - "net/url" "os" "os/exec" "path/filepath" - "regexp" - "strconv" "strings" "time" @@ -35,7 +28,7 @@ const ( namespace = "apiserver-tls-test" // Logging frequency constants for polling loops to reduce noise - logEveryNAttemptsKeyRotation = 12 // Every 12 attempts * 5s = once per minute + logEveryNAttemptsKeyRotation = 12 // Every 12 attempts * 5s = once per minute logEveryNAttemptsSecretRecreate = 20 // Every 20 attempts * 3s = once per minute logEveryNAttemptsKeyAppear = 10 // Every 10 attempts * 6s = once per minute @@ -104,13 +97,13 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer]", func() { g.By("Verifying TLS behavior for core control plane components") for _, target := range targets { g.By(fmt.Sprintf("Checking %s/%s on port %s", target.namespace, target.name, target.port)) - err = forwardPortAndExecute(target.name, target.namespace, target.port, - func(port int) error { return checkTLSConnection(port, tlsShouldWork, tlsShouldNotWork) }) + err = exutil.ForwardPortAndExecute(target.name, target.namespace, target.port, + func(port int) error { return exutil.CheckTLSConnection(port, tlsShouldWork, tlsShouldNotWork) }) o.Expect(err).NotTo(o.HaveOccurred()) } g.By("Checking etcd's TLS behavior") - err = forwardPortAndExecute("etcd", "openshift-etcd", "2379", func(port int) error { + err = exutil.ForwardPortAndExecute("etcd", "openshift-etcd", "2379", func(port int) error { conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldWork) if err != nil { if !strings.Contains(err.Error(), "remote error: tls: bad certificate") { @@ -147,7 +140,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer]", func() { } g.By("Verifying TLS version and cipher behavior via port-forward to apiserver") - err = forwardPortAndExecute("apiserver", "openshift-kube-apiserver", "443", func(port int) error { + err = exutil.ForwardPortAndExecute("apiserver", "openshift-kube-apiserver", "443", func(port int) error { host := fmt.Sprintf("localhost:%d", port) t.Logf("Testing TLS versions and ciphers against %s", host) @@ -212,165 +205,14 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer]", func() { }) }) -func forwardPortAndExecute(serviceName, namespace, remotePort string, toExecute func(localPort int) error) error { - var err error - for i := 0; i < 3; i++ { - if err = func() error { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - localPort := rand.Intn(65534-1025) + 1025 - args := []string{"port-forward", fmt.Sprintf("svc/%s", serviceName), fmt.Sprintf("%d:%s", localPort, remotePort), "-n", namespace} - - cmd := exec.CommandContext(ctx, "oc", args...) - stdout, stderr, err := e2e.StartCmdAndStreamOutput(cmd) - if err != nil { - return err - } - defer stdout.Close() - defer stderr.Close() - defer e2e.TryKill(cmd) - - // Read and discard port-forward output to avoid logging sensitive cluster metadata - _ = readPartialFrom(stdout, 1024) - return toExecute(localPort) - }(); err == nil { - return nil - } else { - e2e.Logf("failed to start oc port-forward command or test: %v", err) - time.Sleep(2 * time.Second) - } - } - return err -} - -func readPartialFrom(r io.Reader, maxBytes int) string { - buf := make([]byte, maxBytes) - n, err := r.Read(buf) - if err != nil && err != io.EOF { - return fmt.Sprintf("error reading: %v", err) - } - return string(buf[:n]) -} - -func checkTLSConnection(port int, tlsShouldWork, tlsShouldNotWork *tls.Config) error { - conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldWork) - if err != nil { - return fmt.Errorf("should work: %w", err) - } - err = conn.Close() - if err != nil { - return fmt.Errorf("failed to close connection: %w", err) - } - - conn, err = tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldNotWork) - if err == nil { - return fmt.Errorf("should not work: connection unexpectedly succeeded, closing conn status: %v", conn.Close()) - } - if !strings.Contains(err.Error(), "protocol version") && - !strings.Contains(err.Error(), "no supported versions satisfy") && - !strings.Contains(err.Error(), "handshake failure") { - return fmt.Errorf("should not work: got error, but not a TLS version mismatch: %w", err) - } - return nil -} - var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Operators / Certs", func() { defer g.GinkgoRecover() var oc = exutil.NewCLIWithoutNamespace("apiserver-certs") - // Verify TLS secrets in openshift-kube-apiserver and CA bundles in - // openshift-config-managed are all valid (non-expired, not-yet-valid) PEM certificates. - g.It("[OTP] should have valid TLS certificates for authentication and encryption between API server components [apigroup:config.openshift.io]", - ote.Informing(), func(ctx g.SpecContext) { - g.By("1) check TLS secrets in openshift-kube-apiserver") - ns := "openshift-kube-apiserver" - e2e.Logf("==================================== OpenShift TLS Secrets Verification ====================================") - - secretsJSON, err := oc.AsAdmin().WithoutNamespace(). - Run("get").Args("secret", "-n", ns, "-ojson").Output() - o.Expect(err).NotTo(o.HaveOccurred()) - - var secretList struct { - Items []struct { - Type string `json:"type"` - Data map[string]string `json:"data"` - Metadata struct { - Name string `json:"name"` - } `json:"metadata"` - } `json:"items"` - } - o.Expect(json.Unmarshal([]byte(secretsJSON), &secretList)).NotTo(o.HaveOccurred(), - "failed to parse oc secret JSON output") - - for _, s := range secretList.Items { - if s.Type != "kubernetes.io/tls" { - continue - } - name := s.Metadata.Name - rawcrt, ok := s.Data["tls.crt"] - if !ok || rawcrt == "" { - e2e.Logf(" %s/%s — no tls.crt found, skipping", ns, name) - continue - } - decoded, decErr := base64.StdEncoding.DecodeString(rawcrt) - if decErr != nil { - decoded, decErr = base64.RawStdEncoding.DecodeString(rawcrt) - o.Expect(decErr).NotTo(o.HaveOccurred(), - "failed to base64-decode tls.crt in secret %s/%s", ns, name) - } - e2e.Logf(" checking secret %s/%s", ns, name) - parseAndCheckPEMs(decoded, fmt.Sprintf("%s/%s", ns, name)) - } - - g.By("2) check CA bundle configmaps in openshift-config-managed") - e2e.Logf("==================================== OpenShift CA Bundles Verification ====================================") - ns = "openshift-config-managed" - cms := []struct{ name, key string }{ - {"kube-apiserver-server-ca", "ca-bundle.crt"}, - {"kube-apiserver-client-ca", "ca-bundle.crt"}, - {"kube-root-ca.crt", "ca.crt"}, - {"trusted-ca-bundle", "ca-bundle.crt"}, - {"service-ca", "ca-bundle.crt"}, - } - for _, cm := range cms { - cmJSON, cmErr := oc.AsAdmin().WithoutNamespace(). - Run("get").Args("cm", cm.name, "-n", ns, "-ojson").Output() - o.Expect(cmErr).NotTo(o.HaveOccurred(), - "failed to get configmap %s/%s", ns, cm.name) - - var cmObj struct { - Data map[string]string `json:"data"` - } - err := json.Unmarshal([]byte(cmJSON), &cmObj) - o.Expect(err).NotTo(o.HaveOccurred(), - "failed to parse configmap JSON for %s/%s", ns, cm.name) - - val, ok := cmObj.Data[cm.key] - if !ok || val == "" { - // fallback: search any key containing "ca" or "crt" - found := false - for k, v := range cmObj.Data { - if kl := strings.ToLower(k); (strings.Contains(kl, "ca") || strings.Contains(kl, "crt")) && v != "" { - e2e.Logf(" checking configmap %s/%s (key=%s)", ns, cm.name, k) - parseAndCheckPEMs([]byte(v), fmt.Sprintf("%s/%s(%s)", ns, cm.name, k)) - found = true - break - } - } - o.Expect(found).To(o.BeTrue(), - "no CA/crt key found in configmap %s/%s", ns, cm.name) - continue - } - e2e.Logf(" checking configmap %s/%s (key=%s)", ns, cm.name, cm.key) - parseAndCheckPEMs([]byte(val), fmt.Sprintf("%s/%s(%s)", ns, cm.name, cm.key)) - } - e2e.Logf("Certificate verification complete.") - }) - // Add a custom TLS certificate to the cluster API server, verify it is served, // then restore the original configuration. - g.It("[OTP] should support adding a custom TLS certificate for the cluster API [Disruptive][Slow][apigroup:config.openshift.io]", + g.It("[OTP][OCP-70020] should support adding a custom TLS certificate for the cluster API [Disruptive][Slow][Timeout:50m][apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { isHyperShift, err := exutil.IsHypershift(ctx, oc.AdminConfigClient()) o.Expect(err).NotTo(o.HaveOccurred()) @@ -380,7 +222,7 @@ var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Operators / Certs" tmpdir := g.GinkgoT().TempDir() // Use unique secret name to avoid conflicts with pre-existing secrets - testSecretName := fmt.Sprintf("custom-api-cert-test-%d", time.Now().Unix()) + testSecretName := fmt.Sprintf("custom-api-cert-test-%d", time.Now().UnixNano()) var ( originKubeconfig = os.Getenv("KUBECONFIG") @@ -415,21 +257,32 @@ var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Operators / Certs" defer func() { g.By("restoring cluster to original state") - _, _ = oc.AsAdmin().WithoutNamespace().Run("patch").Args( + // Use detached context to ensure cleanup runs even if spec context is cancelled + _, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Minute) + defer cancel() + + patchOut, patchErr := oc.AsAdmin().WithoutNamespace().Run("patch").Args( "apiserver/cluster", "--type=merge", "-p", patchToRecover).Output() - if _, cpErr := exec.Command("bash", "-c", - fmt.Sprintf("cp %s %s", originKubeconfigBkp, originKubeconfig)).Output(); cpErr != nil { - e2e.Logf("warning: failed to restore kubeconfig: %v", cpErr) + if patchErr != nil { + e2e.Logf("error restoring apiserver/cluster: %v, output: %s", patchErr, patchOut) } + o.Expect(patchErr).NotTo(o.HaveOccurred(), "failed to restore apiserver/cluster configuration") + + restoreData, readErr := os.ReadFile(originKubeconfigBkp) + o.Expect(readErr).NotTo(o.HaveOccurred(), "failed to read kubeconfig backup") + writeErr := os.WriteFile(originKubeconfig, restoreData, 0600) + o.Expect(writeErr).NotTo(o.HaveOccurred(), "failed to restore kubeconfig") + err := oc.AsAdmin().WithoutNamespace().Run("adm").Args("wait-for-stable-cluster").Execute() - o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(err).NotTo(o.HaveOccurred(), "cluster did not stabilize after restore") + // Only delete the test-specific secret, not any pre-existing secrets err = oc.AsAdmin().WithoutNamespace().Run("delete").Args( "secret", testSecretName, "-n", "openshift-config", "--ignore-not-found").Execute() - o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to delete test secret") }() - fqdnName, port := getAPIServerFQDNAndPort(ctx, oc) + fqdnName, port := exutil.GetAPIServerFQDNAndPort(ctx, oc) g.By("1. take a backup of the original kubeconfig") origData, err := os.ReadFile(originKubeconfig) @@ -458,19 +311,19 @@ var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Operators / Certs" o.Expect(err).NotTo(o.HaveOccurred(), "failed to write CA file") g.By("3. generate a new CA key, CA cert, server key, and server cert with SAN") - // Generate CA private key - out, err := exec.Command("openssl", "genrsa", "-out", caKeypem, "2048").CombinedOutput() - o.Expect(err).NotTo(o.HaveOccurred(), "openssl genrsa CA failed: %s", out) + // Generate CA private key using ECDSA P-256 instead of RSA-2048 + out, err := exec.CommandContext(ctx, "openssl", "ecparam", "-genkey", "-name", "prime256v1", "-out", caKeypem).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "openssl ecparam CA failed: %s", out) // Generate CA certificate - out, err = exec.Command("openssl", "req", "-x509", "-new", "-nodes", + out, err = exec.CommandContext(ctx, "openssl", "req", "-x509", "-new", "-nodes", "-key", caKeypem, "-days", "100000", "-out", caCertpem, "-subj", fmt.Sprintf("/CN=%s_ca", cnBase)).CombinedOutput() o.Expect(err).NotTo(o.HaveOccurred(), "openssl req CA failed: %s", out) - // Generate server private key - out, err = exec.Command("openssl", "genrsa", "-out", serverKeypem, "2048").CombinedOutput() - o.Expect(err).NotTo(o.HaveOccurred(), "openssl genrsa server failed: %s", out) + // Generate server private key using ECDSA P-256 + out, err = exec.CommandContext(ctx, "openssl", "ecparam", "-genkey", "-name", "prime256v1", "-out", serverKeypem).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "openssl ecparam server failed: %s", out) serverconfContent := fmt.Sprintf(`[req] req_extensions = v3_req distinguished_name = req_distinguished_name @@ -485,14 +338,14 @@ DNS.1 = %s`, fqdnName) o.Expect(os.WriteFile(serverconf, []byte(serverconfContent), 0644)).NotTo(o.HaveOccurred()) // Generate server CSR with SAN - out, err = exec.Command("openssl", "req", "-new", + out, err = exec.CommandContext(ctx, "openssl", "req", "-new", "-key", serverKeypem, "-out", serverWithSANcsr, "-subj", fmt.Sprintf("/CN=%s_server", cnBase), "-config", serverconf).CombinedOutput() o.Expect(err).NotTo(o.HaveOccurred(), "openssl req server CSR failed: %s", out) // Sign server certificate with CA - out, err = exec.Command("openssl", "x509", "-req", + out, err = exec.CommandContext(ctx, "openssl", "x509", "-req", "-in", serverWithSANcsr, "-CA", caCertpem, "-CAkey", caKeypem, "-CAcreateserial", "-out", serverCertWithSAN, "-days", "100000", "-extensions", "v3_req", "-extfile", serverconf).CombinedOutput() @@ -528,11 +381,11 @@ DNS.1 = %s`, fqdnName) o.Expect(err).NotTo(o.HaveOccurred()) g.By("7. wait for kube-apiserver operator to start progressing (≤300s)") - err = waitCoBecomes(ctx, oc, "kube-apiserver", 300, map[string]string{"Progressing": "True"}) + err = exutil.WaitCoBecomes(ctx, oc, "kube-apiserver", 300, map[string]string{"Progressing": "True"}) o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not start progressing within 300s") e2e.Logf("waiting for kube-apiserver operator to become stable (≤1500s)") - err = waitCoBecomes(ctx, oc, "kube-apiserver", 1500, map[string]string{ + err = exutil.WaitCoBecomes(ctx, oc, "kube-apiserver", 1500, map[string]string{ "Available": "True", "Progressing": "False", "Degraded": "False", @@ -540,245 +393,53 @@ DNS.1 = %s`, fqdnName) o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not stabilise within 1500s") g.By("8. validate that the custom certificate is now served by the API server") - certDetails, err := getServerCertInfo(fqdnName, port, caCertpem) - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(certDetails.Subject).To(o.ContainSubstring("CN=kas-test-cert_server")) - o.Expect(certDetails.Issuer).To(o.ContainSubstring("CN=kas-test-cert_ca")) - - g.By("9. validate the original CA no longer verifies the new certificate") - _, err = getServerCertInfo(fqdnName, port, originCA) - o.Expect(err).To(o.HaveOccurred(), "original CA should not verify the new custom certificate") - }) - - // Force encryption key rotation for the etcd datastore by patching both - // openshiftapiserver and kubeapiserver with an unsupportedConfigOverride, then verify - // that new encryption keys are generated and the resources are re-encrypted. - g.It("[OTP] should force etcd encryption key rotation and verify resources are re-encrypted [Disruptive][Slow][apigroup:config.openshift.io]", - ote.Informing(), func(ctx g.SpecContext) { - g.By("1. check that the cluster has etcd encryption enabled") - encryptionType, err := oc.WithoutNamespace().Run("get").Args( - "apiserver/cluster", "-o=jsonpath={.spec.encryption.type}").Output() - o.Expect(err).NotTo(o.HaveOccurred()) - - encryptionWasEnabled := (encryptionType == "aescbc" || encryptionType == "aesgcm") - if !encryptionWasEnabled { - e2e.Logf("etcd encryption is not enabled (current type: %s), enabling aescbc encryption", encryptionType) - - // Enable encryption - g.By("1a. enabling etcd encryption with aescbc") - err = oc.WithoutNamespace().Run("patch").Args( - "apiserver", "cluster", "--type=merge", - "-p", `{"spec":{"encryption":{"type":"aescbc"}}}`).Execute() - o.Expect(err).NotTo(o.HaveOccurred()) - - // Wait for encryption to be enabled and migration to complete - g.By("1b. waiting for kube-apiserver operator to process encryption enablement (≤300s)") - err = waitCoBecomes(ctx, oc, "kube-apiserver", 300, map[string]string{"Progressing": "True"}) - o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not start progressing within 300s") - - e2e.Logf("waiting for kube-apiserver operator to stabilize (≤1800s)") - err = waitCoBecomes(ctx, oc, "kube-apiserver", 1800, map[string]string{ - "Available": "True", - "Progressing": "False", - "Degraded": "False", - }) - o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not stabilize within 1800s") - - // Wait for openshift-apiserver as well - g.By("1c. waiting for openshift-apiserver operator to stabilize (≤1800s)") - err = waitCoBecomes(ctx, oc, "openshift-apiserver", 1800, map[string]string{ - "Available": "True", - "Progressing": "False", - "Degraded": "False", - }) - o.Expect(err).NotTo(o.HaveOccurred(), "openshift-apiserver operator did not stabilize within 1800s") - - encryptionType = "aescbc" - e2e.Logf("etcd encryption successfully enabled with type: %s", encryptionType) - - // Cleanup: restore to identity (no encryption) at the end - defer func() { - e2e.Logf("restoring encryption to identity (disabled)") - _ = oc.WithoutNamespace().Run("patch").Args( - "apiserver", "cluster", "--type=merge", - "-p", `{"spec":{"encryption":{"type":"identity"}}}`).Execute() - }() - } else { - e2e.Logf("etcd encryption type: %s", encryptionType) - } - - g.By("2. record current encryption prefixes") - oasEncValPrefix1, err := getEncryptionPrefix(ctx, oc, "/openshift.io/routes") - o.Expect(err).NotTo(o.HaveOccurred(), "failed to get encryption prefix for routes") - e2e.Logf("openshift-apiserver encryption prefix recorded before rotation") - - kasEncValPrefix1, err := getEncryptionPrefix(ctx, oc, "/kubernetes.io/secrets") - o.Expect(err).NotTo(o.HaveOccurred(), "failed to get encryption prefix for secrets") - e2e.Logf("kube-apiserver encryption prefix recorded before rotation") - - // Record current highest key numbers before rotation - oasEncNumberBefore, err := getEncryptionKeyNumber(oc, `encryption-key-openshift-apiserver-[^ ]*`) - o.Expect(err).NotTo(o.HaveOccurred()) - kasEncNumberBefore, err := getEncryptionKeyNumber(oc, `encryption-key-openshift-kube-apiserver-[^ ]*`) - o.Expect(err).NotTo(o.HaveOccurred()) - e2e.Logf("encryption keys before rotation: openshift-apiserver=%d, kube-apiserver=%d", - oasEncNumberBefore, kasEncNumberBefore) - - t := time.Now().Format(time.RFC3339) - patchYamlToRestore := `[{"op":"replace","path":"/spec/unsupportedConfigOverrides","value":null}]` - patchYaml := ` -spec: - unsupportedConfigOverrides: - encryption: - reason: force OAS rotation ` + t - - for i, kind := range []string{"openshiftapiserver", "kubeapiserver"} { - defer func(k string) { - e2e.Logf("restoring %s/cluster unsupportedConfigOverrides", k) - _ = oc.WithoutNamespace().Run("patch").Args( - k, "cluster", "--type=json", "-p", patchYamlToRestore).Execute() - }(kind) - g.By(fmt.Sprintf("3.%d) force %s encryption key rotation", i+1, kind)) - err := oc.WithoutNamespace().Run("patch").Args( - kind, "cluster", "--type=merge", "-p", patchYaml).Execute() - o.Expect(err).NotTo(o.HaveOccurred()) - } - - g.By("4. wait for new encryption key secrets to appear (up to 15 minutes)") - // Use an explicit timeout context derived from the spec context so Ginkgo cancellation still works - // Increased to 15 minutes because kube-apiserver operator can be throttled and take 12+ minutes - pollCtx, cancel := context.WithTimeout(ctx, 15*time.Minute) - defer cancel() - - // Pre-compile regexes outside the polling loop for efficiency - oasPattern := regexp.MustCompile(`encryption-key-openshift-apiserver-[^ ]*`) - kasPattern := regexp.MustCompile(`encryption-key-openshift-kube-apiserver-[^ ]*`) - - var newOASEncNumber, newKASEncNumber int - retryCount := 0 - errKey := wait.PollUntilContextTimeout(pollCtx, 5*time.Second, 15*time.Minute, false, + // Poll for the custom certificate to be served (may take time for all apiservers to reload) + var certDetails *exutil.CertInfo + pollErr := wait.PollUntilContextTimeout(ctx, 10*time.Second, 5*time.Minute, false, func(pollCtx context.Context) (bool, error) { - // Dynamically check for new keys instead of pre-calculating expected numbers - currentOAS, err := getEncryptionKeyNumberWithRegex(oc, oasPattern) + certDetails, err = exutil.GetServerCertInfo(pollCtx, fqdnName, port, caCertpem) if err != nil { + e2e.Logf("custom certificate not yet served, retrying: %v", err) return false, nil } - currentKAS, err := getEncryptionKeyNumberWithRegex(oc, kasPattern) - if err != nil { - return false, nil - } - - // Check if both operators created new keys (number increased) - if currentOAS > oasEncNumberBefore && currentKAS > kasEncNumberBefore { - newOASEncNumber = currentOAS - newKASEncNumber = currentKAS - e2e.Logf("new encryption keys detected after %d attempts: openshift-apiserver=%d, kube-apiserver=%d", - retryCount, newOASEncNumber, newKASEncNumber) - return true, nil - } - - retryCount++ - // Only log every 12th attempt (once per minute) to reduce noise - if retryCount%12 == 1 { - e2e.Logf("waiting for new encryption keys (attempt %d): openshift-apiserver=%d (want >%d), kube-apiserver=%d (want >%d)", - retryCount, currentOAS, oasEncNumberBefore, currentKAS, kasEncNumberBefore) - } - return false, nil + return true, nil }) + o.Expect(pollErr).NotTo(o.HaveOccurred(), "custom certificate was not served within 5 minutes") + o.Expect(certDetails.Subject).To(o.ContainSubstring("CN=kas-test-cert_server")) + o.Expect(certDetails.Issuer).To(o.ContainSubstring("CN=kas-test-cert_ca")) - o.Expect(errKey).NotTo(o.HaveOccurred(), - "new encryption keys not created after 15 minutes (openshift-apiserver: want >%d, kube-apiserver: want >%d)", - oasEncNumberBefore, kasEncNumberBefore) - - g.By("5. wait for kube-apiserver encryption migration to complete") - newKASEncSecretName := "encryption-key-openshift-kube-apiserver-" + strconv.Itoa(newKASEncNumber) - completed, err := waitEncryptionKeyMigration(ctx, oc, newKASEncSecretName) - o.Expect(err).NotTo(o.HaveOccurred(), - "encryption key migration did not complete for %s", newKASEncSecretName) - o.Expect(completed).To(o.BeTrue()) - - g.By("6. verify encryption prefixes changed after rotation") - oasEncValPrefix2, err := getEncryptionPrefix(ctx, oc, "/openshift.io/routes") - o.Expect(err).NotTo(o.HaveOccurred(), "failed to get encryption prefix for routes after rotation") - e2e.Logf("openshift-apiserver encryption prefix verified after rotation") - - kasEncValPrefix2, err := getEncryptionPrefix(ctx, oc, "/kubernetes.io/secrets") - o.Expect(err).NotTo(o.HaveOccurred(), "failed to get encryption prefix for secrets after rotation") - e2e.Logf("kube-apiserver encryption prefix verified after rotation") - - o.Expect(oasEncValPrefix2).To(o.ContainSubstring(fmt.Sprintf("k8s:enc:%s:v1", encryptionType))) - o.Expect(kasEncValPrefix2).To(o.ContainSubstring(fmt.Sprintf("k8s:enc:%s:v1", encryptionType))) - o.Expect(oasEncValPrefix2).NotTo(o.Equal(oasEncValPrefix1), - "encryption prefix for routes did not change after key rotation") - o.Expect(kasEncValPrefix2).NotTo(o.Equal(kasEncValPrefix1), - "encryption prefix for secrets did not change after key rotation") + g.By("9. validate the original CA no longer verifies the new certificate") + _, err = exutil.GetServerCertInfo(ctx, fqdnName, port, originCA) + o.Expect(err).To(o.HaveOccurred(), "original CA should not verify the new custom certificate") }) // Delete etcd encryption config and key secrets, then verify the cluster // self-heals by recreating them and completing re-encryption. - g.It("[OTP] should self-recover when etcd encryption configuration secrets are deleted [Disruptive][Slow][apigroup:config.openshift.io]", + g.It("[OTP][OCP-25811] should self-recover when etcd encryption configuration secrets are deleted [Disruptive][Slow][Timeout:50m][apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { - g.By("1. check that the cluster has etcd encryption enabled") - encryptionType, err := oc.WithoutNamespace().Run("get").Args( - "apiserver/cluster", "-o=jsonpath={.spec.encryption.type}").Output() - o.Expect(err).NotTo(o.HaveOccurred()) - - encryptionWasEnabled := (encryptionType == "aescbc" || encryptionType == "aesgcm") - if !encryptionWasEnabled { - e2e.Logf("etcd encryption is not enabled (current type: %s), enabling aescbc encryption", encryptionType) + g.By("1. ensure etcd encryption is enabled") + _, cleanup, err := exutil.EnsureEncryptionEnabled(ctx, oc) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to ensure encryption is enabled") + defer cleanup() - // Enable encryption - g.By("1a. enabling etcd encryption with aescbc") - err = oc.WithoutNamespace().Run("patch").Args( - "apiserver", "cluster", "--type=merge", - "-p", `{"spec":{"encryption":{"type":"aescbc"}}}`).Execute() - o.Expect(err).NotTo(o.HaveOccurred()) - - // Wait for encryption to be enabled and migration to complete - g.By("1b. waiting for kube-apiserver operator to process encryption enablement (≤300s)") - err = waitCoBecomes(ctx, oc, "kube-apiserver", 300, map[string]string{"Progressing": "True"}) - o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not start progressing within 300s") - - e2e.Logf("waiting for kube-apiserver operator to stabilize (≤1800s)") - err = waitCoBecomes(ctx, oc, "kube-apiserver", 1800, map[string]string{ - "Available": "True", - "Progressing": "False", - "Degraded": "False", - }) - o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not stabilize within 1800s") - - // Wait for openshift-apiserver as well - g.By("1c. waiting for openshift-apiserver operator to stabilize (≤1800s)") - err = waitCoBecomes(ctx, oc, "openshift-apiserver", 1800, map[string]string{ - "Available": "True", - "Progressing": "False", - "Degraded": "False", - }) - o.Expect(err).NotTo(o.HaveOccurred(), "openshift-apiserver operator did not stabilize within 1800s") - - encryptionType = "aescbc" - e2e.Logf("etcd encryption successfully enabled with type: %s", encryptionType) - - // Cleanup: restore to identity (no encryption) at the end - defer func() { - e2e.Logf("restoring encryption to identity (disabled)") - _ = oc.WithoutNamespace().Run("patch").Args( - "apiserver", "cluster", "--type=merge", - "-p", `{"spec":{"encryption":{"type":"identity"}}}`).Execute() - }() - } else { - e2e.Logf("etcd encryption type: %s", encryptionType) - } - - uidsOld, err := oc.WithoutNamespace().Run("get").Args( + nameUIDMapJSON, err := oc.WithoutNamespace().Run("get").Args( "secret", "encryption-config-openshift-apiserver", "encryption-config-openshift-kube-apiserver", "-n", "openshift-config-managed", - "-o=jsonpath={.items[*].metadata.uid}", + "-o=jsonpath={range .items[*]}{.metadata.name}{\" \"}{.metadata.uid}{\"\\n\"}{end}", ).Output() o.Expect(err).NotTo(o.HaveOccurred()) + uidsOldMap := make(map[string]string) // map secret name → UID + for _, line := range strings.Split(strings.TrimSpace(nameUIDMapJSON), "\n") { + parts := strings.Fields(line) + if len(parts) == 2 { + uidsOldMap[parts[0]] = parts[1] + } + } + e2e.Logf("original secrets captured: %v", uidsOldMap) + g.By("2. delete encryption-config-* secrets from openshift-config-managed") for _, item := range []string{ "encryption-config-openshift-apiserver", @@ -796,48 +457,54 @@ spec: o.Expect(err).NotTo(o.HaveOccurred()) } - uidsOldSlice := strings.Split(uidsOld, " ") - e2e.Logf("original secret count: %d", len(uidsOldSlice)) - // Use an explicit timeout context derived from the spec context so Ginkgo cancellation still works pollCtx1, cancel1 := context.WithTimeout(ctx, 5*time.Minute) defer cancel1() retryCount1 := 0 - errSecret := wait.PollUntilContextTimeout(pollCtx1, 3*time.Second, 5*time.Minute, false, + errSecret := wait.PollUntilContextCancel(pollCtx1, 3*time.Second, false, func(pollCtx context.Context) (bool, error) { - uidsNew, err := oc.WithoutNamespace().Run("get").Args( + nameUIDMapJSONNew, err := oc.WithoutNamespace().Run("get").Args( "secret", "encryption-config-openshift-apiserver", "encryption-config-openshift-kube-apiserver", "-n", "openshift-config-managed", - "-o=jsonpath={.items[*].metadata.uid}", + "-o=jsonpath={range .items[*]}{.metadata.name}{\" \"}{.metadata.uid}{\"\\n\"}{end}", ).Output() if err != nil { retryCount1++ - // Only log every 20th attempt (once per minute) to reduce noise - if retryCount1%20 == 1 { + // Only log every Nth attempt to reduce noise + if retryCount1%logEveryNAttemptsSecretRecreate == 1 { e2e.Logf("waiting for encryption-config-* secrets to be recreated (attempt %d)", retryCount1) } return false, nil } - uidsNewSlice := strings.Split(uidsNew, " ") - if len(uidsNewSlice) >= 2 && uidsNewSlice[0] != uidsOldSlice[0] && uidsNewSlice[1] != uidsOldSlice[1] { + uidsNewMap := make(map[string]string) + for _, line := range strings.Split(strings.TrimSpace(nameUIDMapJSONNew), "\n") { + parts := strings.Fields(line) + if len(parts) == 2 { + uidsNewMap[parts[0]] = parts[1] + } + } + // Check that both secrets were recreated (different UIDs) + oasRecreated := uidsOldMap["encryption-config-openshift-apiserver"] != uidsNewMap["encryption-config-openshift-apiserver"] + kasRecreated := uidsOldMap["encryption-config-openshift-kube-apiserver"] != uidsNewMap["encryption-config-openshift-kube-apiserver"] + if len(uidsNewMap) >= 2 && oasRecreated && kasRecreated { e2e.Logf("encryption-config-* secrets recreated after %d attempts (UIDs changed)", retryCount1) return true, nil } return false, nil }) o.Expect(errSecret).NotTo(o.HaveOccurred(), - "encryption-config-* secrets were not recreated within 60s") + "encryption-config-* secrets were not recreated within 5 minutes") - oasEncNumber, err := getEncryptionKeyNumber(oc, `encryption-key-openshift-apiserver-[^ ]*`) - o.Expect(err).NotTo(o.HaveOccurred()) - kasEncNumber, err := getEncryptionKeyNumber(oc, `encryption-key-openshift-kube-apiserver-[^ ]*`) - o.Expect(err).NotTo(o.HaveOccurred()) + oasEncNumber, err := exutil.GetEncryptionKeyNumber(oc, `^encryption-key-openshift-apiserver-\d+$`) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get openshift-apiserver encryption key number") + kasEncNumber, err := exutil.GetEncryptionKeyNumber(oc, `^encryption-key-openshift-kube-apiserver-\d+$`) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get kube-apiserver encryption key number") - oldOASEncSecretName := "encryption-key-openshift-apiserver-" + strconv.Itoa(oasEncNumber) - oldKASEncSecretName := "encryption-key-openshift-kube-apiserver-" + strconv.Itoa(kasEncNumber) + oldOASEncSecretName := exutil.BuildEncryptionKeySecretName(encryptionKeyOASPrefix, oasEncNumber) + oldKASEncSecretName := exutil.BuildEncryptionKeySecretName(encryptionKeyKASPrefix, kasEncNumber) g.By("3. delete current encryption-key-* secrets from openshift-config-managed") for _, item := range []string{oldOASEncSecretName, oldKASEncSecretName} { @@ -853,8 +520,8 @@ spec: o.Expect(err).NotTo(o.HaveOccurred()) } - newOASEncSecretName := "encryption-key-openshift-apiserver-" + strconv.Itoa(oasEncNumber+1) - newKASEncSecretName := "encryption-key-openshift-kube-apiserver-" + strconv.Itoa(kasEncNumber+1) + newOASEncSecretName := exutil.BuildEncryptionKeySecretName(encryptionKeyOASPrefix, oasEncNumber+1) + newKASEncSecretName := exutil.BuildEncryptionKeySecretName(encryptionKeyKASPrefix, kasEncNumber+1) g.By("4. wait for new encryption-key-* secrets to appear (up to 10 minutes)") // Use an explicit timeout context derived from the spec context so Ginkgo cancellation still works @@ -862,15 +529,15 @@ spec: defer cancel2() retryCount2 := 0 - errKey := wait.PollUntilContextTimeout(pollCtx2, 6*time.Second, 10*time.Minute, false, + errKey := wait.PollUntilContextCancel(pollCtx2, 6*time.Second, false, func(pollCtx context.Context) (bool, error) { _, err := oc.WithoutNamespace().Run("get").Args( "secrets", newOASEncSecretName, newKASEncSecretName, "-n", "openshift-config-managed").Output() if err != nil { retryCount2++ - // Only log every 10th attempt (once per minute) to reduce noise - if retryCount2%10 == 1 { + // Only log every Nth attempt to reduce noise + if retryCount2%logEveryNAttemptsKeyAppear == 1 { e2e.Logf("waiting for new encryption-key-* secrets (attempt %d)", retryCount2) } return false, nil @@ -882,315 +549,14 @@ spec: "new encryption key secrets %s, %s not found after 10 minutes", newOASEncSecretName, newKASEncSecretName) g.By("5. wait for encryption migration to complete for both components") - completedOAS, errOAS := waitEncryptionKeyMigration(ctx, oc, newOASEncSecretName) + completedOAS, errOAS := exutil.WaitEncryptionKeyMigration(ctx, oc, newOASEncSecretName) o.Expect(errOAS).NotTo(o.HaveOccurred(), "encryption key migration did not complete for %s", newOASEncSecretName) o.Expect(completedOAS).To(o.BeTrue()) - completedKAS, errKAS := waitEncryptionKeyMigration(ctx, oc, newKASEncSecretName) + completedKAS, errKAS := exutil.WaitEncryptionKeyMigration(ctx, oc, newKASEncSecretName) o.Expect(errKAS).NotTo(o.HaveOccurred(), "encryption key migration did not complete for %s", newKASEncSecretName) o.Expect(completedKAS).To(o.BeTrue()) }) }) - -// ---- helper functions for Operators / Certs tests ---- - -// parseAndCheckPEMs decodes all PEM-encoded certificates in data and asserts each one is -// currently valid (NotBefore in the past, NotAfter in the future). -func parseAndCheckPEMs(data []byte, source string) { - rest := data - count := 0 - for { - block, r := pem.Decode(rest) - if block == nil { - break - } - rest = r - if block.Type != "CERTIFICATE" { - continue - } - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - e2e.Logf(" [%s] failed to parse certificate block: %v", source, err) - continue - } - count++ - now := time.Now() - e2e.Logf(" [%s] cert #%d NotBefore=%s NotAfter=%s", - source, count, - cert.NotBefore.Format(time.RFC3339), cert.NotAfter.Format(time.RFC3339)) - o.Expect(cert.NotAfter.After(now)).To(o.BeTrue(), - "certificate %q from %s has expired (NotAfter=%s)", cert.Subject.CommonName, source, cert.NotAfter) - o.Expect(cert.NotBefore.Before(now)).To(o.BeTrue(), - "certificate %q from %s is not yet valid (NotBefore=%s)", cert.Subject.CommonName, source, cert.NotBefore) - } - e2e.Logf(" [%s] checked %d certificates", source, count) -} - -// certInfo holds the Subject and Issuer DN strings from a TLS leaf certificate. -type certInfo struct { - Subject string - Issuer string -} - -// getServerCertInfo opens a TLS connection to fqdn:port using caPath as the trusted root CA -// and returns the leaf certificate's Subject and Issuer DNs. -func getServerCertInfo(fqdn, port, caPath string) (*certInfo, error) { - caData, err := os.ReadFile(caPath) - if err != nil { - return nil, fmt.Errorf("failed to read CA file %s: %w", caPath, err) - } - pool := x509.NewCertPool() - pool.AppendCertsFromPEM(caData) - - conn, err := tls.Dial("tcp", fmt.Sprintf("%s:%s", fqdn, port), - &tls.Config{RootCAs: pool, ServerName: fqdn}) - if err != nil { - return nil, err - } - defer conn.Close() - - certs := conn.ConnectionState().PeerCertificates - if len(certs) == 0 { - return nil, fmt.Errorf("no peer certificates returned by %s:%s", fqdn, port) - } - return &certInfo{ - Subject: certs[0].Subject.String(), - Issuer: certs[0].Issuer.String(), - }, nil -} - -// getAPIServerFQDNAndPort returns the external API server hostname and port from the -// cluster's infrastructure config. -func getAPIServerFQDNAndPort(ctx context.Context, oc *exutil.CLI) (string, string) { - infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get( - ctx, "cluster", metav1.GetOptions{}) - o.Expect(err).NotTo(o.HaveOccurred()) - - rawURL := infra.Status.APIServerURL - u, err := url.Parse(rawURL) - o.Expect(err).NotTo(o.HaveOccurred(), "failed to parse API server URL %q", rawURL) - - host := u.Hostname() - port := u.Port() - if port == "" { - port = "443" - } - return host, port -} - -// waitCoBecomes polls the named ClusterOperator until all entries in conditions match -// (e.g. {"Available": "True", "Progressing": "False"}) or the timeout elapses. -func waitCoBecomes(ctx context.Context, oc *exutil.CLI, coName string, timeoutSec int, conditions map[string]string) error { - lastLogTime := time.Time{} - logInterval := 30 * time.Second - return wait.PollUntilContextTimeout( - ctx, 5*time.Second, time.Duration(timeoutSec)*time.Second, false, - func(pollCtx context.Context) (bool, error) { - co, err := oc.AdminConfigClient().ConfigV1().ClusterOperators().Get( - pollCtx, coName, metav1.GetOptions{}) - if err != nil { - e2e.Logf("failed to get ClusterOperator %s: %v — retrying", coName, err) - return false, nil - } - - // Build a map of observed conditions - observed := make(map[string]string) - for _, cond := range co.Status.Conditions { - observed[string(cond.Type)] = string(cond.Status) - } - - // Verify all requested conditions are present and match - for condType, want := range conditions { - got, found := observed[condType] - if !found || got != want { - // Only log every 30 seconds to reduce verbosity - now := time.Now() - if now.Sub(lastLogTime) >= logInterval { - if !found { - e2e.Logf("ClusterOperator %s: condition %s not found (want %s)", coName, condType, want) - } else { - e2e.Logf("ClusterOperator %s: condition %s=%s (want %s)", coName, condType, got, want) - } - lastLogTime = now - } - return false, nil - } - } - return true, nil - }) -} - -// getEncryptionPrefix returns the first 30 bytes (as a string) of the etcd value stored at -// etcdPath. For an encrypted cluster this prefix will contain the encryption scheme identifier, -// e.g. "k8s:enc:aescbc:v1:1:". -func getEncryptionPrefix(ctx context.Context, oc *exutil.CLI, etcdPath string) (string, error) { - pods, err := oc.AsAdmin().KubeClient().CoreV1().Pods("openshift-etcd").List( - ctx, metav1.ListOptions{LabelSelector: "app=etcd"}) - if err != nil { - return "", fmt.Errorf("failed to list etcd pods: %w", err) - } - if len(pods.Items) == 0 { - return "", fmt.Errorf("no etcd pods found in openshift-etcd") - } - podName := pods.Items[0].Name - - out, err := oc.AsAdmin().Run("exec").Args( - "-n", "openshift-etcd", podName, "-c", "etcdctl", "--", - "etcdctl", "get", etcdPath, "--prefix", "--limit=1", "--print-value-only", - ).Output() - if err != nil { - return "", fmt.Errorf("etcdctl get %s failed: %w", etcdPath, err) - } - // Trim to the ASCII encryption prefix (at most 30 chars) so the output is safe to compare. - if len(out) > 30 { - out = out[:30] - } - return out, nil -} - -// getEncryptionKeyNumber lists secrets in openshift-config-managed whose names match -// pattern and returns the highest numeric suffix found. For example, if secrets -// "encryption-key-openshift-apiserver-3" and "-4" exist, it returns 4. -func getEncryptionKeyNumber(oc *exutil.CLI, pattern string) (int, error) { - re, err := regexp.Compile(pattern) - if err != nil { - return 0, fmt.Errorf("invalid pattern %q: %w", pattern, err) - } - return getEncryptionKeyNumberWithRegex(oc, re) -} - -// getEncryptionKeyNumberWithRegex is like getEncryptionKeyNumber but accepts a pre-compiled -// regex for efficiency when called repeatedly in polling loops. -func getEncryptionKeyNumberWithRegex(oc *exutil.CLI, re *regexp.Regexp) (int, error) { - out, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( - "secrets", "-n", "openshift-config-managed", - "-o=jsonpath={.items[*].metadata.name}", - ).Output() - if err != nil { - return 0, fmt.Errorf("failed to list secrets in openshift-config-managed: %w", err) - } - - maxNum := 0 - found := false - for _, name := range strings.Fields(out) { - if !re.MatchString(name) { - continue - } - parts := strings.Split(name, "-") - if len(parts) == 0 { - continue - } - n, err := strconv.Atoi(parts[len(parts)-1]) - if err == nil { - found = true - if n > maxNum { - maxNum = n - } - } - } - if !found { - return 0, fmt.Errorf("no encryption key secrets matching pattern %q found", re.String()) - } - return maxNum, nil -} - -// waitEncryptionKeyMigration polls the named secret in openshift-config-managed until the -// "encryption.apiserver.operator.openshift.io/migrated-resources" annotation is non-empty, -// indicating the encryption key migration has completed (up to 30 minutes). -func waitEncryptionKeyMigration(ctx context.Context, oc *exutil.CLI, secretName string) (bool, error) { - const migrationAnnotation = "encryption.apiserver.operator.openshift.io/migrated-resources" - err := wait.PollUntilContextTimeout( - ctx, 30*time.Second, 30*time.Minute, false, - func(pollCtx context.Context) (bool, error) { - secret, err := oc.AsAdmin().KubeClient().CoreV1().Secrets("openshift-config-managed").Get( - pollCtx, secretName, metav1.GetOptions{}) - if err != nil { - e2e.Logf("failed to get secret %s: %v — retrying", secretName, err) - return false, nil - } - if v := secret.Annotations[migrationAnnotation]; v != "" { - e2e.Logf("encryption migration complete for %s: %s", secretName, v) - return true, nil - } - e2e.Logf("waiting for migration to complete for secret %s", secretName) - return false, nil - }) - if err != nil { - return false, err - } - return true, nil -} - -// ensureEncryptionEnabled checks if etcd encryption is enabled, and if not, enables it with aescbc. -// Returns the encryption type and a cleanup function that should be deferred. -// The cleanup function will restore encryption to identity (disabled) if it was originally disabled. -func ensureEncryptionEnabled(ctx context.Context, oc *exutil.CLI) (encryptionType string, cleanup func(), err error) { - currentType, err := oc.WithoutNamespace().Run("get").Args( - "apiserver/cluster", "-o=jsonpath={.spec.encryption.type}").Output() - if err != nil { - return "", nil, fmt.Errorf("failed to get encryption type: %w", err) - } - - wasEnabled := (currentType == "aescbc" || currentType == "aesgcm") - if wasEnabled { - e2e.Logf("etcd encryption already enabled with type: %s", currentType) - return currentType, func() {}, nil - } - - // Encryption not enabled - enable it - e2e.Logf("etcd encryption is not enabled (current type: %s), enabling aescbc encryption", currentType) - - err = oc.WithoutNamespace().Run("patch").Args( - "apiserver", "cluster", "--type=merge", - "-p", `{"spec":{"encryption":{"type":"aescbc"}}}`).Execute() - if err != nil { - return "", nil, fmt.Errorf("failed to enable encryption: %w", err) - } - - // Wait for kube-apiserver operator to start progressing - err = waitCoBecomes(ctx, oc, "kube-apiserver", 300, map[string]string{"Progressing": "True"}) - if err != nil { - return "", nil, fmt.Errorf("kube-apiserver operator did not start progressing within 300s: %w", err) - } - - // Wait for kube-apiserver to stabilize - e2e.Logf("waiting for kube-apiserver operator to stabilize (≤1800s)") - err = waitCoBecomes(ctx, oc, "kube-apiserver", 1800, map[string]string{ - "Available": "True", - "Progressing": "False", - "Degraded": "False", - }) - if err != nil { - return "", nil, fmt.Errorf("kube-apiserver operator did not stabilize within 1800s: %w", err) - } - - // Wait for openshift-apiserver to stabilize - e2e.Logf("waiting for openshift-apiserver operator to stabilize (≤1800s)") - err = waitCoBecomes(ctx, oc, "openshift-apiserver", 1800, map[string]string{ - "Available": "True", - "Progressing": "False", - "Degraded": "False", - }) - if err != nil { - return "", nil, fmt.Errorf("openshift-apiserver operator did not stabilize within 1800s: %w", err) - } - - e2e.Logf("etcd encryption successfully enabled with type: aescbc") - - // Return cleanup function that restores to identity - cleanupFunc := func() { - e2e.Logf("restoring encryption to identity (disabled)") - _ = oc.WithoutNamespace().Run("patch").Args( - "apiserver", "cluster", "--type=merge", - "-p", `{"spec":{"encryption":{"type":"identity"}}}`).Execute() - } - - return "aescbc", cleanupFunc, nil -} - -// buildEncryptionKeySecretName constructs the encryption key secret name from prefix and number. -func buildEncryptionKeySecretName(prefix string, keyNumber int) string { - return prefix + strconv.Itoa(keyNumber) -} diff --git a/test/extended/cluster/audit.go b/test/extended/cluster/audit.go index bdc454d3cf0a..077821145deb 100644 --- a/test/extended/cluster/audit.go +++ b/test/extended/cluster/audit.go @@ -106,9 +106,14 @@ func expectAuditLines(f *framework.Framework, expected []auditEvent) { expectations[event] = false } - stream, err := os.Open(filepath.Join(os.Getenv("LOG_DIR"), "audit.log")) - defer stream.Close() + logDir := os.Getenv("LOG_DIR") + if logDir == "" { + framework.Failf("LOG_DIR environment variable must be set") + } + + stream, err := os.Open(filepath.Join(logDir, "audit.log")) framework.ExpectNoError(err, "error opening audit log") + defer stream.Close() scanner := bufio.NewScanner(stream) for scanner.Scan() { line := scanner.Text() @@ -171,7 +176,7 @@ var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Audit", func() { oc := exutil.NewCLIWithoutNamespace("apiserver-audit") // Read current audit profile and assert it is one of the four known valid values. - g.It("[OTP] should report the current audit profile as a known valid value [apigroup:config.openshift.io]", + g.It("[OTP][OCP-24698] should report the current audit profile as a known valid value [apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { apiServer, err := oc.AdminConfigClient().ConfigV1().APIServers().Get( ctx, "cluster", metav1.GetOptions{}) @@ -184,7 +189,7 @@ var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Audit", func() { }) // API server must reject an unrecognised audit profile name at admission time. - g.It("[OTP] should reject an invalid audit profile name in the APIServer configuration [apigroup:config.openshift.io]", + g.It("[OTP][OCP-24699] should reject an invalid audit profile name in the APIServer configuration [apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args( "apiserver", "cluster", @@ -198,7 +203,7 @@ var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Audit", func() { // Custom audit rule must specify a non-empty group name; an empty value should be // rejected at admission time without altering the live configuration. - g.It("[OTP] should reject a custom audit rule with an empty group name [apigroup:config.openshift.io]", + g.It("[OTP][OCP-68361] should reject a custom audit rule with an empty group name [apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args( "apiserver", "cluster", @@ -212,7 +217,7 @@ var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Audit", func() { // Custom audit rule that references an unknown profile name must be rejected at // admission time without altering the live configuration. - g.It("[OTP] should reject a custom audit rule with an invalid profile name [apigroup:config.openshift.io]", + g.It("[OTP][OCP-68362] should reject a custom audit rule with an invalid profile name [apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args( "apiserver", "cluster", @@ -226,7 +231,7 @@ var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Audit", func() { // Every documented profile name (Default, WriteRequestBodies, AllRequestBodies, None) // must be accepted by the API server. A server-side dry-run is used so no rollout is triggered. - g.It("[OTP] should accept all valid audit profile names via server-side dry-run [apigroup:config.openshift.io]", + g.It("[OTP][OCP-68363] should accept all valid audit profile names via server-side dry-run [apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { for _, profile := range []configv1.AuditProfileType{ configv1.DefaultAuditProfileType, @@ -250,7 +255,7 @@ var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Audit", func() { // Audit log files must be present under /var/log/kube-apiserver/ on every master node. // Skipped on HyperShift where the control plane is hosted externally and master nodes are not // directly accessible through node-logs. - g.It("[OTP] should have audit log files present on master nodes [apigroup:config.openshift.io]", + g.It("[OTP][OCP-10592] should have audit log files present on master nodes [apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { if ok, _ := exutil.IsHypershift(ctx, oc.AdminConfigClient()); ok { g.Skip("HyperShift hosts the control plane externally; master node logs are not accessible via node-logs") @@ -278,7 +283,7 @@ var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Audit", func() { // (kind, apiVersion, level, requestURI, verb, user, stage). // Skipped on HyperShift where the control plane is hosted externally and master nodes are not // directly accessible through node-logs. - g.It("[OTP] should write audit log entries in valid JSON format with required fields [apigroup:config.openshift.io]", + g.It("[OTP][OCP-33830] should write audit log entries in valid JSON format with required fields [apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { if ok, _ := exutil.IsHypershift(ctx, oc.AdminConfigClient()); ok { g.Skip("HyperShift hosts the control plane externally; master node logs are not accessible via node-logs") diff --git a/test/extended/operators/certs.go b/test/extended/operators/certs.go index 08325e106048..c08a32bff0d0 100644 --- a/test/extended/operators/certs.go +++ b/test/extended/operators/certs.go @@ -329,6 +329,75 @@ var _ = g.Describe(fmt.Sprintf("[sig-arch][Late][Jira:%q]", "kube-apiserver"), g testresult.Flakef("Errors found: %s", utilerrors.NewAggregate(errs).Error()) } }) + + g.It("all certificates should be valid (not expired, not yet valid)", ote.Informing(), func() { + var errs []error + now := time.Now() + + // Check all in-cluster certificate key pairs + for _, certKeyPair := range actualPKIContent.CertKeyPairs.Items { + // Skip certificates with no actual cert data + if certKeyPair.Spec.CertMetadata.CertIdentifier.CommonName == "" { + continue + } + + // Parse validity period (NotBefore and NotAfter are strings in RFC3339 format) + if certKeyPair.Spec.CertMetadata.NotBefore == "" || certKeyPair.Spec.CertMetadata.NotAfter == "" { + continue + } + + notBefore, err := time.Parse(time.RFC3339, certKeyPair.Spec.CertMetadata.NotBefore) + if err != nil { + continue // Skip if parsing fails + } + notAfter, err := time.Parse(time.RFC3339, certKeyPair.Spec.CertMetadata.NotAfter) + if err != nil { + continue // Skip if parsing fails + } + + // Check if certificate is currently valid + if now.Before(notBefore) { + errs = append(errs, fmt.Errorf("certificate %q is not yet valid (NotBefore=%s, now=%s)", + certKeyPair.Name, notBefore.Format(time.RFC3339), now.Format(time.RFC3339))) + } + if now.After(notAfter) { + errs = append(errs, fmt.Errorf("certificate %q has expired (NotAfter=%s, now=%s)", + certKeyPair.Name, notAfter.Format(time.RFC3339), now.Format(time.RFC3339))) + } + } + + // Check all CA bundles + for _, caBundle := range actualPKIContent.CertificateAuthorityBundles.Items { + for i, ca := range caBundle.Spec.CertificateMetadata { + if ca.NotBefore == "" || ca.NotAfter == "" { + continue + } + + notBefore, err := time.Parse(time.RFC3339, ca.NotBefore) + if err != nil { + continue // Skip if parsing fails + } + notAfter, err := time.Parse(time.RFC3339, ca.NotAfter) + if err != nil { + continue // Skip if parsing fails + } + + // Check if CA certificate is currently valid + if now.Before(notBefore) { + errs = append(errs, fmt.Errorf("CA certificate #%d in bundle %q is not yet valid (NotBefore=%s, now=%s)", + i+1, caBundle.Name, notBefore.Format(time.RFC3339), now.Format(time.RFC3339))) + } + if now.After(notAfter) { + errs = append(errs, fmt.Errorf("CA certificate #%d in bundle %q has expired (NotAfter=%s, now=%s)", + i+1, caBundle.Name, notAfter.Format(time.RFC3339), now.Format(time.RFC3339))) + } + } + } + + if len(errs) > 0 { + testresult.Flakef("Invalid certificates found: %s", utilerrors.NewAggregate(errs).Error()) + } + }) }) func fetchOnDiskCertificates(ctx context.Context, kubeClient kubernetes.Interface, podRESTConfig *rest.Config, nodeList []*corev1.Node, testPullSpec string) (*certgraphapi.PKIList, error) { diff --git a/test/extended/prometheus/prometheus.go b/test/extended/prometheus/prometheus.go index 652399afaf3d..ed6463ca3056 100644 --- a/test/extended/prometheus/prometheus.go +++ b/test/extended/prometheus/prometheus.go @@ -1215,7 +1215,7 @@ var _ = g.Describe("[sig-instrumentation] [Jira:apiserver-auth] Prometheus Alert // ExtremelyHighIndividualControlPlaneCPU alert rule must be defined // and contain the mandatory summary and description annotations. - g.It("[OTP] should have ExtremelyHighIndividualControlPlaneCPU alert rule defined with required annotations [apigroup:config.openshift.io]", + g.It("[OTP][OCP-41899] should have ExtremelyHighIndividualControlPlaneCPU alert rule defined with required annotations [apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) @@ -1234,7 +1234,7 @@ var _ = g.Describe("[sig-instrumentation] [Jira:apiserver-auth] Prometheus Alert // KubeAPIErrorBudgetBurn alert rule must be defined and // carry a non-empty severity label. - g.It("[OTP] should have KubeAPIErrorBudgetBurn alert rule defined with a severity label [apigroup:config.openshift.io]", + g.It("[OTP][OCP-25926] should have KubeAPIErrorBudgetBurn alert rule defined with a severity label [apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) @@ -1254,7 +1254,7 @@ var _ = g.Describe("[sig-instrumentation] [Jira:apiserver-auth] Prometheus Alert // AuditLogError alert rule must be defined, its PromQL // expression must reference audit-related metrics, and it must carry a summary annotation. - g.It("[OTP] should have AuditLogError alert rule defined and referencing audit metrics [apigroup:config.openshift.io]", + g.It("[OTP][OCP-33831] should have AuditLogError alert rule defined and referencing audit metrics [apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) @@ -1275,7 +1275,7 @@ var _ = g.Describe("[sig-instrumentation] [Jira:apiserver-auth] Prometheus Alert // KubeAggregatedAPIErrors alert rule must be defined and // carry a severity label. - g.It("[OTP] should have KubeAggregatedAPIErrors alert rule defined with severity label [apigroup:config.openshift.io]", + g.It("[OTP][OCP-41900] should have KubeAggregatedAPIErrors alert rule defined with severity label [apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) @@ -1293,7 +1293,7 @@ var _ = g.Describe("[sig-instrumentation] [Jira:apiserver-auth] Prometheus Alert // KubeAPIDown alert must be defined with severity=critical // and must not be currently in a firing state. - g.It("[OTP] should have KubeAPIDown alert rule defined and not currently firing [apigroup:config.openshift.io]", + g.It("[OTP][OCP-41901] should have KubeAPIDown alert rule defined and not currently firing [Serial][apigroup:config.openshift.io]", ote.Informing(), func(ctx g.SpecContext) { prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) diff --git a/test/extended/util/tls.go b/test/extended/util/tls.go new file mode 100644 index 000000000000..4ca439a743b7 --- /dev/null +++ b/test/extended/util/tls.go @@ -0,0 +1,374 @@ +package util + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "math/rand" + "net/url" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// CertInfo holds the Subject and Issuer DN strings from a TLS leaf certificate. +type CertInfo struct { + Subject string + Issuer string +} + +// ForwardPortAndExecute forwards a port to a service and executes a function with the local port. +// It retries up to 3 times on failure. +func ForwardPortAndExecute(serviceName, namespace, remotePort string, toExecute func(localPort int) error) error { + var err error + for i := 0; i < 3; i++ { + if err = func() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + localPort := rand.Intn(65534-1025) + 1025 + args := []string{"port-forward", fmt.Sprintf("svc/%s", serviceName), fmt.Sprintf("%d:%s", localPort, remotePort), "-n", namespace} + + cmd := exec.CommandContext(ctx, "oc", args...) + stdout, stderr, err := e2e.StartCmdAndStreamOutput(cmd) + if err != nil { + return err + } + defer stdout.Close() + defer stderr.Close() + defer e2e.TryKill(cmd) + + // Read and discard port-forward output to avoid logging sensitive cluster metadata + _ = ReadPartialFrom(stdout, 1024) + return toExecute(localPort) + }(); err == nil { + return nil + } else { + e2e.Logf("failed to start oc port-forward command or test: %v", err) + time.Sleep(2 * time.Second) + } + } + return err +} + +// ReadPartialFrom reads up to maxBytes from a reader and returns the content as a string. +func ReadPartialFrom(r io.Reader, maxBytes int) string { + buf := make([]byte, maxBytes) + n, err := r.Read(buf) + if err != nil && err != io.EOF { + return fmt.Sprintf("error reading: %v", err) + } + return string(buf[:n]) +} + +// CheckTLSConnection verifies that a TLS connection works with the expected config and fails with the other. +func CheckTLSConnection(port int, tlsShouldWork, tlsShouldNotWork *tls.Config) error { + conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldWork) + if err != nil { + return fmt.Errorf("should work: %w", err) + } + err = conn.Close() + if err != nil { + return fmt.Errorf("failed to close connection: %w", err) + } + + conn, err = tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldNotWork) + if err == nil { + return fmt.Errorf("should not work: connection unexpectedly succeeded, closing conn status: %v", conn.Close()) + } + if !strings.Contains(err.Error(), "protocol version") && + !strings.Contains(err.Error(), "no supported versions satisfy") && + !strings.Contains(err.Error(), "handshake failure") { + return fmt.Errorf("should not work: got error, but not a TLS version mismatch: %w", err) + } + return nil +} + +// GetServerCertInfo opens a TLS connection to fqdn:port using caPath as the trusted root CA +// and returns the leaf certificate's Subject and Issuer DNs. +func GetServerCertInfo(ctx context.Context, fqdn, port, caPath string) (*CertInfo, error) { + caData, err := os.ReadFile(caPath) + if err != nil { + return nil, fmt.Errorf("failed to read CA file %s: %w", caPath, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caData) { + return nil, fmt.Errorf("failed to parse any valid certificates from CA file %s", caPath) + } + + dialer := &tls.Dialer{ + Config: &tls.Config{RootCAs: pool, ServerName: fqdn}, + } + conn, err := dialer.DialContext(ctx, "tcp", fmt.Sprintf("%s:%s", fqdn, port)) + if err != nil { + return nil, err + } + defer conn.Close() + + tlsConn, ok := conn.(*tls.Conn) + if !ok { + return nil, fmt.Errorf("connection is not a TLS connection") + } + + certs := tlsConn.ConnectionState().PeerCertificates + if len(certs) == 0 { + return nil, fmt.Errorf("no peer certificates returned by %s:%s", fqdn, port) + } + return &CertInfo{ + Subject: certs[0].Subject.String(), + Issuer: certs[0].Issuer.String(), + }, nil +} + +// GetAPIServerFQDNAndPort returns the external API server hostname and port from the +// cluster's infrastructure config. +func GetAPIServerFQDNAndPort(ctx context.Context, oc *CLI) (string, string) { + infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get( + ctx, "cluster", metav1.GetOptions{}) + if err != nil { + e2e.Failf("failed to get infrastructure: %v", err) + } + + rawURL := infra.Status.APIServerURL + u, err := url.Parse(rawURL) + if err != nil { + e2e.Failf("failed to parse API server URL %q: %v", rawURL, err) + } + + host := u.Hostname() + port := u.Port() + if port == "" { + port = "443" + } + return host, port +} + +// WaitCoBecomes polls the named ClusterOperator until all entries in conditions match +// (e.g. {"Available": "True", "Progressing": "False"}) or the timeout elapses. +func WaitCoBecomes(ctx context.Context, oc *CLI, coName string, timeoutSec int, conditions map[string]string) error { + lastLogTime := time.Time{} + logInterval := 30 * time.Second + return wait.PollUntilContextTimeout( + ctx, 5*time.Second, time.Duration(timeoutSec)*time.Second, false, + func(pollCtx context.Context) (bool, error) { + co, err := oc.AdminConfigClient().ConfigV1().ClusterOperators().Get( + pollCtx, coName, metav1.GetOptions{}) + if err != nil { + e2e.Logf("failed to get ClusterOperator %s: %v — retrying", coName, err) + return false, nil + } + + // Build a map of observed conditions + observed := make(map[string]string) + for _, cond := range co.Status.Conditions { + observed[string(cond.Type)] = string(cond.Status) + } + + // Verify all requested conditions are present and match + for condType, want := range conditions { + got, found := observed[condType] + if !found || got != want { + // Only log every 30 seconds to reduce verbosity + now := time.Now() + if now.Sub(lastLogTime) >= logInterval { + if !found { + e2e.Logf("ClusterOperator %s: condition %s not found (want %s)", coName, condType, want) + } else { + e2e.Logf("ClusterOperator %s: condition %s=%s (want %s)", coName, condType, got, want) + } + lastLogTime = now + } + return false, nil + } + } + return true, nil + }) +} + +// GetEncryptionPrefix returns the first 30 bytes (as a string) of the etcd value stored at +// etcdPath. For an encrypted cluster this prefix will contain the encryption scheme identifier, +// e.g. "k8s:enc:aescbc:v1:1:". +func GetEncryptionPrefix(ctx context.Context, oc *CLI, etcdPath string) (string, error) { + pods, err := oc.AsAdmin().KubeClient().CoreV1().Pods("openshift-etcd").List( + ctx, metav1.ListOptions{LabelSelector: "app=etcd"}) + if err != nil { + return "", fmt.Errorf("failed to list etcd pods: %w", err) + } + if len(pods.Items) == 0 { + return "", fmt.Errorf("no etcd pods found in openshift-etcd") + } + podName := pods.Items[0].Name + + out, err := oc.AsAdmin().Run("exec").Args( + "-n", "openshift-etcd", podName, "-c", "etcdctl", "--", + "etcdctl", "get", etcdPath, "--prefix", "--limit=1", "--print-value-only", + ).Output() + if err != nil { + return "", fmt.Errorf("etcdctl get %s failed: %w", etcdPath, err) + } + // Trim to the ASCII encryption prefix (at most 30 chars) so the output is safe to compare. + if len(out) > 30 { + out = out[:30] + } + return out, nil +} + +// GetEncryptionKeyNumber lists secrets in openshift-config-managed whose names match +// pattern and returns the highest numeric suffix found. For example, if secrets +// "encryption-key-openshift-apiserver-3" and "-4" exist, it returns 4. +func GetEncryptionKeyNumber(oc *CLI, pattern string) (int, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return 0, fmt.Errorf("invalid pattern %q: %w", pattern, err) + } + return GetEncryptionKeyNumberWithRegex(oc, re) +} + +// GetEncryptionKeyNumberWithRegex is like GetEncryptionKeyNumber but accepts a pre-compiled +// regex for efficiency when called repeatedly in polling loops. +func GetEncryptionKeyNumberWithRegex(oc *CLI, re *regexp.Regexp) (int, error) { + out, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "secrets", "-n", "openshift-config-managed", + "-o=jsonpath={.items[*].metadata.name}", + ).Output() + if err != nil { + return 0, fmt.Errorf("failed to list secrets in openshift-config-managed: %w", err) + } + + maxNum := 0 + found := false + for _, name := range strings.Fields(out) { + if !re.MatchString(name) { + continue + } + parts := strings.Split(name, "-") + if len(parts) == 0 { + continue + } + n, err := strconv.Atoi(parts[len(parts)-1]) + if err == nil { + found = true + if n > maxNum { + maxNum = n + } + } + } + if !found { + return 0, fmt.Errorf("no encryption key secrets matching pattern %q found", re.String()) + } + return maxNum, nil +} + +// WaitEncryptionKeyMigration polls the named secret in openshift-config-managed until the +// "encryption.apiserver.operator.openshift.io/migrated-resources" annotation is non-empty, +// indicating the encryption key migration has completed (up to 30 minutes). +func WaitEncryptionKeyMigration(ctx context.Context, oc *CLI, secretName string) (bool, error) { + const migrationAnnotation = "encryption.apiserver.operator.openshift.io/migrated-resources" + err := wait.PollUntilContextTimeout( + ctx, 30*time.Second, 30*time.Minute, false, + func(pollCtx context.Context) (bool, error) { + secret, err := oc.AsAdmin().KubeClient().CoreV1().Secrets("openshift-config-managed").Get( + pollCtx, secretName, metav1.GetOptions{}) + if err != nil { + e2e.Logf("failed to get secret %s: %v — retrying", secretName, err) + return false, nil + } + if v := secret.Annotations[migrationAnnotation]; v != "" { + e2e.Logf("encryption migration complete for %s: %s", secretName, v) + return true, nil + } + e2e.Logf("waiting for migration to complete for secret %s", secretName) + return false, nil + }) + if err != nil { + return false, err + } + return true, nil +} + +// EnsureEncryptionEnabled checks if etcd encryption is enabled, and if not, enables it with aescbc. +// Returns the encryption type and a cleanup function that should be deferred. +// The cleanup function will restore encryption to identity (disabled) if it was originally disabled. +func EnsureEncryptionEnabled(ctx context.Context, oc *CLI) (encryptionType string, cleanup func(), err error) { + currentType, err := oc.WithoutNamespace().Run("get").Args( + "apiserver/cluster", "-o=jsonpath={.spec.encryption.type}").Output() + if err != nil { + return "", nil, fmt.Errorf("failed to get encryption type: %w", err) + } + + wasEnabled := (currentType == "aescbc" || currentType == "aesgcm") + if wasEnabled { + e2e.Logf("etcd encryption already enabled with type: %s", currentType) + return currentType, func() {}, nil + } + + // Encryption not enabled - enable it + e2e.Logf("etcd encryption is not enabled (current type: %s), enabling aescbc encryption", currentType) + + err = oc.WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", "--type=merge", + "-p", `{"spec":{"encryption":{"type":"aescbc"}}}`).Execute() + if err != nil { + return "", nil, fmt.Errorf("failed to enable encryption: %w", err) + } + + // Wait for kube-apiserver operator to start progressing + err = WaitCoBecomes(ctx, oc, "kube-apiserver", 300, map[string]string{"Progressing": "True"}) + if err != nil { + return "", nil, fmt.Errorf("kube-apiserver operator did not start progressing within 300s: %w", err) + } + + // Wait for kube-apiserver to stabilize + e2e.Logf("waiting for kube-apiserver operator to stabilize (≤1800s)") + err = WaitCoBecomes(ctx, oc, "kube-apiserver", 1800, map[string]string{ + "Available": "True", + "Progressing": "False", + "Degraded": "False", + }) + if err != nil { + return "", nil, fmt.Errorf("kube-apiserver operator did not stabilize within 1800s: %w", err) + } + + // Wait for openshift-apiserver to stabilize + e2e.Logf("waiting for openshift-apiserver operator to stabilize (≤1800s)") + err = WaitCoBecomes(ctx, oc, "openshift-apiserver", 1800, map[string]string{ + "Available": "True", + "Progressing": "False", + "Degraded": "False", + }) + if err != nil { + return "", nil, fmt.Errorf("openshift-apiserver operator did not stabilize within 1800s: %w", err) + } + + e2e.Logf("etcd encryption successfully enabled with type: aescbc") + + // Return cleanup function that restores to identity + cleanupFunc := func() { + e2e.Logf("restoring encryption to identity (disabled)") + // Use detached context to ensure cleanup runs even if spec context is cancelled + _, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + patchErr := oc.WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", "--type=merge", + "-p", `{"spec":{"encryption":{"type":"identity"}}}`).Execute() + if patchErr != nil { + e2e.Logf("error restoring encryption to identity: %v", patchErr) + } + } + + return "aescbc", cleanupFunc, nil +} + +// BuildEncryptionKeySecretName constructs the encryption key secret name from prefix and number. +func BuildEncryptionKeySecretName(prefix string, keyNumber int) string { + return prefix + strconv.Itoa(keyNumber) +}