diff --git a/test/extended/apiserver/tls.go b/test/extended/apiserver/tls.go index 58ac08e1524d..15c34df576b5 100644 --- a/test/extended/apiserver/tls.go +++ b/test/extended/apiserver/tls.go @@ -3,16 +3,20 @@ package apiserver import ( "context" "crypto/tls" + "encoding/base64" + "encoding/json" "fmt" - "io" - "math/rand" + "os" "os/exec" + "path/filepath" "strings" "time" g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" + ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" e2e "k8s.io/kubernetes/test/e2e/framework" configv1 "github.com/openshift/api/config/v1" @@ -22,6 +26,15 @@ import ( const ( namespace = "apiserver-tls-test" + + // Logging frequency constants for polling loops to reduce noise + logEveryNAttemptsKeyRotation = 12 // Every 12 attempts * 5s = once per minute + logEveryNAttemptsSecretRecreate = 20 // Every 20 attempts * 3s = once per minute + logEveryNAttemptsKeyAppear = 10 // Every 10 attempts * 6s = once per minute + + // Encryption key secret name prefixes + encryptionKeyOASPrefix = "encryption-key-openshift-apiserver-" + encryptionKeyKASPrefix = "encryption-key-openshift-kube-apiserver-" ) // This test only checks whether components are serving the proper TLS version based @@ -84,13 +97,13 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer]", func() { g.By("Verifying TLS behavior for core control plane components") for _, target := range targets { g.By(fmt.Sprintf("Checking %s/%s on port %s", target.namespace, target.name, target.port)) - err = forwardPortAndExecute(target.name, target.namespace, target.port, - func(port int) error { return checkTLSConnection(port, tlsShouldWork, tlsShouldNotWork) }) + err = exutil.ForwardPortAndExecute(target.name, target.namespace, target.port, + func(port int) error { return exutil.CheckTLSConnection(port, tlsShouldWork, tlsShouldNotWork) }) o.Expect(err).NotTo(o.HaveOccurred()) } g.By("Checking etcd's TLS behavior") - err = forwardPortAndExecute("etcd", "openshift-etcd", "2379", func(port int) error { + err = exutil.ForwardPortAndExecute("etcd", "openshift-etcd", "2379", func(port int) error { conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldWork) if err != nil { if !strings.Contains(err.Error(), "remote error: tls: bad certificate") { @@ -127,7 +140,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer]", func() { } g.By("Verifying TLS version and cipher behavior via port-forward to apiserver") - err = forwardPortAndExecute("apiserver", "openshift-kube-apiserver", "443", func(port int) error { + err = exutil.ForwardPortAndExecute("apiserver", "openshift-kube-apiserver", "443", func(port int) error { host := fmt.Sprintf("localhost:%d", port) t.Logf("Testing TLS versions and ciphers against %s", host) @@ -192,63 +205,358 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer]", func() { }) }) -func forwardPortAndExecute(serviceName, namespace, remotePort string, toExecute func(localPort int) error) error { - var err error - for i := 0; i < 3; i++ { - if err = func() error { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - localPort := rand.Intn(65534-1025) + 1025 - args := []string{"port-forward", fmt.Sprintf("svc/%s", serviceName), fmt.Sprintf("%d:%s", localPort, remotePort), "-n", namespace} - - cmd := exec.CommandContext(ctx, "oc", args...) - stdout, stderr, err := e2e.StartCmdAndStreamOutput(cmd) - if err != nil { - return err +var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Operators / Certs", func() { + defer g.GinkgoRecover() + + var oc = exutil.NewCLIWithoutNamespace("apiserver-certs") + + // Add a custom TLS certificate to the cluster API server, verify it is served, + // then restore the original configuration. + g.It("[OTP][OCP-70020] should support adding a custom TLS certificate for the cluster API [Disruptive][Slow][Timeout:50m][apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + isHyperShift, err := exutil.IsHypershift(ctx, oc.AdminConfigClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isHyperShift { + g.Skip("custom serving certificates for the API server are managed by HyperShift — skipping") } - defer stdout.Close() - defer stderr.Close() - defer e2e.TryKill(cmd) - e2e.Logf("oc port-forward output: %s", readPartialFrom(stdout, 1024)) - return toExecute(localPort) - }(); err == nil { - return nil - } else { - e2e.Logf("failed to start oc port-forward command or test: %v", err) - time.Sleep(2 * time.Second) - } - } - return err -} - -func readPartialFrom(r io.Reader, maxBytes int) string { - buf := make([]byte, maxBytes) - n, err := r.Read(buf) - if err != nil && err != io.EOF { - return fmt.Sprintf("error reading: %v", err) - } - return string(buf[:n]) -} - -func checkTLSConnection(port int, tlsShouldWork, tlsShouldNotWork *tls.Config) error { - conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldWork) - if err != nil { - return fmt.Errorf("should work: %w", err) - } - err = conn.Close() - if err != nil { - return fmt.Errorf("failed to close connection: %w", err) - } - - conn, err = tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldNotWork) - if err == nil { - return fmt.Errorf("should not work: connection unexpectedly succeeded, closing conn status: %v", conn.Close()) - } - if !strings.Contains(err.Error(), "protocol version") && - !strings.Contains(err.Error(), "no supported versions satisfy") && - !strings.Contains(err.Error(), "handshake failure") { - return fmt.Errorf("should not work: got error, but not a TLS version mismatch: %w", err) - } - return nil -} + tmpdir := g.GinkgoT().TempDir() + // Use unique secret name to avoid conflicts with pre-existing secrets + testSecretName := fmt.Sprintf("custom-api-cert-test-%d", time.Now().UnixNano()) + + var ( + originKubeconfig = os.Getenv("KUBECONFIG") + originKubeconfigBkp = filepath.Join(tmpdir, "kubeconfig.origin.bkp") + originCA = filepath.Join(tmpdir, "certificate-authority-data-origin.crt") + newCA = filepath.Join(tmpdir, "certificate-authority-data-origin-new.crt") + cnBase = "kas-test-cert" + caKeypem = filepath.Join(tmpdir, "caKey.pem") + caCertpem = filepath.Join(tmpdir, "caCert.pem") + serverKeypem = filepath.Join(tmpdir, "serverKey.pem") + serverconf = filepath.Join(tmpdir, "server.conf") + serverWithSANcsr = filepath.Join(tmpdir, "serverWithSAN.csr") + serverCertWithSAN = filepath.Join(tmpdir, "serverCertWithSAN.pem") + ) + + // Snapshot original apiserver configuration before modification + g.By("0. snapshot original apiserver configuration") + origAPIServer, err := oc.AdminConfigClient().ConfigV1().APIServers().Get( + ctx, "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get original apiserver config") + + origNamedCerts := origAPIServer.Spec.ServingCerts.NamedCertificates + var patchToRecover string + if origNamedCerts == nil || len(origNamedCerts) == 0 { + patchToRecover = `{"spec":{"servingCerts": {"namedCertificates": null}}}` + } else { + // Marshal original namedCertificates to restore exact state + certsJSON, err := json.Marshal(origNamedCerts) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to marshal original namedCertificates") + patchToRecover = fmt.Sprintf(`{"spec":{"servingCerts": {"namedCertificates": %s}}}`, string(certsJSON)) + } + + defer func() { + g.By("restoring cluster to original state") + // Use detached context to ensure cleanup runs even if spec context is cancelled + _, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Minute) + defer cancel() + + patchOut, patchErr := oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "apiserver/cluster", "--type=merge", "-p", patchToRecover).Output() + if patchErr != nil { + e2e.Logf("error restoring apiserver/cluster: %v, output: %s", patchErr, patchOut) + } + o.Expect(patchErr).NotTo(o.HaveOccurred(), "failed to restore apiserver/cluster configuration") + + restoreData, readErr := os.ReadFile(originKubeconfigBkp) + o.Expect(readErr).NotTo(o.HaveOccurred(), "failed to read kubeconfig backup") + writeErr := os.WriteFile(originKubeconfig, restoreData, 0600) + o.Expect(writeErr).NotTo(o.HaveOccurred(), "failed to restore kubeconfig") + + err := oc.AsAdmin().WithoutNamespace().Run("adm").Args("wait-for-stable-cluster").Execute() + o.Expect(err).NotTo(o.HaveOccurred(), "cluster did not stabilize after restore") + + // Only delete the test-specific secret, not any pre-existing secrets + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args( + "secret", testSecretName, "-n", "openshift-config", "--ignore-not-found").Execute() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to delete test secret") + }() + + fqdnName, port := exutil.GetAPIServerFQDNAndPort(ctx, oc) + + g.By("1. take a backup of the original kubeconfig") + origData, err := os.ReadFile(originKubeconfig) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(os.WriteFile(originKubeconfigBkp, origData, 0600)).NotTo(o.HaveOccurred()) + + g.By("2. extract the original CA certificate from kubeconfig") + kubeconfigData, err := os.ReadFile(originKubeconfig) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to read kubeconfig") + + // Parse kubeconfig to extract certificate-authority-data + var caDataBase64 string + for _, line := range strings.Split(string(kubeconfigData), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "certificate-authority-data:") { + caDataBase64 = strings.TrimSpace(strings.TrimPrefix(line, "certificate-authority-data:")) + break + } + } + o.Expect(caDataBase64).NotTo(o.BeEmpty(), "certificate-authority-data not found in kubeconfig") + + // Decode base64 and write to file + caDataDecoded, err := base64.StdEncoding.DecodeString(caDataBase64) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to decode certificate-authority-data") + err = os.WriteFile(originCA, caDataDecoded, 0600) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to write CA file") + + g.By("3. generate a new CA key, CA cert, server key, and server cert with SAN") + // Generate CA private key using ECDSA P-256 instead of RSA-2048 + out, err := exec.CommandContext(ctx, "openssl", "ecparam", "-genkey", "-name", "prime256v1", "-out", caKeypem).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "openssl ecparam CA failed: %s", out) + + // Generate CA certificate + out, err = exec.CommandContext(ctx, "openssl", "req", "-x509", "-new", "-nodes", + "-key", caKeypem, "-days", "100000", "-out", caCertpem, + "-subj", fmt.Sprintf("/CN=%s_ca", cnBase)).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "openssl req CA failed: %s", out) + + // Generate server private key using ECDSA P-256 + out, err = exec.CommandContext(ctx, "openssl", "ecparam", "-genkey", "-name", "prime256v1", "-out", serverKeypem).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "openssl ecparam server failed: %s", out) + serverconfContent := fmt.Sprintf(`[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = %s`, fqdnName) + o.Expect(os.WriteFile(serverconf, []byte(serverconfContent), 0644)).NotTo(o.HaveOccurred()) + + // Generate server CSR with SAN + out, err = exec.CommandContext(ctx, "openssl", "req", "-new", + "-key", serverKeypem, "-out", serverWithSANcsr, + "-subj", fmt.Sprintf("/CN=%s_server", cnBase), + "-config", serverconf).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "openssl req server CSR failed: %s", out) + + // Sign server certificate with CA + out, err = exec.CommandContext(ctx, "openssl", "x509", "-req", + "-in", serverWithSANcsr, "-CA", caCertpem, "-CAkey", caKeypem, + "-CAcreateserial", "-out", serverCertWithSAN, + "-days", "100000", "-extensions", "v3_req", "-extfile", serverconf).CombinedOutput() + o.Expect(err).NotTo(o.HaveOccurred(), "openssl x509 sign failed: %s", out) + + g.By("4. create a TLS secret for the custom API certificate") + err = oc.AsAdmin().WithoutNamespace().Run("create").Args( + "secret", "tls", testSecretName, + "--cert="+serverCertWithSAN, "--key="+serverKeypem, + "-n", "openshift-config").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("5. patch apiserver/cluster to use the new named certificate") + patchCmd := fmt.Sprintf( + `{"spec":{"servingCerts": {"namedCertificates": [{"names": ["%s"], "servingCertificate": {"name": "%s"}}]}}}`, + fqdnName, testSecretName) + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "apiserver/cluster", "--type=merge", "-p", patchCmd).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("6. update kubeconfig to include the new CA alongside the original CA") + caCertData, err := os.ReadFile(caCertpem) + o.Expect(err).NotTo(o.HaveOccurred()) + originCAData, err := os.ReadFile(originCA) + o.Expect(err).NotTo(o.HaveOccurred()) + concatenated := append(caCertData, originCAData...) + o.Expect(os.WriteFile(newCA, concatenated, 0644)).NotTo(o.HaveOccurred()) + b64Cert := base64.StdEncoding.EncodeToString(concatenated) + updateKubeconfCmd := fmt.Sprintf( + `sed -i "s/certificate-authority-data: .*/certificate-authority-data: %s/" %s`, + b64Cert, originKubeconfig) + _, err = exec.Command("bash", "-c", updateKubeconfCmd).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("7. wait for kube-apiserver operator to start progressing (≤300s)") + err = exutil.WaitCoBecomes(ctx, oc, "kube-apiserver", 300, map[string]string{"Progressing": "True"}) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not start progressing within 300s") + + e2e.Logf("waiting for kube-apiserver operator to become stable (≤1500s)") + err = exutil.WaitCoBecomes(ctx, oc, "kube-apiserver", 1500, map[string]string{ + "Available": "True", + "Progressing": "False", + "Degraded": "False", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not stabilise within 1500s") + + g.By("8. validate that the custom certificate is now served by the API server") + // Poll for the custom certificate to be served (may take time for all apiservers to reload) + var certDetails *exutil.CertInfo + pollErr := wait.PollUntilContextTimeout(ctx, 10*time.Second, 5*time.Minute, false, + func(pollCtx context.Context) (bool, error) { + certDetails, err = exutil.GetServerCertInfo(pollCtx, fqdnName, port, caCertpem) + if err != nil { + e2e.Logf("custom certificate not yet served, retrying: %v", err) + return false, nil + } + return true, nil + }) + o.Expect(pollErr).NotTo(o.HaveOccurred(), "custom certificate was not served within 5 minutes") + o.Expect(certDetails.Subject).To(o.ContainSubstring("CN=kas-test-cert_server")) + o.Expect(certDetails.Issuer).To(o.ContainSubstring("CN=kas-test-cert_ca")) + + g.By("9. validate the original CA no longer verifies the new certificate") + _, err = exutil.GetServerCertInfo(ctx, fqdnName, port, originCA) + o.Expect(err).To(o.HaveOccurred(), "original CA should not verify the new custom certificate") + }) + + // Delete etcd encryption config and key secrets, then verify the cluster + // self-heals by recreating them and completing re-encryption. + g.It("[OTP][OCP-25811] should self-recover when etcd encryption configuration secrets are deleted [Disruptive][Slow][Timeout:50m][apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + g.By("1. ensure etcd encryption is enabled") + _, cleanup, err := exutil.EnsureEncryptionEnabled(ctx, oc) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to ensure encryption is enabled") + defer cleanup() + + nameUIDMapJSON, err := oc.WithoutNamespace().Run("get").Args( + "secret", + "encryption-config-openshift-apiserver", + "encryption-config-openshift-kube-apiserver", + "-n", "openshift-config-managed", + "-o=jsonpath={range .items[*]}{.metadata.name}{\" \"}{.metadata.uid}{\"\\n\"}{end}", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + uidsOldMap := make(map[string]string) // map secret name → UID + for _, line := range strings.Split(strings.TrimSpace(nameUIDMapJSON), "\n") { + parts := strings.Fields(line) + if len(parts) == 2 { + uidsOldMap[parts[0]] = parts[1] + } + } + e2e.Logf("original secrets captured: %v", uidsOldMap) + + g.By("2. delete encryption-config-* secrets from openshift-config-managed") + for _, item := range []string{ + "encryption-config-openshift-apiserver", + "encryption-config-openshift-kube-apiserver", + } { + e2e.Logf("removing finalizers from secret %s", item) + err := oc.WithoutNamespace().Run("patch").Args( + "secret", item, "-n", "openshift-config-managed", + `-p={"metadata":{"finalizers":null}}`).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + e2e.Logf("deleting secret %s", item) + err = oc.WithoutNamespace().Run("delete").Args( + "secret", item, "-n", "openshift-config-managed").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + // Use an explicit timeout context derived from the spec context so Ginkgo cancellation still works + pollCtx1, cancel1 := context.WithTimeout(ctx, 5*time.Minute) + defer cancel1() + + retryCount1 := 0 + errSecret := wait.PollUntilContextCancel(pollCtx1, 3*time.Second, false, + func(pollCtx context.Context) (bool, error) { + nameUIDMapJSONNew, err := oc.WithoutNamespace().Run("get").Args( + "secret", + "encryption-config-openshift-apiserver", + "encryption-config-openshift-kube-apiserver", + "-n", "openshift-config-managed", + "-o=jsonpath={range .items[*]}{.metadata.name}{\" \"}{.metadata.uid}{\"\\n\"}{end}", + ).Output() + if err != nil { + retryCount1++ + // Only log every Nth attempt to reduce noise + if retryCount1%logEveryNAttemptsSecretRecreate == 1 { + e2e.Logf("waiting for encryption-config-* secrets to be recreated (attempt %d)", retryCount1) + } + return false, nil + } + uidsNewMap := make(map[string]string) + for _, line := range strings.Split(strings.TrimSpace(nameUIDMapJSONNew), "\n") { + parts := strings.Fields(line) + if len(parts) == 2 { + uidsNewMap[parts[0]] = parts[1] + } + } + // Check that both secrets were recreated (different UIDs) + oasRecreated := uidsOldMap["encryption-config-openshift-apiserver"] != uidsNewMap["encryption-config-openshift-apiserver"] + kasRecreated := uidsOldMap["encryption-config-openshift-kube-apiserver"] != uidsNewMap["encryption-config-openshift-kube-apiserver"] + if len(uidsNewMap) >= 2 && oasRecreated && kasRecreated { + e2e.Logf("encryption-config-* secrets recreated after %d attempts (UIDs changed)", retryCount1) + return true, nil + } + return false, nil + }) + o.Expect(errSecret).NotTo(o.HaveOccurred(), + "encryption-config-* secrets were not recreated within 5 minutes") + + oasEncNumber, err := exutil.GetEncryptionKeyNumber(oc, `^encryption-key-openshift-apiserver-\d+$`) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get openshift-apiserver encryption key number") + kasEncNumber, err := exutil.GetEncryptionKeyNumber(oc, `^encryption-key-openshift-kube-apiserver-\d+$`) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get kube-apiserver encryption key number") + + oldOASEncSecretName := exutil.BuildEncryptionKeySecretName(encryptionKeyOASPrefix, oasEncNumber) + oldKASEncSecretName := exutil.BuildEncryptionKeySecretName(encryptionKeyKASPrefix, kasEncNumber) + + g.By("3. delete current encryption-key-* secrets from openshift-config-managed") + for _, item := range []string{oldOASEncSecretName, oldKASEncSecretName} { + e2e.Logf("removing finalizers from secret %s", item) + err := oc.WithoutNamespace().Run("patch").Args( + "secret", item, "-n", "openshift-config-managed", + `-p={"metadata":{"finalizers":null}}`).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + e2e.Logf("deleting secret %s", item) + err = oc.WithoutNamespace().Run("delete").Args( + "secret", item, "-n", "openshift-config-managed").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + newOASEncSecretName := exutil.BuildEncryptionKeySecretName(encryptionKeyOASPrefix, oasEncNumber+1) + newKASEncSecretName := exutil.BuildEncryptionKeySecretName(encryptionKeyKASPrefix, kasEncNumber+1) + + g.By("4. wait for new encryption-key-* secrets to appear (up to 10 minutes)") + // Use an explicit timeout context derived from the spec context so Ginkgo cancellation still works + pollCtx2, cancel2 := context.WithTimeout(ctx, 10*time.Minute) + defer cancel2() + + retryCount2 := 0 + errKey := wait.PollUntilContextCancel(pollCtx2, 6*time.Second, false, + func(pollCtx context.Context) (bool, error) { + _, err := oc.WithoutNamespace().Run("get").Args( + "secrets", newOASEncSecretName, newKASEncSecretName, + "-n", "openshift-config-managed").Output() + if err != nil { + retryCount2++ + // Only log every Nth attempt to reduce noise + if retryCount2%logEveryNAttemptsKeyAppear == 1 { + e2e.Logf("waiting for new encryption-key-* secrets (attempt %d)", retryCount2) + } + return false, nil + } + e2e.Logf("new encryption-key-* secrets found after %d attempts", retryCount2) + return true, nil + }) + o.Expect(errKey).NotTo(o.HaveOccurred(), + "new encryption key secrets %s, %s not found after 10 minutes", newOASEncSecretName, newKASEncSecretName) + + g.By("5. wait for encryption migration to complete for both components") + completedOAS, errOAS := exutil.WaitEncryptionKeyMigration(ctx, oc, newOASEncSecretName) + o.Expect(errOAS).NotTo(o.HaveOccurred(), + "encryption key migration did not complete for %s", newOASEncSecretName) + o.Expect(completedOAS).To(o.BeTrue()) + + completedKAS, errKAS := exutil.WaitEncryptionKeyMigration(ctx, oc, newKASEncSecretName) + o.Expect(errKAS).NotTo(o.HaveOccurred(), + "encryption key migration did not complete for %s", newKASEncSecretName) + o.Expect(completedKAS).To(o.BeTrue()) + }) +}) diff --git a/test/extended/cluster/audit.go b/test/extended/cluster/audit.go index de51025c4810..077821145deb 100644 --- a/test/extended/cluster/audit.go +++ b/test/extended/cluster/audit.go @@ -3,6 +3,7 @@ package cluster import ( "bufio" "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -10,6 +11,10 @@ import ( g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" + ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" + + configv1 "github.com/openshift/api/config/v1" + exutil "github.com/openshift/origin/test/extended/util" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -101,9 +106,14 @@ func expectAuditLines(f *framework.Framework, expected []auditEvent) { expectations[event] = false } - stream, err := os.Open(filepath.Join(os.Getenv("LOG_DIR"), "audit.log")) - defer stream.Close() + logDir := os.Getenv("LOG_DIR") + if logDir == "" { + framework.Failf("LOG_DIR environment variable must be set") + } + + stream, err := os.Open(filepath.Join(logDir, "audit.log")) framework.ExpectNoError(err, "error opening audit log") + defer stream.Close() scanner := bufio.NewScanner(stream) for scanner.Scan() { line := scanner.Text() @@ -152,3 +162,164 @@ func parseAuditLine(line string) (auditEvent, error) { } return event, nil } + +// validAuditProfiles is the set of profile names accepted by the APIServer CR. +var validAuditProfiles = map[configv1.AuditProfileType]struct{}{ + configv1.DefaultAuditProfileType: {}, + configv1.WriteRequestBodiesAuditProfileType: {}, + configv1.AllRequestBodiesAuditProfileType: {}, + configv1.NoneAuditProfileType: {}, + configv1.AuditProfileType(""): {}, // unset is treated as Default +} + +var _ = g.Describe("[sig-api-machinery] [Jira:apiserver-auth] Audit", func() { + oc := exutil.NewCLIWithoutNamespace("apiserver-audit") + + // Read current audit profile and assert it is one of the four known valid values. + g.It("[OTP][OCP-24698] should report the current audit profile as a known valid value [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + apiServer, err := oc.AdminConfigClient().ConfigV1().APIServers().Get( + ctx, "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + _, ok := validAuditProfiles[apiServer.Spec.Audit.Profile] + o.Expect(ok).To(o.BeTrue(), + "unexpected audit profile %q, expected one of Default, WriteRequestBodies, AllRequestBodies, None", + apiServer.Spec.Audit.Profile) + }) + + // API server must reject an unrecognised audit profile name at admission time. + g.It("[OTP][OCP-24699] should reject an invalid audit profile name in the APIServer configuration [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", + "--type=merge", + `--patch={"spec":{"audit":{"profile":"InvalidAuditProfile"}}}`, + "--dry-run=server", + ).Output() + o.Expect(err).To(o.HaveOccurred(), + "expected server-side validation to reject an invalid audit profile name") + }) + + // Custom audit rule must specify a non-empty group name; an empty value should be + // rejected at admission time without altering the live configuration. + g.It("[OTP][OCP-68361] should reject a custom audit rule with an empty group name [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", + "--type=merge", + `--patch={"spec":{"audit":{"customRules":[{"group":"","profile":"Default"}]}}}`, + "--dry-run=server", + ).Output() + o.Expect(err).To(o.HaveOccurred(), + "expected server-side validation to reject a custom audit rule with an empty group name") + }) + + // Custom audit rule that references an unknown profile name must be rejected at + // admission time without altering the live configuration. + g.It("[OTP][OCP-68362] should reject a custom audit rule with an invalid profile name [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", + "--type=merge", + `--patch={"spec":{"audit":{"customRules":[{"group":"system:authenticated","profile":"BadProfile"}]}}}`, + "--dry-run=server", + ).Output() + o.Expect(err).To(o.HaveOccurred(), + "expected server-side validation to reject a custom audit rule with an invalid profile name") + }) + + // Every documented profile name (Default, WriteRequestBodies, AllRequestBodies, None) + // must be accepted by the API server. A server-side dry-run is used so no rollout is triggered. + g.It("[OTP][OCP-68363] should accept all valid audit profile names via server-side dry-run [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + for _, profile := range []configv1.AuditProfileType{ + configv1.DefaultAuditProfileType, + configv1.WriteRequestBodiesAuditProfileType, + configv1.AllRequestBodiesAuditProfileType, + configv1.NoneAuditProfileType, + } { + g.By(fmt.Sprintf("verifying profile %q is accepted by the API server", profile)) + patch := fmt.Sprintf(`{"spec":{"audit":{"profile":%q}}}`, profile) + _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", + "--type=merge", + "--patch="+patch, + "--dry-run=server", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred(), + "expected profile %q to be accepted by the API server", profile) + } + }) + + // Audit log files must be present under /var/log/kube-apiserver/ on every master node. + // Skipped on HyperShift where the control plane is hosted externally and master nodes are not + // directly accessible through node-logs. + g.It("[OTP][OCP-10592] should have audit log files present on master nodes [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + if ok, _ := exutil.IsHypershift(ctx, oc.AdminConfigClient()); ok { + g.Skip("HyperShift hosts the control plane externally; master node logs are not accessible via node-logs") + } + + masters, err := oc.AsAdmin().KubeClient().CoreV1().Nodes().List( + ctx, metav1.ListOptions{LabelSelector: "node-role.kubernetes.io/master"}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(masters.Items).NotTo(o.BeEmpty(), "expected at least one master node") + + for _, master := range masters.Items { + g.By(fmt.Sprintf("checking for audit log files on master node %s", master.Name)) + output, err := oc.AsAdmin().WithoutNamespace().Run("adm").Args( + "node-logs", master.Name, "--path=kube-apiserver/", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred(), + "failed to list kube-apiserver log files on master node %s", master.Name) + o.Expect(output).To(o.MatchRegexp(`audit`), + "expected audit log files to be present on master node %s", master.Name) + } + }) + + // Each line in the kube-apiserver audit log must be valid JSON and contain the + // mandatory fields defined by the audit.k8s.io/v1 event schema + // (kind, apiVersion, level, requestURI, verb, user, stage). + // Skipped on HyperShift where the control plane is hosted externally and master nodes are not + // directly accessible through node-logs. + g.It("[OTP][OCP-33830] should write audit log entries in valid JSON format with required fields [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + if ok, _ := exutil.IsHypershift(ctx, oc.AdminConfigClient()); ok { + g.Skip("HyperShift hosts the control plane externally; master node logs are not accessible via node-logs") + } + + masters, err := oc.AsAdmin().KubeClient().CoreV1().Nodes().List( + ctx, metav1.ListOptions{LabelSelector: "node-role.kubernetes.io/master"}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(masters.Items).NotTo(o.BeEmpty(), "expected at least one master node") + + // Inspect the most recent audit log entries from the first master only to + // keep the test fast while still validating the format is correct. + master := masters.Items[0] + g.By(fmt.Sprintf("reading recent audit log entries from master node %s", master.Name)) + output, err := oc.AsAdmin().WithoutNamespace().Run("adm").Args( + "node-logs", master.Name, "--path=kube-apiserver/audit.log", "--tail=20", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred(), + "failed to read kube-apiserver audit log from master node %s", master.Name) + + checkedLines := 0 + for _, line := range strings.Split(strings.TrimSpace(output), "\n") { + if strings.TrimSpace(line) == "" { + continue + } + var entry map[string]interface{} + o.Expect(json.Unmarshal([]byte(line), &entry)).NotTo(o.HaveOccurred(), + "audit log entry is not valid JSON on node %s: %s", master.Name, line) + + for _, field := range []string{"kind", "apiVersion", "level", "requestURI", "verb", "user", "stage"} { + o.Expect(entry).To(o.HaveKey(field), + "audit log entry on node %s is missing required field %q: %s", master.Name, field, line) + } + checkedLines++ + } + o.Expect(checkedLines).To(o.BeNumerically(">", 0), + "expected at least one non-empty audit log line on node %s", master.Name) + }) +}) diff --git a/test/extended/operators/certs.go b/test/extended/operators/certs.go index 08325e106048..c08a32bff0d0 100644 --- a/test/extended/operators/certs.go +++ b/test/extended/operators/certs.go @@ -329,6 +329,75 @@ var _ = g.Describe(fmt.Sprintf("[sig-arch][Late][Jira:%q]", "kube-apiserver"), g testresult.Flakef("Errors found: %s", utilerrors.NewAggregate(errs).Error()) } }) + + g.It("all certificates should be valid (not expired, not yet valid)", ote.Informing(), func() { + var errs []error + now := time.Now() + + // Check all in-cluster certificate key pairs + for _, certKeyPair := range actualPKIContent.CertKeyPairs.Items { + // Skip certificates with no actual cert data + if certKeyPair.Spec.CertMetadata.CertIdentifier.CommonName == "" { + continue + } + + // Parse validity period (NotBefore and NotAfter are strings in RFC3339 format) + if certKeyPair.Spec.CertMetadata.NotBefore == "" || certKeyPair.Spec.CertMetadata.NotAfter == "" { + continue + } + + notBefore, err := time.Parse(time.RFC3339, certKeyPair.Spec.CertMetadata.NotBefore) + if err != nil { + continue // Skip if parsing fails + } + notAfter, err := time.Parse(time.RFC3339, certKeyPair.Spec.CertMetadata.NotAfter) + if err != nil { + continue // Skip if parsing fails + } + + // Check if certificate is currently valid + if now.Before(notBefore) { + errs = append(errs, fmt.Errorf("certificate %q is not yet valid (NotBefore=%s, now=%s)", + certKeyPair.Name, notBefore.Format(time.RFC3339), now.Format(time.RFC3339))) + } + if now.After(notAfter) { + errs = append(errs, fmt.Errorf("certificate %q has expired (NotAfter=%s, now=%s)", + certKeyPair.Name, notAfter.Format(time.RFC3339), now.Format(time.RFC3339))) + } + } + + // Check all CA bundles + for _, caBundle := range actualPKIContent.CertificateAuthorityBundles.Items { + for i, ca := range caBundle.Spec.CertificateMetadata { + if ca.NotBefore == "" || ca.NotAfter == "" { + continue + } + + notBefore, err := time.Parse(time.RFC3339, ca.NotBefore) + if err != nil { + continue // Skip if parsing fails + } + notAfter, err := time.Parse(time.RFC3339, ca.NotAfter) + if err != nil { + continue // Skip if parsing fails + } + + // Check if CA certificate is currently valid + if now.Before(notBefore) { + errs = append(errs, fmt.Errorf("CA certificate #%d in bundle %q is not yet valid (NotBefore=%s, now=%s)", + i+1, caBundle.Name, notBefore.Format(time.RFC3339), now.Format(time.RFC3339))) + } + if now.After(notAfter) { + errs = append(errs, fmt.Errorf("CA certificate #%d in bundle %q has expired (NotAfter=%s, now=%s)", + i+1, caBundle.Name, notAfter.Format(time.RFC3339), now.Format(time.RFC3339))) + } + } + } + + if len(errs) > 0 { + testresult.Flakef("Invalid certificates found: %s", utilerrors.NewAggregate(errs).Error()) + } + }) }) func fetchOnDiskCertificates(ctx context.Context, kubeClient kubernetes.Interface, podRESTConfig *rest.Config, nodeList []*corev1.Node, testPullSpec string) (*certgraphapi.PKIList, error) { diff --git a/test/extended/prometheus/prometheus.go b/test/extended/prometheus/prometheus.go index 19a6ba5b8837..ed6463ca3056 100644 --- a/test/extended/prometheus/prometheus.go +++ b/test/extended/prometheus/prometheus.go @@ -19,6 +19,7 @@ import ( g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" + ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" promv1 "github.com/prometheus/client_golang/api/prometheus/v1" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" @@ -1196,3 +1197,162 @@ func SkipOperatorHubMetricsCheck(oc *exutil.CLI) bool { } return stdout == "true" } + +// Verify that specific kube-apiserver and audit alert rules are present +// in the cluster's Prometheus instance and are correctly configured. +var _ = g.Describe("[sig-instrumentation] [Jira:apiserver-auth] Prometheus Alerts", func() { + oc := exutil.NewCLIWithoutNamespace("apiserver-alerts") + + g.BeforeEach(func() { + kubeClient, err := kubernetes.NewForConfig(oc.AdminConfig()) + o.Expect(err).NotTo(o.HaveOccurred()) + nsExist, err := exutil.IsNamespaceExist(kubeClient, "openshift-monitoring") + o.Expect(err).NotTo(o.HaveOccurred()) + if !nsExist { + g.Skip("openshift-monitoring namespace does not exist") + } + }) + + // ExtremelyHighIndividualControlPlaneCPU alert rule must be defined + // and contain the mandatory summary and description annotations. + g.It("[OTP][OCP-41899] should have ExtremelyHighIndividualControlPlaneCPU alert rule defined with required annotations [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) + + alertName := "ExtremelyHighIndividualControlPlaneCPU" + g.By(fmt.Sprintf("looking for alert rule %q", alertName)) + rule := lookupAlertingRule(prometheusURL, bearerToken, alertName) + o.Expect(rule).NotTo(o.BeNil(), + "alert rule %q was not found in Prometheus", alertName) + + g.By("verifying required annotations are present") + o.Expect(rule.Annotations).To(o.HaveKey(model.LabelName("summary")), + "alert rule %q is missing 'summary' annotation", alertName) + o.Expect(rule.Annotations).To(o.HaveKey(model.LabelName("description")), + "alert rule %q is missing 'description' annotation", alertName) + }) + + // KubeAPIErrorBudgetBurn alert rule must be defined and + // carry a non-empty severity label. + g.It("[OTP][OCP-25926] should have KubeAPIErrorBudgetBurn alert rule defined with a severity label [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) + + alertName := "KubeAPIErrorBudgetBurn" + g.By(fmt.Sprintf("looking for alert rule %q", alertName)) + rule := lookupAlertingRule(prometheusURL, bearerToken, alertName) + o.Expect(rule).NotTo(o.BeNil(), + "alert rule %q was not found in Prometheus", alertName) + + g.By("verifying severity label is present and non-empty") + severity, ok := rule.Labels["severity"] + o.Expect(ok).To(o.BeTrue(), + "alert rule %q is missing 'severity' label", alertName) + o.Expect(string(severity)).NotTo(o.BeEmpty(), + "alert rule %q has an empty 'severity' label", alertName) + }) + + // AuditLogError alert rule must be defined, its PromQL + // expression must reference audit-related metrics, and it must carry a summary annotation. + g.It("[OTP][OCP-33831] should have AuditLogError alert rule defined and referencing audit metrics [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) + + alertName := "AuditLogError" + g.By(fmt.Sprintf("looking for alert rule %q", alertName)) + rule := lookupAlertingRule(prometheusURL, bearerToken, alertName) + o.Expect(rule).NotTo(o.BeNil(), + "alert rule %q was not found in Prometheus", alertName) + + g.By("verifying the PromQL expression references audit-related metrics") + o.Expect(string(rule.Query)).To(o.MatchRegexp(`audit`), + "alert rule %q query %q does not reference audit metrics", alertName, rule.Query) + + g.By("verifying required annotations are present") + o.Expect(rule.Annotations).To(o.HaveKey(model.LabelName("summary")), + "alert rule %q is missing 'summary' annotation", alertName) + }) + + // KubeAggregatedAPIErrors alert rule must be defined and + // carry a severity label. + g.It("[OTP][OCP-41900] should have KubeAggregatedAPIErrors alert rule defined with severity label [apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) + + alertName := "KubeAggregatedAPIErrors" + g.By(fmt.Sprintf("looking for alert rule %q", alertName)) + rule := lookupAlertingRule(prometheusURL, bearerToken, alertName) + o.Expect(rule).NotTo(o.BeNil(), + "alert rule %q was not found in Prometheus", alertName) + + g.By("verifying severity label is present") + _, ok := rule.Labels["severity"] + o.Expect(ok).To(o.BeTrue(), + "alert rule %q is missing 'severity' label", alertName) + }) + + // KubeAPIDown alert must be defined with severity=critical + // and must not be currently in a firing state. + g.It("[OTP][OCP-41901] should have KubeAPIDown alert rule defined and not currently firing [Serial][apigroup:config.openshift.io]", + ote.Informing(), func(ctx g.SpecContext) { + prometheusURL, bearerToken := mustGetPrometheusURLAndToken(ctx, oc) + + alertName := "KubeAPIDown" + g.By(fmt.Sprintf("looking for alert rule %q", alertName)) + rule := lookupAlertingRule(prometheusURL, bearerToken, alertName) + o.Expect(rule).NotTo(o.BeNil(), + "alert rule %q was not found in Prometheus", alertName) + + g.By("verifying severity=critical label") + severity, ok := rule.Labels["severity"] + o.Expect(ok).To(o.BeTrue(), + "alert rule %q is missing 'severity' label", alertName) + o.Expect(string(severity)).To(o.Equal("critical"), + "alert rule %q expected severity=critical, got %q", alertName, severity) + + g.By(fmt.Sprintf("verifying %q is not currently firing", alertName)) + prometheusClient := oc.NewPrometheusClient(ctx) + err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 2*time.Minute, true, + func(ctx context.Context) (bool, error) { + resp, queryErr := helper.RunQuery(ctx, prometheusClient, + fmt.Sprintf(`ALERTS{alertname=%q,alertstate="firing"}`, alertName)) + if queryErr != nil { + e2e.Logf("prometheus query error (will retry): %v", queryErr) + return false, nil + } + o.Expect(resp.Data.Result).To(o.BeEmpty(), + "alert %q is unexpectedly firing", alertName) + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), + "timed out waiting to confirm %q is not firing", alertName) + }) +}) + +// lookupAlertingRule searches all Prometheus rule groups for a rule with the given +// name and returns it, or nil if not found. +func lookupAlertingRule(prometheusURL, bearerToken, name string) *promv1.AlertingRule { + rules, err := helper.FetchAlertingRules(prometheusURL, bearerToken) + if err != nil { + e2e.Logf("failed to fetch alerting rules: %v", err) + return nil + } + for _, group := range rules { + for i := range group { + if group[i].Name == name { + return &group[i] + } + } + } + return nil +} + +// mustGetPrometheusURLAndToken is a test helper that fetches the Prometheus route URL +// and a service-account bearer token, failing the test immediately on error. +func mustGetPrometheusURLAndToken(ctx g.SpecContext, oc *exutil.CLI) (string, string) { + prometheusURL, err := helper.PrometheusRouteURL(ctx, oc) + o.Expect(err).NotTo(o.HaveOccurred(), "get public URL of prometheus") + bearerToken, err := helper.RequestPrometheusServiceAccountAPIToken(ctx, oc) + o.Expect(err).NotTo(o.HaveOccurred(), "request prometheus service account API token") + return prometheusURL, bearerToken +} diff --git a/test/extended/util/tls.go b/test/extended/util/tls.go new file mode 100644 index 000000000000..4ca439a743b7 --- /dev/null +++ b/test/extended/util/tls.go @@ -0,0 +1,374 @@ +package util + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "math/rand" + "net/url" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// CertInfo holds the Subject and Issuer DN strings from a TLS leaf certificate. +type CertInfo struct { + Subject string + Issuer string +} + +// ForwardPortAndExecute forwards a port to a service and executes a function with the local port. +// It retries up to 3 times on failure. +func ForwardPortAndExecute(serviceName, namespace, remotePort string, toExecute func(localPort int) error) error { + var err error + for i := 0; i < 3; i++ { + if err = func() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + localPort := rand.Intn(65534-1025) + 1025 + args := []string{"port-forward", fmt.Sprintf("svc/%s", serviceName), fmt.Sprintf("%d:%s", localPort, remotePort), "-n", namespace} + + cmd := exec.CommandContext(ctx, "oc", args...) + stdout, stderr, err := e2e.StartCmdAndStreamOutput(cmd) + if err != nil { + return err + } + defer stdout.Close() + defer stderr.Close() + defer e2e.TryKill(cmd) + + // Read and discard port-forward output to avoid logging sensitive cluster metadata + _ = ReadPartialFrom(stdout, 1024) + return toExecute(localPort) + }(); err == nil { + return nil + } else { + e2e.Logf("failed to start oc port-forward command or test: %v", err) + time.Sleep(2 * time.Second) + } + } + return err +} + +// ReadPartialFrom reads up to maxBytes from a reader and returns the content as a string. +func ReadPartialFrom(r io.Reader, maxBytes int) string { + buf := make([]byte, maxBytes) + n, err := r.Read(buf) + if err != nil && err != io.EOF { + return fmt.Sprintf("error reading: %v", err) + } + return string(buf[:n]) +} + +// CheckTLSConnection verifies that a TLS connection works with the expected config and fails with the other. +func CheckTLSConnection(port int, tlsShouldWork, tlsShouldNotWork *tls.Config) error { + conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldWork) + if err != nil { + return fmt.Errorf("should work: %w", err) + } + err = conn.Close() + if err != nil { + return fmt.Errorf("failed to close connection: %w", err) + } + + conn, err = tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldNotWork) + if err == nil { + return fmt.Errorf("should not work: connection unexpectedly succeeded, closing conn status: %v", conn.Close()) + } + if !strings.Contains(err.Error(), "protocol version") && + !strings.Contains(err.Error(), "no supported versions satisfy") && + !strings.Contains(err.Error(), "handshake failure") { + return fmt.Errorf("should not work: got error, but not a TLS version mismatch: %w", err) + } + return nil +} + +// GetServerCertInfo opens a TLS connection to fqdn:port using caPath as the trusted root CA +// and returns the leaf certificate's Subject and Issuer DNs. +func GetServerCertInfo(ctx context.Context, fqdn, port, caPath string) (*CertInfo, error) { + caData, err := os.ReadFile(caPath) + if err != nil { + return nil, fmt.Errorf("failed to read CA file %s: %w", caPath, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caData) { + return nil, fmt.Errorf("failed to parse any valid certificates from CA file %s", caPath) + } + + dialer := &tls.Dialer{ + Config: &tls.Config{RootCAs: pool, ServerName: fqdn}, + } + conn, err := dialer.DialContext(ctx, "tcp", fmt.Sprintf("%s:%s", fqdn, port)) + if err != nil { + return nil, err + } + defer conn.Close() + + tlsConn, ok := conn.(*tls.Conn) + if !ok { + return nil, fmt.Errorf("connection is not a TLS connection") + } + + certs := tlsConn.ConnectionState().PeerCertificates + if len(certs) == 0 { + return nil, fmt.Errorf("no peer certificates returned by %s:%s", fqdn, port) + } + return &CertInfo{ + Subject: certs[0].Subject.String(), + Issuer: certs[0].Issuer.String(), + }, nil +} + +// GetAPIServerFQDNAndPort returns the external API server hostname and port from the +// cluster's infrastructure config. +func GetAPIServerFQDNAndPort(ctx context.Context, oc *CLI) (string, string) { + infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get( + ctx, "cluster", metav1.GetOptions{}) + if err != nil { + e2e.Failf("failed to get infrastructure: %v", err) + } + + rawURL := infra.Status.APIServerURL + u, err := url.Parse(rawURL) + if err != nil { + e2e.Failf("failed to parse API server URL %q: %v", rawURL, err) + } + + host := u.Hostname() + port := u.Port() + if port == "" { + port = "443" + } + return host, port +} + +// WaitCoBecomes polls the named ClusterOperator until all entries in conditions match +// (e.g. {"Available": "True", "Progressing": "False"}) or the timeout elapses. +func WaitCoBecomes(ctx context.Context, oc *CLI, coName string, timeoutSec int, conditions map[string]string) error { + lastLogTime := time.Time{} + logInterval := 30 * time.Second + return wait.PollUntilContextTimeout( + ctx, 5*time.Second, time.Duration(timeoutSec)*time.Second, false, + func(pollCtx context.Context) (bool, error) { + co, err := oc.AdminConfigClient().ConfigV1().ClusterOperators().Get( + pollCtx, coName, metav1.GetOptions{}) + if err != nil { + e2e.Logf("failed to get ClusterOperator %s: %v — retrying", coName, err) + return false, nil + } + + // Build a map of observed conditions + observed := make(map[string]string) + for _, cond := range co.Status.Conditions { + observed[string(cond.Type)] = string(cond.Status) + } + + // Verify all requested conditions are present and match + for condType, want := range conditions { + got, found := observed[condType] + if !found || got != want { + // Only log every 30 seconds to reduce verbosity + now := time.Now() + if now.Sub(lastLogTime) >= logInterval { + if !found { + e2e.Logf("ClusterOperator %s: condition %s not found (want %s)", coName, condType, want) + } else { + e2e.Logf("ClusterOperator %s: condition %s=%s (want %s)", coName, condType, got, want) + } + lastLogTime = now + } + return false, nil + } + } + return true, nil + }) +} + +// GetEncryptionPrefix returns the first 30 bytes (as a string) of the etcd value stored at +// etcdPath. For an encrypted cluster this prefix will contain the encryption scheme identifier, +// e.g. "k8s:enc:aescbc:v1:1:". +func GetEncryptionPrefix(ctx context.Context, oc *CLI, etcdPath string) (string, error) { + pods, err := oc.AsAdmin().KubeClient().CoreV1().Pods("openshift-etcd").List( + ctx, metav1.ListOptions{LabelSelector: "app=etcd"}) + if err != nil { + return "", fmt.Errorf("failed to list etcd pods: %w", err) + } + if len(pods.Items) == 0 { + return "", fmt.Errorf("no etcd pods found in openshift-etcd") + } + podName := pods.Items[0].Name + + out, err := oc.AsAdmin().Run("exec").Args( + "-n", "openshift-etcd", podName, "-c", "etcdctl", "--", + "etcdctl", "get", etcdPath, "--prefix", "--limit=1", "--print-value-only", + ).Output() + if err != nil { + return "", fmt.Errorf("etcdctl get %s failed: %w", etcdPath, err) + } + // Trim to the ASCII encryption prefix (at most 30 chars) so the output is safe to compare. + if len(out) > 30 { + out = out[:30] + } + return out, nil +} + +// GetEncryptionKeyNumber lists secrets in openshift-config-managed whose names match +// pattern and returns the highest numeric suffix found. For example, if secrets +// "encryption-key-openshift-apiserver-3" and "-4" exist, it returns 4. +func GetEncryptionKeyNumber(oc *CLI, pattern string) (int, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return 0, fmt.Errorf("invalid pattern %q: %w", pattern, err) + } + return GetEncryptionKeyNumberWithRegex(oc, re) +} + +// GetEncryptionKeyNumberWithRegex is like GetEncryptionKeyNumber but accepts a pre-compiled +// regex for efficiency when called repeatedly in polling loops. +func GetEncryptionKeyNumberWithRegex(oc *CLI, re *regexp.Regexp) (int, error) { + out, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "secrets", "-n", "openshift-config-managed", + "-o=jsonpath={.items[*].metadata.name}", + ).Output() + if err != nil { + return 0, fmt.Errorf("failed to list secrets in openshift-config-managed: %w", err) + } + + maxNum := 0 + found := false + for _, name := range strings.Fields(out) { + if !re.MatchString(name) { + continue + } + parts := strings.Split(name, "-") + if len(parts) == 0 { + continue + } + n, err := strconv.Atoi(parts[len(parts)-1]) + if err == nil { + found = true + if n > maxNum { + maxNum = n + } + } + } + if !found { + return 0, fmt.Errorf("no encryption key secrets matching pattern %q found", re.String()) + } + return maxNum, nil +} + +// WaitEncryptionKeyMigration polls the named secret in openshift-config-managed until the +// "encryption.apiserver.operator.openshift.io/migrated-resources" annotation is non-empty, +// indicating the encryption key migration has completed (up to 30 minutes). +func WaitEncryptionKeyMigration(ctx context.Context, oc *CLI, secretName string) (bool, error) { + const migrationAnnotation = "encryption.apiserver.operator.openshift.io/migrated-resources" + err := wait.PollUntilContextTimeout( + ctx, 30*time.Second, 30*time.Minute, false, + func(pollCtx context.Context) (bool, error) { + secret, err := oc.AsAdmin().KubeClient().CoreV1().Secrets("openshift-config-managed").Get( + pollCtx, secretName, metav1.GetOptions{}) + if err != nil { + e2e.Logf("failed to get secret %s: %v — retrying", secretName, err) + return false, nil + } + if v := secret.Annotations[migrationAnnotation]; v != "" { + e2e.Logf("encryption migration complete for %s: %s", secretName, v) + return true, nil + } + e2e.Logf("waiting for migration to complete for secret %s", secretName) + return false, nil + }) + if err != nil { + return false, err + } + return true, nil +} + +// EnsureEncryptionEnabled checks if etcd encryption is enabled, and if not, enables it with aescbc. +// Returns the encryption type and a cleanup function that should be deferred. +// The cleanup function will restore encryption to identity (disabled) if it was originally disabled. +func EnsureEncryptionEnabled(ctx context.Context, oc *CLI) (encryptionType string, cleanup func(), err error) { + currentType, err := oc.WithoutNamespace().Run("get").Args( + "apiserver/cluster", "-o=jsonpath={.spec.encryption.type}").Output() + if err != nil { + return "", nil, fmt.Errorf("failed to get encryption type: %w", err) + } + + wasEnabled := (currentType == "aescbc" || currentType == "aesgcm") + if wasEnabled { + e2e.Logf("etcd encryption already enabled with type: %s", currentType) + return currentType, func() {}, nil + } + + // Encryption not enabled - enable it + e2e.Logf("etcd encryption is not enabled (current type: %s), enabling aescbc encryption", currentType) + + err = oc.WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", "--type=merge", + "-p", `{"spec":{"encryption":{"type":"aescbc"}}}`).Execute() + if err != nil { + return "", nil, fmt.Errorf("failed to enable encryption: %w", err) + } + + // Wait for kube-apiserver operator to start progressing + err = WaitCoBecomes(ctx, oc, "kube-apiserver", 300, map[string]string{"Progressing": "True"}) + if err != nil { + return "", nil, fmt.Errorf("kube-apiserver operator did not start progressing within 300s: %w", err) + } + + // Wait for kube-apiserver to stabilize + e2e.Logf("waiting for kube-apiserver operator to stabilize (≤1800s)") + err = WaitCoBecomes(ctx, oc, "kube-apiserver", 1800, map[string]string{ + "Available": "True", + "Progressing": "False", + "Degraded": "False", + }) + if err != nil { + return "", nil, fmt.Errorf("kube-apiserver operator did not stabilize within 1800s: %w", err) + } + + // Wait for openshift-apiserver to stabilize + e2e.Logf("waiting for openshift-apiserver operator to stabilize (≤1800s)") + err = WaitCoBecomes(ctx, oc, "openshift-apiserver", 1800, map[string]string{ + "Available": "True", + "Progressing": "False", + "Degraded": "False", + }) + if err != nil { + return "", nil, fmt.Errorf("openshift-apiserver operator did not stabilize within 1800s: %w", err) + } + + e2e.Logf("etcd encryption successfully enabled with type: aescbc") + + // Return cleanup function that restores to identity + cleanupFunc := func() { + e2e.Logf("restoring encryption to identity (disabled)") + // Use detached context to ensure cleanup runs even if spec context is cancelled + _, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + patchErr := oc.WithoutNamespace().Run("patch").Args( + "apiserver", "cluster", "--type=merge", + "-p", `{"spec":{"encryption":{"type":"identity"}}}`).Execute() + if patchErr != nil { + e2e.Logf("error restoring encryption to identity: %v", patchErr) + } + } + + return "aescbc", cleanupFunc, nil +} + +// BuildEncryptionKeySecretName constructs the encryption key secret name from prefix and number. +func BuildEncryptionKeySecretName(prefix string, keyNumber int) string { + return prefix + strconv.Itoa(keyNumber) +}