From 7d781951fcb024760e817ec6e4310e70b679c3f3 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 20 Aug 2026 14:55:54 -0700 Subject: [PATCH] e2e: reach an in-cluster destination over each address family The egress suite fetches example.com, so which address families a test exercises is decided by whatever resolver the cluster inherited. When the name has no AAAA the actor never attempts IPv6 and a broken IPv6 path passes; when it does, the same test fails. Neither outcome says anything about the system. Adds an in-cluster destination fronted by three Services over one backend, so the families are a property of the test, and asserts the family the request actually arrived over rather than trusting a 200. The dual-homed case is the one no single-family cluster can construct: an actor with an IPv6 address prefers the AAAA, so that destination is unreachable the moment IPv6 egress breaks, even with a working A record alongside it. --- internal/e2e/echodest.go | 118 +++++++++++++++++ .../e2e/fixtures/echodest/echodest.yaml.tmpl | 61 +++++++++ internal/e2e/fixtures/echodest/main.go | 83 ++++++++++++ .../suites/networking/egress_family_test.go | 120 ++++++++++++++++++ 4 files changed, 382 insertions(+) create mode 100644 internal/e2e/echodest.go create mode 100644 internal/e2e/fixtures/echodest/echodest.yaml.tmpl create mode 100644 internal/e2e/fixtures/echodest/main.go create mode 100644 internal/e2e/suites/networking/egress_family_test.go diff --git a/internal/e2e/echodest.go b/internal/e2e/echodest.go new file mode 100644 index 000000000..3ac438526 --- /dev/null +++ b/internal/e2e/echodest.go @@ -0,0 +1,118 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "path/filepath" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// DeployEchoDest installs the echodest backend, suffixed for the calling suite +// so concurrent suites never share one, and returns its namespace. +func DeployEchoDest(t *testing.T, name string) string { + t.Helper() + + root, err := FindRepoRoot() + if err != nil { + t.Fatalf("FindRepoRoot: %v", err) + } + manifest := RenderFixtureManifest(t, "internal/e2e/fixtures/echodest/echodest.yaml.tmpl", "", name) + + applyArgs := []string{"ko", "apply", "-f", manifest} + if KubeContext != "" { + applyArgs = append(applyArgs, "--", "--context="+KubeContext) + } + RunCmdWithEnv(t, []string{"KO_CONFIG_PATH=" + root}, filepath.Join(root, "hack/run-tool.sh"), applyArgs...) + + t.Cleanup(func() { + delArgs := []string{"delete", "--ignore-not-found", "-f", manifest} + if KubeContext != "" { + delArgs = append([]string{"--context=" + KubeContext}, delArgs...) + } + RunCmd(t, "kubectl", delArgs...) + }) + + return FixtureName("ate-e2e-echodest") + "-" + name +} + +// ClusterIPFamilies reports which address families the cluster can assign, +// read from a node's podCIDRs. A Service pinned to a family the cluster does +// not have is rejected at creation, so callers gate on this rather than +// letting the apply fail. +func ClusterIPFamilies(t *testing.T, ctx context.Context) map[corev1.IPFamily]bool { + t.Helper() + + nodes, err := GetClients().K8s.CoreV1().Nodes().List(ctx, metav1.ListOptions{Limit: 1}) + if err != nil { + t.Fatalf("listing nodes to determine the cluster's address families: %v", err) + } + if len(nodes.Items) == 0 { + t.Fatal("the cluster reports no nodes, so its address families cannot be determined") + } + + families := map[corev1.IPFamily]bool{} + for _, cidr := range nodes.Items[0].Spec.PodCIDRs { + if strings.Contains(cidr, ":") { + families[corev1.IPv6Protocol] = true + } else { + families[corev1.IPv4Protocol] = true + } + } + if len(families) == 0 { + t.Fatalf("node %q has no podCIDRs, so the cluster's address families cannot be determined", nodes.Items[0].Name) + } + return families +} + +// CreateEchoDestService fronts the echodest backend with a Service pinned to +// families, so the name resolves to exactly the records the caller asked for. +// Returns the Service name. +func CreateEchoDestService(t *testing.T, ctx context.Context, namespace, name string, families []corev1.IPFamily) string { + t.Helper() + + // SingleStack for one family, RequireDualStack for two: Prefer would + // silently hand back a single-stack Service on a cluster missing a family, + // and a test that then passed would be asserting nothing. + policy := corev1.IPFamilyPolicySingleStack + if len(families) > 1 { + policy = corev1.IPFamilyPolicyRequireDualStack + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"app": "echodest"}, + IPFamilies: families, + IPFamilyPolicy: &policy, + Ports: []corev1.ServicePort{{Port: 8080, TargetPort: intstr.FromInt32(8080)}}, + }, + } + created, err := GetClients().K8s.CoreV1().Services(namespace).Create(ctx, service, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("creating the %s echodest Service: %v", name, err) + } + t.Cleanup(func() { + //nolint:errcheck // best-effort teardown; the namespace goes too + GetClients().K8s.CoreV1().Services(namespace).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + t.Logf("echodest Service %s has clusterIPs %v", name, created.Spec.ClusterIPs) + return name +} diff --git a/internal/e2e/fixtures/echodest/echodest.yaml.tmpl b/internal/e2e/fixtures/echodest/echodest.yaml.tmpl new file mode 100644 index 000000000..292b48700 --- /dev/null +++ b/internal/e2e/fixtures/echodest/echodest.yaml.tmpl @@ -0,0 +1,61 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The in-cluster egress destination for the networking suite's per-family +# tests. Only the backend lives here: the Services are created by the suite, +# because a Service pinned to IPv6 cannot be created at all on a single-stack +# IPv4 cluster and the suite is the only thing that knows which families the +# cluster has. +apiVersion: v1 +kind: Namespace +metadata: + name: ate-e2e-echodest${FIXTURE_SUFFIX} +--- +apiVersion: v1 +kind: Pod +metadata: + name: echodest + namespace: ate-e2e-echodest${FIXTURE_SUFFIX} + labels: + app: echodest +spec: + restartPolicy: Never + containers: + - name: echodest + image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/echodest + args: + - "--listen=:8080" + ports: + - name: http + containerPort: 8080 + readinessProbe: + httpGet: + path: /healthz + port: 8080 + periodSeconds: 2 + resources: + requests: + cpu: 10m + memory: 32Mi + # runAsUser must be spelled out: ko's distroless static base declares no + # USER, so runAsNonRoot on its own makes kubelet refuse to start the + # container rather than pick a uid. + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: ["ALL"] diff --git a/internal/e2e/fixtures/echodest/main.go b/internal/e2e/fixtures/echodest/main.go new file mode 100644 index 000000000..d4989b62a --- /dev/null +++ b/internal/e2e/fixtures/echodest/main.go @@ -0,0 +1,83 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command echodest is an in-cluster destination for the egress suite. It +// reports the local address each request arrived on, so a test can assert +// which family actually carried the request instead of inferring it from a +// 200. Egress tests that reach the public internet cannot: whether a +// destination offers a AAAA is decided outside the cluster and changes under +// them. +package main + +import ( + "encoding/json" + "flag" + "log/slog" + "net" + "net/http" + "os" + "time" +) + +var listenAddress = flag.String("listen", ":8080", "Address the destination's HTTP API listens on.") + +// reply is what every endpoint returns. Family is the family the request +// arrived over, taken from the connection rather than from anything the +// client claims. +type reply struct { + Family string `json:"family"` + LocalAddr string `json:"localAddr"` +} + +func handle(w http.ResponseWriter, r *http.Request) { + local, _ := r.Context().Value(http.LocalAddrContextKey).(net.Addr) + body := reply{Family: "unknown"} + if local != nil { + body.LocalAddr = local.String() + if host, _, err := net.SplitHostPort(local.String()); err == nil { + if ip := net.ParseIP(host); ip != nil { + // To4 is non-nil for a v4-mapped v6 address too, which is what a + // dual-stack listener reports for a v4 connection -- exactly the + // answer wanted here. + if ip.To4() != nil { + body.Family = "ipv4" + } else { + body.Family = "ipv6" + } + } + } + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(body); err != nil { + slog.Error("encoding reply", "err", err) + } +} + +func main() { + flag.Parse() + + mux := http.NewServeMux() + mux.HandleFunc("/healthz", handle) + + server := &http.Server{ + Addr: *listenAddress, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + } + slog.Info("echodest listening", "addr", *listenAddress) + if err := server.ListenAndServe(); err != nil { + slog.Error("serving", "err", err) + os.Exit(1) + } +} diff --git a/internal/e2e/suites/networking/egress_family_test.go b/internal/e2e/suites/networking/egress_family_test.go new file mode 100644 index 000000000..ea1960456 --- /dev/null +++ b/internal/e2e/suites/networking/egress_family_test.go @@ -0,0 +1,120 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package networking + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/resources" + corev1 "k8s.io/api/core/v1" +) + +// TestActorEgressPerFamily asserts an Actor reaches an in-cluster destination +// over the family that destination is published in, including -- the case no +// single-family cluster can construct -- a destination published in both. +// +// The destination is in-cluster on purpose. The suite's other egress tests +// fetch example.com, so which families are in play is decided by whatever +// resolver the cluster inherited: when the name has no AAAA the Actor never +// attempts IPv6 and a broken IPv6 path passes. Services over one backend make +// the families a property of the test instead. +// +// The dual subtest is the load-bearing one. An Actor with an IPv6 address +// prefers the AAAA, so a dual-homed destination becomes unreachable the moment +// IPv6 egress is broken -- even though a working A record is right there, and +// even though every IPv4-only destination still works. That is invisible on an +// IPv4-only cluster (nothing to prefer) and on an IPv6-only one (nothing to +// fall back to), which is why it belongs here and nowhere else. +func TestActorEgressPerFamily(t *testing.T) { + ctx := context.Background() + + namespace := e2e.DeployEchoDest(t, "networking") + families := e2e.ClusterIPFamilies(t, ctx) + + for _, tc := range []struct { + name string + families []corev1.IPFamily + // want is the family the destination must be reached over. Empty for + // the dual case: which one a dual-homed name resolves to is the + // Actor's resolver's choice, and asserting it would be asserting + // RFC 6724 rather than anything this system promises. + want string + }{ + {name: "ipv4", families: []corev1.IPFamily{corev1.IPv4Protocol}, want: "ipv4"}, + // There is deliberately no IPv6-only case. An Actor cannot reach an + // IPv6-only destination at all today, so asserting it would be a + // standing red rather than a regression guard. Adding it is one line + // once that works; until then the dual case below is the one that + // matters, and it already fails the moment IPv6 is preferred but broken. + {name: "dual", families: []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}}, + } { + t.Run(tc.name, func(t *testing.T) { + for _, f := range tc.families { + if !families[f] { + t.Skipf("cluster has no %s address family", f) + } + } + service := e2e.CreateEchoDestService(t, ctx, namespace, "echodest-"+tc.name, tc.families) + + actorName, _ := createAndResumeActor(t, ctx, "egress-"+tc.name, e2e.EgressFixture()) + router := mustRouterClient(t, ctx) + defer router.Close() + + actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} + url := fmt.Sprintf("http://%s.%s.svc.cluster.local:8080/healthz", service, namespace) + status, body := fetchThroughEgressActor(t, ctx, router, actorRef, url) + if status != http.StatusOK { + t.Fatalf("Actor egress to the %s destination returned HTTP %d, want 200; body: %s", tc.name, status, body) + } + + // The Actor echoes the destination's response back, so the family + // assertion reads the destination's view of the connection rather + // than trusting a 200. + got := destinationFamily(t, body) + if tc.want != "" && got != tc.want { + t.Fatalf("Actor egress to the %s destination arrived over %s, want %s", tc.name, got, tc.want) + } + t.Logf("Actor egress to the %s destination arrived over %s", tc.name, got) + }) + } +} + +// destinationFamily pulls echodest's reported family out of the body the +// egress Actor echoed back. +func destinationFamily(t *testing.T, body []byte) string { + t.Helper() + var echoed struct { + Body string `json:"body"` + } + payload := body + if err := json.Unmarshal(body, &echoed); err == nil && echoed.Body != "" { + payload = []byte(echoed.Body) + } + var reply struct { + Family string `json:"family"` + } + if err := json.Unmarshal(payload, &reply); err != nil { + t.Fatalf("parsing the destination's reply %q: %v", payload, err) + } + if reply.Family == "" { + t.Fatalf("the destination reported no family; reply: %s", payload) + } + return reply.Family +}