POC: pluggable ingress for substrate

cmd/atenet/internal/dns.go

@@ -33,10 +33,11 @@ import (
 )
 
 type DnsConfig struct {
-	LogLevel          string
-	Kubeconfig        string
-	ReconcileInterval time.Duration
-	CorefilePath      string
+	LogLevel           string
+	Kubeconfig         string
+	ReconcileInterval  time.Duration
+	CorefilePath       string
+	IngressServiceName string
 }
 
 func NewDnsCmd() *cobra.Command {
@@ -87,10 +88,11 @@ func NewDnsCmd() *cobra.Command {
 			}
 
 			dnsController := &dns.Controller{
-				Client:       k8sClient,
-				Interval:     cfg.ReconcileInterval,
-				CorefilePath: cfg.CorefilePath,
-				Reloader:     dns.NewConfigReloader(),
+				Client:             k8sClient,
+				Interval:           cfg.ReconcileInterval,
+				CorefilePath:       cfg.CorefilePath,
+				Reloader:           dns.NewConfigReloader(),
+				IngressServiceName: cfg.IngressServiceName,
 			}
 
 			slog.InfoContext(ctx, "Starting DNS Controller subsystem")
@@ -102,6 +104,9 @@ func NewDnsCmd() *cobra.Command {
 	cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file")
 	cmd.Flags().DurationVar(&cfg.ReconcileInterval, "interval", 10*time.Second, "Interval for reconciling DNS configurations")
 	cmd.Flags().StringVar(&cfg.CorefilePath, "corefile-path", "/etc/coredns/Corefile", "Path to the local Corefile configuration on shared volume")
+	cmd.Flags().StringVar(&cfg.IngressServiceName, "ingress-service-name", dns.DefaultIngressServiceName,
+		"Kubernetes Service in ate-system whose ClusterIP actor DNS resolves to. "+
+			"Set this to whatever Service fronts your gateway if you didn't name it \"atenet-gateway\".")
 
 	return cmd
 }

cmd/atenet/internal/dns/dns.go

@@ -37,6 +37,13 @@ const (
 	// serviceName is the name of the CoreDNS service.
 	serviceName     = "dns"
 	systemNamespace = "ate-system"
+
+	// DefaultIngressServiceName is the Kubernetes Service whose ClusterIP
+	// CoreDNS resolves actor hostnames to. atenet-router only serves ext_proc
+	// gRPC, not HTTP, so this must be the Service fronting whatever gateway
+	// you bring (see docs/dev/ingress.md); atenet-gateway is the name used by
+	// the reference configs there.
+	DefaultIngressServiceName = "atenet-gateway"
 )
 
 // Controller manages the DNS configuration for the ATE.
@@ -45,6 +52,10 @@ type Controller struct {
 	Interval     time.Duration
 	CorefilePath string
 	Reloader     ConfigReloader
+	// IngressServiceName is the Kubernetes Service in ate-system whose
+	// ClusterIP is written into the CoreDNS Corefile as the ingress target.
+	// Defaults to DefaultIngressServiceName ("atenet-gateway") when empty.
+	IngressServiceName string
 }
 
 // Run the DNS orchestration loop until ctx is canceled.
@@ -71,19 +82,25 @@ func (c *Controller) Run(ctx context.Context) error {
 func (c *Controller) reconcile(ctx context.Context) error {
 	slog.DebugContext(ctx, "Reconciling DNS orchestration configuration...")
 
-	// 1. Get the ClusterIP of atenet-router in ate-system namespace
+	// 1. Get the ClusterIP of the ingress service (your gateway) in ate-system.
+	ingressSvcName := c.IngressServiceName
+	if ingressSvcName == "" {
+		ingressSvcName = DefaultIngressServiceName
+	}
 	routerSvc := &corev1.Service{}
-	if err := c.Client.Get(ctx, types.NamespacedName{Name: "atenet-router", Namespace: systemNamespace}, routerSvc); err != nil {
+	if err := c.Client.Get(ctx, types.NamespacedName{Name: ingressSvcName, Namespace: systemNamespace}, routerSvc); err != nil {
 		if errors.IsNotFound(err) {
-			slog.WarnContext(ctx, "atenet-router service not found, skipping until it is available")
+			slog.WarnContext(ctx, "ingress service not found, skipping until it is available",
+				slog.String("service", ingressSvcName))
 			return nil
 		}
-		return fmt.Errorf("failed to get atenet-router service: %w", err)
+		return fmt.Errorf("failed to get ingress service %q: %w", ingressSvcName, err)
 	}
 
 	routerIP := routerSvc.Spec.ClusterIP
 	if routerIP == "" || routerIP == "None" {
-		slog.WarnContext(ctx, "atenet-router service has no ClusterIP yet, waiting...")
+		slog.WarnContext(ctx, "ingress service has no ClusterIP yet, waiting...",
+			slog.String("service", ingressSvcName))
 		return nil
 	}
 

cmd/atenet/internal/dns/dns_test.go

@@ -46,7 +46,7 @@ func TestReconcile(t *testing.T) {
 	// 1. Create mock services
 	routerSvc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "atenet-router",
+			Name:      "atenet-gateway",
 			Namespace: "ate-system",
 		},
 		Spec: corev1.ServiceSpec{
@@ -151,7 +151,7 @@ func TestReconcileKubeDNSNotFound(t *testing.T) {
 
 	routerSvc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "atenet-router",
+			Name:      "atenet-gateway",
 			Namespace: "ate-system",
 		},
 		Spec: corev1.ServiceSpec{

cmd/atenet/internal/router.go

@@ -29,7 +29,7 @@ func NewRouterCmd() *cobra.Command {
 
 	cmd := &cobra.Command{
 		Use:   "router",
-		Short: "Router components including xDS server and Envoy ExtProc gateway processing server",
+		Short: "Substrate routing core and ExtProc server",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			srv, err := router.NewRouterServer(cfg)
 			if err != nil {
@@ -47,17 +47,11 @@ func NewRouterCmd() *cobra.Command {
 	cmd.Flags().StringVar(&cfg.Namespace, "namespace", "default", "Target operations namespace")
 	cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file")
 	cmd.Flags().StringVar(&cfg.AteapiAddr, "ateapi-address", "api.ate-system.svc:443", "gRPC host address of the cluster ateapi Control instance")
-	cmd.Flags().IntVar(&cfg.HttpPort, "port-http", 8080, "TCP port for workload traffic entering through the Envoy Router")
-	cmd.Flags().IntVar(&cfg.XdsPort, "port-xds", 18000, "TCP port listening for the xDS dynamic Envoy connections")
-	cmd.Flags().IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the Envoy dynamic External Processing (ext_proc) server")
-	cmd.Flags().StringVar(&cfg.ExtprocAddr, "extproc-address", "127.0.0.1", "Host IP or address of the Envoy External Processing (ext_proc) server")
-	cmd.Flags().StringVar(&cfg.EnvoyImage, "envoy-image", "envoyproxy/envoy:v1.30-latest", "Image URI used for dynamically launched router instances")
+	cmd.Flags().IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the ext_proc server")
+	cmd.Flags().StringVar(&cfg.ExtprocAddr, "extproc-address", "127.0.0.1", "Host IP or address of the ext_proc server")
 	cmd.Flags().StringVar(&cfg.TemplatesFile, "actor-templates-file", "", "Path to offline YAML configuration file listing ActorTemplates")
 	cmd.Flags().IntVar(&cfg.StatusPort, "status-port", 4040, "Port to serve /statusz on (set <= 0 to disable serving status)")
 	cmd.Flags().DurationVar(&cfg.HealthInterval, "health-interval", 1*time.Second, "Interval for checking health of dependent services")
-	cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router")
-	cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file (if empty, a self-signed cert will be generated for testing)")
-	cmd.Flags().StringVar(&cfg.OtlpCollectorAddress, "otlp-collector-address", "", "host:port of the OTLP gRPC collector that Envoy reports tracing spans to (empty disables Envoy tracing)")
 	cmd.Flags().StringVar(&cfg.AteapiAuthMode, "ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) dials with insecure TLS and relies on pod-projected mTLS credentials for identity. 'jwt' verifies the server cert and sends a Bearer SA token.")
 	cmd.Flags().StringVar(&cfg.AteapiCAFile, "ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert. Required for jwt.")
 	cmd.Flags().StringVar(&cfg.AteapiServerName, "ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.")

cmd/atenet/internal/router/README.md

@@ -2,21 +2,18 @@
 
 Router has several responsibilities:
 
-* (Optional) manages a Deployment of Envoy to function as a router for ATE requests.
-  * This is optional to enable testing the router component in a standalone mode without managing the Kubernetes objects.
-  * Envoy will be configured to send traffic to via xDS served by the Router.
-* ext_proc server for the Envoy. To make the deployment and debugging easier, we will run this component together
-  with the router, but this will be split later into its own component.
-  * ext_proc will call into the ATE gRPC API to get the set of relevant backends (specific the worker IP) and
-    route the traffic accordingly
+* Runs the actor-aware routing core: resolve actor identity from a request,
+  resume/locate the actor via the ATE gRPC API, and pick a worker endpoint.
   * Make sure the interface with ATE API is pluggable so that we can test with a mock ATE API.
-* Runs an xDS server for the Envoy deployment that defines the Cluster information for the ATEs.
-  * the xDS configuration will configure Envoy to send traffic to ext_proc
+* Exposes that routing core over ext_proc, so any proxy that supports the
+  ExtProc protocol (and dynamic forwarding) can call it. Substrate does not
+  ship or manage a proxy — see [`docs/dev/ingress.md`](../../../../docs/dev/ingress.md)
+  for reference gateway configs.
 * Watches the ActorTemplates to get out the definitions for how to route the session IDs.
 
 ## status page
 
-Serve a `/statusz` page on port 8080.
+Serve a `/statusz` page on the router's status port.
 
 Contents:
 

cmd/atenet/internal/router/controller.go

@@ -23,28 +23,22 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-// Controller monitors ActorTemplates and coordinates configuration updates
-// for the Envoy xDS and external processing servers.
+// Controller monitors ActorTemplates for the external processing server.
 type Controller struct {
 	k8sClient  client.Client
 	clientset  kubernetes.Interface
 	cfg        RouterConfig
-	xdsSrv     *XdsServer
 	extprocSrv *ExtProcServer
 
-	atStore     atStore
-	envoyRunner *envoyrunner
+	atStore atStore
 }
 
 func NewController(
 	k8sClient client.Client,
 	clientset kubernetes.Interface,
 	cfg RouterConfig,
-	xdsSrv *XdsServer,
 	extprocSrv *ExtProcServer,
 ) *Controller {
-	xdsSrv.SetConfig(cfg.HttpPort, cfg.ExtprocPort, cfg.ExtprocAddr)
-
 	var store atStore
 	if cfg.TemplatesFile != "" {
 		store = newFileATStore(cfg.TemplatesFile)
@@ -56,11 +50,9 @@ func NewController(
 		k8sClient:  k8sClient,
 		clientset:  clientset,
 		cfg:        cfg,
-		xdsSrv:     xdsSrv,
 		extprocSrv: extprocSrv,
 
-		atStore:     store,
-		envoyRunner: newEnvoyRunner(k8sClient, cfg),
+		atStore: store,
 	}
 }
 
@@ -92,19 +84,5 @@ func (c *Controller) reconcile(ctx context.Context) error {
 		return err
 	}
 
-	if err := c.xdsSrv.UpdateSnapshot(); err != nil {
-		slog.ErrorContext(ctx, "xDS Configuration generation problem", slog.String("err", err.Error()))
-		return err
-	}
-
-	if !c.cfg.Standalone && c.cfg.TemplatesFile == "" {
-		// Reconcile Envoy router Deployment and Kubernetes cluster entities
-		err := c.envoyRunner.reconcile(ctx)
-		if err != nil {
-			slog.ErrorContext(ctx, "Error during Envoy router reconciliation", slog.String("err", err.Error()))
-			return err
-		}
-	}
-
 	return nil
 }

cmd/atenet/internal/router/dashboard.html

@@ -256,14 +256,6 @@ <h1>atenet Router Status</h1>
 
             <div class="card">
                 <div class="card-title">Component Network Allocation</div>
-                <div class="metadata-item">
-                    <span class="label">Workload Port (Http Envoy)</span>
-                    <span class="value">{{ .HttpPort }}</span>
-                </div>
-                <div class="metadata-item">
-                    <span class="label">xDS Interface Port</span>
-                    <span class="value">{{ .XdsPort }}</span>
-                </div>
                 <div class="metadata-item">
                     <span class="label">External Processing Port</span>
                     <span class="value">{{ .ExtprocPort }}</span>
@@ -277,25 +269,6 @@ <h1>atenet Router Status</h1>
             <div class="card">
                 <div class="card-title">System Component Health Checks</div>
 
-                <div class="metadata-item">
-                    <span class="label">Envoy Health</span>
-                    {{ if .Health.Envoy.Healthy }}
-                    <span class="value badge" style="background: rgba(16, 185, 129, 0.1); color: #10b981;">Healthy</span>
-                    {{ else }}
-                    <span class="value badge" style="background: rgba(239, 68, 68, 0.1); color: #ef4444;">Degraded</span>
-                    {{ end }}
-                </div>
-                <div class="metadata-item" style="margin-top: -0.25rem; margin-bottom: 0.5rem; font-size: 0.8rem;">
-                    <span class="label">Message / Err:</span>
-                    <span class="value" style="color: var(--text-secondary);">{{ .Health.Envoy.Message }}</span>
-                </div>
-                <div class="metadata-item" style="font-size: 0.75rem; color: var(--text-secondary);">
-                    <span>Ok: {{ .Health.Envoy.SuccessCount }} | Err: {{ .Health.Envoy.FailureCount }}</span>
-                    <span>LKG: {{ if not .Health.Envoy.LastSuccess.IsZero }}{{ .Health.Envoy.LastSuccess.Format "15:04:05" }}{{ else }}N/A{{ end }}</span>
-                </div>
-
-                <div style="border-bottom: 1px solid var(--card-border); margin: 0.5rem 0;"></div>
-
                 <div class="metadata-item">
                     <span class="label">Kubernetes API</span>
                     {{ if .Health.K8sAPI.Healthy }}