From b54d7b4d29219f971e8c6236d2e74c1de4d84ba4 Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Thu, 4 Jun 2026 12:06:51 -0400 Subject: [PATCH] feat: inject projected SA token volume into sandbox pods The gateway's K8s SA authenticator requires tokens with audience "openshell-gateway". Replace the default SA token path with a projected serviceAccountToken volume so the supervisor authenticates correctly via IssueSandboxToken. Changes: - Add SATokenAudience/SATokenTTLSecs config fields and CLI flags - Add projected volume with configurable audience and TTL - Mount at /var/run/secrets/openshell/token (read-only, mode 0400) - Update OPENSHELL_K8S_SA_TOKEN_FILE to the new path Fixes: kagenti/kagenti#1815 Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- cmd/driver/main.go | 4 +++ internal/driver/config.go | 4 +++ internal/driver/config_test.go | 5 +++ internal/driver/provisioner.go | 26 +++++++++++++++- internal/driver/provisioner_test.go | 48 +++++++++++++++++++++++++---- 5 files changed, 80 insertions(+), 7 deletions(-) diff --git a/cmd/driver/main.go b/cmd/driver/main.go index 5cf547b..b1cc9e6 100644 --- a/cmd/driver/main.go +++ b/cmd/driver/main.go @@ -40,6 +40,10 @@ func main() { "Secret name containing tls.crt and tls.key for sandbox mTLS client auth") flag.StringVar(&cfg.ImagePullPolicy, "sandbox-image-pull-policy", cfg.ImagePullPolicy, "Image pull policy for sandbox pod containers (Always, IfNotPresent, Never); empty uses K8s default") + flag.StringVar(&cfg.SATokenAudience, "sa-token-audience", cfg.SATokenAudience, + "Audience for the projected ServiceAccount token injected into sandbox pods") + flag.Int64Var(&cfg.SATokenTTLSecs, "sa-token-ttl-secs", cfg.SATokenTTLSecs, + "Expiration (seconds) for the projected ServiceAccount token") flag.Parse() if cfg.Tenant == "" { diff --git a/internal/driver/config.go b/internal/driver/config.go index 0237e78..6bf52d3 100644 --- a/internal/driver/config.go +++ b/internal/driver/config.go @@ -10,6 +10,8 @@ type Config struct { TLSCASecret string // Secret name containing ca.crt for gateway TLS verification TLSClientSecret string // Secret name containing tls.crt and tls.key for mTLS client auth ImagePullPolicy string // Policy for sandbox pod containers (Always, IfNotPresent, Never); empty means K8s default + SATokenAudience string // audience for the projected SA token volume + SATokenTTLSecs int64 // expiration seconds for the projected SA token } func DefaultConfig() Config { @@ -18,5 +20,7 @@ func DefaultConfig() Config { SupervisorImage: "quay.io/azaalouk/openshell-supervisor:latest", SupervisorBinaryPath: "/openshell-sandbox", SupervisorMountPath: "/opt/openshell/bin", + SATokenAudience: "openshell-gateway", + SATokenTTLSecs: 3600, } } diff --git a/internal/driver/config_test.go b/internal/driver/config_test.go index 7e6bd02..d022f67 100644 --- a/internal/driver/config_test.go +++ b/internal/driver/config_test.go @@ -14,6 +14,7 @@ func TestDefaultConfig(t *testing.T) { {"SupervisorImage", cfg.SupervisorImage, "quay.io/azaalouk/openshell-supervisor:latest"}, {"SupervisorBinaryPath", cfg.SupervisorBinaryPath, "/openshell-sandbox"}, {"SupervisorMountPath", cfg.SupervisorMountPath, "/opt/openshell/bin"}, + {"SATokenAudience", cfg.SATokenAudience, "openshell-gateway"}, } for _, tt := range tests { @@ -21,6 +22,10 @@ func TestDefaultConfig(t *testing.T) { t.Errorf("DefaultConfig().%s = %q, want %q", tt.field, tt.got, tt.want) } } + + if cfg.SATokenTTLSecs != 3600 { + t.Errorf("DefaultConfig().SATokenTTLSecs = %d, want 3600", cfg.SATokenTTLSecs) + } } func TestConfigZeroValue(t *testing.T) { diff --git a/internal/driver/provisioner.go b/internal/driver/provisioner.go index 1a6daba..b2041a5 100644 --- a/internal/driver/provisioner.go +++ b/internal/driver/provisioner.go @@ -278,6 +278,13 @@ func (p *K8sProvisioner) buildSandboxSpec(sb *pb.DriverSandbox) map[string]inter "readOnly": true, }) } + if p.cfg.SATokenAudience != "" { + agentVolumeMounts = append(agentVolumeMounts, map[string]interface{}{ + "name": "openshell-sa-token", + "mountPath": "/var/run/secrets/openshell", + "readOnly": true, + }) + } container := map[string]interface{}{ "name": "agent", @@ -329,6 +336,23 @@ func (p *K8sProvisioner) buildSandboxSpec(sb *pb.DriverSandbox) map[string]inter }, }) } + if p.cfg.SATokenAudience != "" { + volumes = append(volumes, map[string]interface{}{ + "name": "openshell-sa-token", + "projected": map[string]interface{}{ + "sources": []interface{}{ + map[string]interface{}{ + "serviceAccountToken": map[string]interface{}{ + "audience": p.cfg.SATokenAudience, + "expirationSeconds": p.cfg.SATokenTTLSecs, + "path": "token", + }, + }, + }, + "defaultMode": int64(0400), + }, + }) + } podSpec := map[string]interface{}{ "initContainers": []interface{}{initContainer}, @@ -390,7 +414,7 @@ func (p *K8sProvisioner) buildFullEnvList( gatewayEnv["OPENSHELL_TLS_KEY"] = "/tls/client/tls.key" } - gatewayEnv["OPENSHELL_K8S_SA_TOKEN_FILE"] = "/var/run/secrets/kubernetes.io/serviceaccount/token" + gatewayEnv["OPENSHELL_K8S_SA_TOKEN_FILE"] = "/var/run/secrets/openshell/token" gatewayEnv["OPENSHELL_LOG_LEVEL"] = "debug" gatewayEnv["ANTHROPIC_BASE_URL"] = "https://inference.local" gatewayEnv["OPENAI_BASE_URL"] = "https://inference.local/v1" diff --git a/internal/driver/provisioner_test.go b/internal/driver/provisioner_test.go index 00aba3d..9f3c974 100644 --- a/internal/driver/provisioner_test.go +++ b/internal/driver/provisioner_test.go @@ -247,8 +247,8 @@ func TestBuildSandboxSpec_SupervisorInitContainer(t *testing.T) { // Verify volume mounts on agent container. agentMounts := agentC["volumeMounts"].([]interface{}) - if len(agentMounts) != 1 { - t.Fatalf("expected 1 volume mount on agent, got %d", len(agentMounts)) + if len(agentMounts) != 2 { + t.Fatalf("expected 2 volume mounts on agent, got %d", len(agentMounts)) } mount := agentMounts[0].(map[string]interface{}) if mount["name"] != "supervisor-bin" { @@ -257,11 +257,21 @@ func TestBuildSandboxSpec_SupervisorInitContainer(t *testing.T) { if mount["readOnly"] != true { t.Error("expected readOnly=true on agent volume mount") } + saMount := agentMounts[1].(map[string]interface{}) + if saMount["name"] != "openshell-sa-token" { + t.Errorf("expected mount name openshell-sa-token, got %v", saMount["name"]) + } + if saMount["mountPath"] != "/var/run/secrets/openshell" { + t.Errorf("expected mountPath /var/run/secrets/openshell, got %v", saMount["mountPath"]) + } + if saMount["readOnly"] != true { + t.Error("expected readOnly=true on SA token volume mount") + } // Verify volumes. volumes, ok := podSpec["volumes"].([]interface{}) - if !ok || len(volumes) == 0 { - t.Fatal("missing volumes") + if !ok || len(volumes) < 2 { + t.Fatalf("expected at least 2 volumes, got %d", len(volumes)) } vol := volumes[0].(map[string]interface{}) if vol["name"] != "supervisor-bin" { @@ -270,6 +280,32 @@ func TestBuildSandboxSpec_SupervisorInitContainer(t *testing.T) { if _, ok := vol["emptyDir"]; !ok { t.Error("expected emptyDir volume") } + saVol := volumes[1].(map[string]interface{}) + if saVol["name"] != "openshell-sa-token" { + t.Errorf("expected volume name openshell-sa-token, got %v", saVol["name"]) + } + projected, ok := saVol["projected"].(map[string]interface{}) + if !ok { + t.Fatal("expected projected volume") + } + sources, ok := projected["sources"].([]interface{}) + if !ok || len(sources) == 0 { + t.Fatal("expected projected sources") + } + src := sources[0].(map[string]interface{}) + saToken, ok := src["serviceAccountToken"].(map[string]interface{}) + if !ok { + t.Fatal("expected serviceAccountToken source") + } + if saToken["audience"] != cfg.SATokenAudience { + t.Errorf("expected audience %s, got %v", cfg.SATokenAudience, saToken["audience"]) + } + if saToken["expirationSeconds"] != cfg.SATokenTTLSecs { + t.Errorf("expected expirationSeconds %d, got %v", cfg.SATokenTTLSecs, saToken["expirationSeconds"]) + } + if saToken["path"] != "token" { + t.Errorf("expected path token, got %v", saToken["path"]) + } } func TestBuildSandboxSpec_Labels(t *testing.T) { @@ -488,8 +524,8 @@ func TestBuildSandboxSpec_SATokenEnv(t *testing.T) { env := e.(map[string]interface{}) if env["name"] == "OPENSHELL_K8S_SA_TOKEN_FILE" { found = true - if env["value"] != "/var/run/secrets/kubernetes.io/serviceaccount/token" { - t.Errorf("expected SA token path, got %v", env["value"]) + if env["value"] != "/var/run/secrets/openshell/token" { + t.Errorf("expected /var/run/secrets/openshell/token, got %v", env["value"]) } break }