From 0c7ff83294726fab7cb16007483cb96446df7b2a Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 14:40:40 +0000 Subject: [PATCH 01/98] test(cluster): define operator and chart contracts --- .../api/v1alpha1/types_test.go | 161 +++++++++ .../charttests/chart_contract_test.go | 196 +++++++++++ .../cluster-operator/cmd/manager/main_test.go | 25 ++ .../controllers/reconciler_test.go | 308 ++++++++++++++++++ packages/cluster-operator/go.mod | 14 + 5 files changed, 704 insertions(+) create mode 100644 packages/cluster-operator/api/v1alpha1/types_test.go create mode 100644 packages/cluster-operator/charttests/chart_contract_test.go create mode 100644 packages/cluster-operator/cmd/manager/main_test.go create mode 100644 packages/cluster-operator/controllers/reconciler_test.go create mode 100644 packages/cluster-operator/go.mod diff --git a/packages/cluster-operator/api/v1alpha1/types_test.go b/packages/cluster-operator/api/v1alpha1/types_test.go new file mode 100644 index 00000000..a43a88d9 --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/types_test.go @@ -0,0 +1,161 @@ +package v1alpha1_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" +) + +func TestKindsAreNamespacedAndRegistered(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + for _, kind := range []string{"T4ClusterHost", "T4Workspace", "T4Session"} { + gvk := clusterv1alpha1.GroupVersion.WithKind(kind) + if _, err := scheme.New(gvk); err != nil { + t.Fatalf("%s is not registered: %v", gvk, err) + } + } +} + +func TestStatusIsInfrastructureOnlyAndBounded(t *testing.T) { + statuses := []struct { + name string + generation int64 + conditions []metav1.Condition + }{ + {"host", clusterv1alpha1.T4ClusterHostStatus{}.ObservedGeneration, clusterv1alpha1.T4ClusterHostStatus{}.Conditions}, + {"workspace", clusterv1alpha1.T4WorkspaceStatus{}.ObservedGeneration, clusterv1alpha1.T4WorkspaceStatus{}.Conditions}, + {"session", clusterv1alpha1.T4SessionStatus{}.ObservedGeneration, clusterv1alpha1.T4SessionStatus{}.Conditions}, + } + for _, status := range statuses { + if status.generation != 0 || status.conditions != nil { + t.Fatalf("zero %s status must be empty", status.name) + } + } + + // Compile-time API guards: infrastructure references are explicit; no OMP ids, + // prompts, transcript, agent tree, or lifecycle ownership is represented here. + _ = clusterv1alpha1.T4WorkspaceStatus{PVCName: "pvc", Phase: clusterv1alpha1.InfrastructurePending} + _ = clusterv1alpha1.T4SessionStatus{PodName: "pod", ServiceName: "service", Phase: clusterv1alpha1.InfrastructurePending} +} + +func TestEnumsRejectUnboundedValuesAtTheGoBoundary(t *testing.T) { + if !clusterv1alpha1.ValidRetentionPolicy(clusterv1alpha1.RetentionPolicyRetain) || + !clusterv1alpha1.ValidRetentionPolicy(clusterv1alpha1.RetentionPolicyDelete) || + clusterv1alpha1.ValidRetentionPolicy("Archive") { + t.Fatal("retention policy allowlist is not exact") + } + if !clusterv1alpha1.ValidInfrastructurePhase(clusterv1alpha1.InfrastructureRunning) || + clusterv1alpha1.ValidInfrastructurePhase("OMPRunning") { + t.Fatal("infrastructure phase allowlist accepts non-infrastructure state") + } +} + +func TestCRDContractConstants(t *testing.T) { + if got, want := clusterv1alpha1.GroupVersion.String(), "cluster.t4.dev/v1alpha1"; got != want { + t.Fatalf("group version = %q, want %q", got, want) + } + if got, want := clusterv1alpha1.WorkspaceFinalizer, "cluster.t4.dev/workspace-protection"; got != want { + t.Fatalf("workspace finalizer = %q", got) + } + if got, want := clusterv1alpha1.SessionFinalizer, "cluster.t4.dev/session-cleanup"; got != want { + t.Fatalf("session finalizer = %q", got) + } + if got, want := clusterv1alpha1.RWXStorageClassAnnotation, "cluster.t4.dev/access-modes"; got != want { + t.Fatalf("RWX storage annotation = %q", got) + } +} + +func TestCRDSchemasAreStructuralBoundedAndValidated(t *testing.T) { + paths := []string{ + "t4clusterhosts.cluster.t4.dev.yaml", + "t4workspaces.cluster.t4.dev.yaml", + "t4sessions.cluster.t4.dev.yaml", + } + for _, name := range paths { + raw, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "deploy", "charts", "t4-cluster", "crds", name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(raw, &crd); err != nil { + t.Fatalf("decode %s: %v", name, err) + } + if crd.Spec.Scope != apiextensionsv1.NamespaceScoped { + t.Fatalf("%s must be namespaced", crd.Name) + } + if len(crd.Spec.Versions) != 1 || !crd.Spec.Versions[0].Served || !crd.Spec.Versions[0].Storage { + t.Fatalf("%s must have one served storage version", crd.Name) + } + version := crd.Spec.Versions[0] + if version.Subresources == nil || version.Subresources.Status == nil { + t.Fatalf("%s lacks the status subresource", crd.Name) + } + if version.Schema == nil || version.Schema.OpenAPIV3Schema == nil { + t.Fatalf("%s lacks an OpenAPI schema", crd.Name) + } + if _, err := structuralschema.NewStructural(version.Schema.OpenAPIV3Schema); err != nil { + t.Fatalf("%s is not structural: %v", crd.Name, err) + } + root := version.Schema.OpenAPIV3Schema + if root.XPreserveUnknownFields != nil && *root.XPreserveUnknownFields { + t.Fatalf("%s preserves unknown fields", crd.Name) + } + status := root.Properties["status"] + conditions := status.Properties["conditions"] + if conditions.MaxItems == nil || *conditions.MaxItems > 8 { + t.Fatalf("%s status conditions are not bounded", crd.Name) + } + if _, ok := status.Properties["observedGeneration"]; !ok { + t.Fatalf("%s status lacks observedGeneration", crd.Name) + } + assertBoundedSchema(t, crd.Name+".spec", root.Properties["spec"]) + } +} + +func TestCRDsHaveCrossFieldCELAndForbidClientRuntimeAuthority(t *testing.T) { + for _, name := range []string{"t4clusterhosts.cluster.t4.dev.yaml", "t4workspaces.cluster.t4.dev.yaml", "t4sessions.cluster.t4.dev.yaml"} { + raw, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "deploy", "charts", "t4-cluster", "crds", name)) + if err != nil { + t.Fatal(err) + } + text := string(raw) + if !strings.Contains(text, "x-kubernetes-validations:") { + t.Fatalf("%s has no CEL validation", name) + } + for _, forbidden := range []string{"image:", "prompt:", "shell:", "token:", "ompSession", "agentId", "transcript"} { + if strings.Contains(text, forbidden) { + t.Fatalf("%s exposes forbidden authority field %q", name, forbidden) + } + } + } +} + +func assertBoundedSchema(t *testing.T, path string, schema apiextensionsv1.JSONSchemaProps) { + t.Helper() + if schema.Type == "string" && schema.MaxLength == nil && schema.Enum == nil { + t.Fatalf("%s is an unbounded string", path) + } + if schema.Type == "array" { + if schema.MaxItems == nil { + t.Fatalf("%s is an unbounded array", path) + } + if schema.Items != nil && schema.Items.Schema != nil { + assertBoundedSchema(t, path+"[]", *schema.Items.Schema) + } + } + for key, child := range schema.Properties { + assertBoundedSchema(t, path+"."+key, child) + } +} diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go new file mode 100644 index 00000000..e36a2615 --- /dev/null +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -0,0 +1,196 @@ +package charttests + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +const fakeDigest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func TestChartIsDefaultOff(t *testing.T) { + output := helmTemplate(t) + if strings.TrimSpace(output) != "" { + t.Fatalf("default values rendered workloads/resources:\n%s", output) + } +} + +func TestEnabledChartRendersHARestrictedWorkloads(t *testing.T) { + output := helmTemplate(t, enabledValues()...) + assertCount(t, output, "kind: Deployment", 2) + assertContains(t, output, + "replicas: 2", + "replicas: 3", + "maxUnavailable: 0", + "kind: PodDisruptionBudget", + "minAvailable: 2", + "kubernetes.io/hostname", + "k3s-worker-02", + "topologySpreadConstraints:", + "podAntiAffinity:", + "readOnlyRootFilesystem: true", + "runAsNonRoot: true", + "allowPrivilegeEscalation: false", + "type: RuntimeDefault", + "drop:", + "- ALL", + "automountServiceAccountToken: false", + "startupProbe:", + "readinessProbe:", + "livenessProbe:", + "preStop:", + "path: /drainz", + "kind: NetworkPolicy", + "policyTypes:", + "kind: Role", + "kind: ClusterRole", + "coordination.k8s.io", + "resources:", + ) + if strings.Contains(output, "privileged: true") || strings.Contains(output, "hostNetwork: true") || strings.Contains(output, "hostPID: true") { + t.Fatal("enabled chart contains a privileged shortcut") + } + if strings.Contains(output, "kind: PersistentVolumeClaim") || strings.Contains(output, "nfs:") || strings.Contains(output, "hostPath:") { + t.Fatal("portable chart rendered storage backend or workload PVC") + } +} + +func TestRBACSeparatesControllerMutationFromServerProjection(t *testing.T) { + output := helmTemplate(t, enabledValues()...) + controllerRole := documentContaining(t, output, "name: release-name-t4-cluster-controller") + serverRole := documentContaining(t, output, "name: release-name-t4-cluster-server") + assertContains(t, controllerRole, "persistentvolumeclaims", "pods", "services", "t4sessions/status", "leases") + assertContains(t, serverRole, "t4clusterhosts", "t4workspaces", "t4sessions", "create", "list", "watch") + if strings.Contains(serverRole, "secrets") || strings.Contains(serverRole, "persistentvolumeclaims") || strings.Contains(serverRole, "t4sessions/status") { + t.Fatal("server role can read secrets or mutate controller-owned infrastructure/status") + } +} + +func TestNetworkPoliciesDefaultDenyAndAllowOnlyDeclaredFlows(t *testing.T) { + output := helmTemplate(t, append(enabledValues(), + "--set", "networkPolicy.kubernetesApiCIDRs[0]=192.0.2.10/32", + "--set", "networkPolicy.modelRouteCIDRs[0]=198.51.100.4/32", + "--set", "networkPolicy.ciProviderCIDRs[0]=203.0.113.8/32", + )...) + assertContains(t, output, + "name: release-name-t4-cluster-default-deny", + "192.0.2.10/32", + "198.51.100.4/32", + "203.0.113.8/32", + "port: 53", + "port: 8787", + ) + if strings.Contains(output, "0.0.0.0/0") { + t.Fatal("network policy contains broad Internet egress") + } +} + +func TestCRDsRemainExplicitAcrossUpgradeAndUninstall(t *testing.T) { + withoutCRDs := helmTemplate(t, enabledValues()...) + if strings.Contains(withoutCRDs, "kind: CustomResourceDefinition") { + t.Fatal("CRDs must live in Helm crds/, not upgrade-rendered templates") + } + withCRDs := helmTemplate(t, append([]string{"--include-crds"}, enabledValues()...)...) + assertCount(t, withCRDs, "kind: CustomResourceDefinition", 3) + assertContains(t, withCRDs, "t4clusterhosts.cluster.t4.dev", "t4workspaces.cluster.t4.dev", "t4sessions.cluster.t4.dev") + + docs, err := os.ReadFile(filepath.Join(repoRoot(t), "docs", "CLUSTER_OPERATOR.md")) + if err != nil { + t.Fatal(err) + } + for _, required := range []string{"helm upgrade", "helm rollback", "helm uninstall", "Retain", "Delete", "CRDs are not removed"} { + if !strings.Contains(string(docs), required) { + t.Fatalf("operator guide lacks upgrade/uninstall contract %q", required) + } + } +} + +func TestImageContractsArePinnedAndAuthorityCompatible(t *testing.T) { + root := repoRoot(t) + controller := mustRead(t, filepath.Join(root, "cluster", "images", "controller", "Dockerfile")) + server := mustRead(t, filepath.Join(root, "cluster", "images", "cluster-server", "Dockerfile")) + session := mustRead(t, filepath.Join(root, "cluster", "images", "session-runtime", "Dockerfile")) + for name, content := range map[string]string{"controller": controller, "server": server, "session": session} { + if !strings.Contains(content, "@sha256:") { + t.Fatalf("%s image uses an unpinned base", name) + } + } + assertContains(t, session, + "8476f4451ed95c5d5401785d279a93d3c659fac4", + "t4code-17.0.5-appserver-10", + "t4-omp-authority/1", + "packages/cluster-server/src/session-host-main.ts", + "chromium", + "Xvfb", + ) + assertContains(t, server, "packages/cluster-server/src/main.ts") +} + +func helmTemplate(t *testing.T, extra ...string) string { + t.Helper() + args := []string{"template", "release-name", filepath.Join(repoRoot(t), "deploy", "charts", "t4-cluster"), "--namespace", "t4-system"} + args = append(args, extra...) + command := exec.Command("helm", args...) + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("helm %s: %v\n%s", strings.Join(args, " "), err, output) + } + return string(output) +} + +func enabledValues() []string { + return []string{ + "--set", "enabled=true", + "--set", "storage.adminRWXStorageClass=portable-rwx", + "--set", "images.controller.digest=" + fakeDigest, + "--set", "images.server.digest=" + fakeDigest, + "--set", "images.sessionRuntime.digest=" + fakeDigest, + } +} + +func repoRoot(t *testing.T) string { + t.Helper() + root, err := filepath.Abs(filepath.Join("..", "..", "..")) + if err != nil { + t.Fatal(err) + } + return root +} + +func mustRead(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func assertContains(t *testing.T, value string, required ...string) { + t.Helper() + for _, item := range required { + if !strings.Contains(value, item) { + t.Fatalf("output lacks %q", item) + } + } +} + +func assertCount(t *testing.T, value, needle string, want int) { + t.Helper() + if got := strings.Count(value, needle); got != want { + t.Fatalf("count(%q) = %d, want %d", needle, got, want) + } +} + +func documentContaining(t *testing.T, rendered, needle string) string { + t.Helper() + for _, document := range strings.Split(rendered, "\n---") { + if strings.Contains(document, needle) { + return document + } + } + t.Fatalf("no rendered document contains %q", needle) + return "" +} diff --git a/packages/cluster-operator/cmd/manager/main_test.go b/packages/cluster-operator/cmd/manager/main_test.go new file mode 100644 index 00000000..cc7416a3 --- /dev/null +++ b/packages/cluster-operator/cmd/manager/main_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "testing" + "time" +) + +func TestManagerUsesLeaseLeaderElection(t *testing.T) { + options := managerOptions() + if !options.LeaderElection { + t.Fatal("controller manager must enable leader election") + } + if options.LeaderElectionResourceLock != "leases" { + t.Fatalf("leader-election resource lock = %q, want leases", options.LeaderElectionResourceLock) + } + if options.LeaderElectionID != "t4-cluster-operator.cluster.t4.dev" { + t.Fatalf("leader-election ID = %q", options.LeaderElectionID) + } + if options.LeaseDuration == nil || options.RenewDeadline == nil || options.RetryPeriod == nil { + t.Fatal("leader election timing must be explicit") + } + if *options.LeaseDuration < 15*time.Second || *options.RenewDeadline >= *options.LeaseDuration || *options.RetryPeriod >= *options.RenewDeadline { + t.Fatalf("unsafe leader-election timing: lease=%v renew=%v retry=%v", *options.LeaseDuration, *options.RenewDeadline, *options.RetryPeriod) + } +} diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go new file mode 100644 index 00000000..fdd5ec3e --- /dev/null +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -0,0 +1,308 @@ +package controllers_test + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apiresource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" + "github.com/LycaonLLC/t4-code/packages/cluster-operator/controllers" +) + +func TestWorkspaceReconcileIsIdempotentAcrossDuplicateEvents(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4ClusterHost{}, &clusterv1alpha1.T4Workspace{}, &clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), rwxStorageClass(), workspace).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + reconcileMany(t, 4, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) + return err + }) + + var pvcs corev1.PersistentVolumeClaimList + if err := c.List(context.Background(), &pvcs, client.InNamespace("team")); err != nil { + t.Fatal(err) + } + if len(pvcs.Items) != 1 { + t.Fatalf("duplicate events created %d PVCs, want 1", len(pvcs.Items)) + } + pvc := pvcs.Items[0] + if len(pvc.Spec.AccessModes) != 1 || pvc.Spec.AccessModes[0] != corev1.ReadWriteMany { + t.Fatalf("PVC access modes = %v, want only ReadWriteMany", pvc.Spec.AccessModes) + } + if pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName != "portable-rwx" { + t.Fatalf("PVC storage class = %v", pvc.Spec.StorageClassName) + } + + var got clusterv1alpha1.T4Workspace + if err := c.Get(context.Background(), client.ObjectKeyFromObject(workspace), &got); err != nil { + t.Fatal(err) + } + if got.Status.ObservedGeneration != got.Generation || got.Status.PVCName != pvc.Name { + t.Fatalf("workspace status not converged: %#v", got.Status) + } + if !contains(got.Finalizers, clusterv1alpha1.WorkspaceFinalizer) { + t.Fatal("workspace protection finalizer missing") + } +} + +func TestWorkspaceStorageFailsClosedWhenClassMissingOrNotRWX(t *testing.T) { + for _, test := range []struct { + name string + class *storagev1.StorageClass + reason string + }{ + {name: "missing", reason: controllers.ReasonStorageClassNotFound}, + {name: "not-rwx", class: &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "portable-rwx"}, Provisioner: "example.invalid/csi"}, reason: controllers.ReasonStorageClassNotRWX}, + } { + t.Run(test.name, func(t *testing.T) { + scheme := testScheme(t) + objects := []client.Object{testHost(), testWorkspace(clusterv1alpha1.RetentionPolicyDelete)} + if test.class != nil { + objects = append(objects, test.class) + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(objects...).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "team", Name: "workspace-a"}}) + return err + }) + var pvcs corev1.PersistentVolumeClaimList + if err := c.List(context.Background(), &pvcs, client.InNamespace("team")); err != nil { + t.Fatal(err) + } + if len(pvcs.Items) != 0 { + t.Fatalf("fail-closed path created %d PVCs", len(pvcs.Items)) + } + var got clusterv1alpha1.T4Workspace + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: "workspace-a"}, &got); err != nil { + t.Fatal(err) + } + condition := findCondition(got.Status.Conditions, "StorageReady") + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != test.reason { + t.Fatalf("StorageReady = %#v, want False/%s", condition, test.reason) + } + }) + } +} + +func TestRetainDeletionOrphansPVCBeforeRemovingFinalizer(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) + workspace.UID = "workspace-uid" + workspace.Finalizers = []string{clusterv1alpha1.WorkspaceFinalizer} + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllers.WorkspacePVCName(workspace), + Namespace: workspace.Namespace, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true), + }}, + }, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(testHost(), rwxStorageClass(), workspace, pvc).Build() + if err := c.Delete(context.Background(), workspace); err != nil { + t.Fatal(err) + } + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) + return err + }) + var retained corev1.PersistentVolumeClaim + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pvc), &retained); err != nil { + t.Fatalf("retained PVC was deleted: %v", err) + } + if len(retained.OwnerReferences) != 0 || retained.Annotations[clusterv1alpha1.RetainedPVCAnnotation] != "true" { + t.Fatalf("retained PVC was not orphaned safely: %#v", retained.ObjectMeta) + } + var gone clusterv1alpha1.T4Workspace + if err := c.Get(context.Background(), client.ObjectKeyFromObject(workspace), &gone); !apierrors.IsNotFound(err) { + t.Fatalf("workspace should be deleted after retention, got %v", err) + } +} + +func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + workspace.Status.Phase = clusterv1alpha1.InfrastructurePending + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending}, + } + session := testSession() + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := &controllers.SessionReconciler{Client: c, Scheme: scheme, RuntimeImage: "registry.example/t4/session@sha256:0123456789abcdef"} + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + assertObjectCounts(t, c, 0, 0) + + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pvc), pvc); err != nil { + t.Fatal(err) + } + pvc.Status.Phase = corev1.ClaimBound + pvc.Status.Capacity = corev1.ResourceList{corev1.ResourceStorage: apiresource.MustParse("10Gi")} + if err := c.Status().Update(context.Background(), pvc); err != nil { + t.Fatal(err) + } + reconcileMany(t, 4, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + assertObjectCounts(t, c, 1, 1) + + var pods corev1.PodList + if err := c.List(context.Background(), &pods, client.InNamespace("team")); err != nil { + t.Fatal(err) + } + pod := pods.Items[0] + if pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { + t.Fatal("session pod must not mount a service account token") + } + if pod.Spec.Containers[0].Image != r.RuntimeImage { + t.Fatalf("controller did not use administrator-owned runtime image: %q", pod.Spec.Containers[0].Image) + } + if pod.Spec.Containers[0].SecurityContext == nil || pod.Spec.Containers[0].SecurityContext.Privileged != nil && *pod.Spec.Containers[0].SecurityContext.Privileged { + t.Fatal("session runtime is not restricted") + } + if !hasMount(pod.Spec.Containers[0].VolumeMounts, "workspace", "/workspace") || !hasMount(pod.Spec.Containers[0].VolumeMounts, "shared-memory", "/dev/shm") { + t.Fatalf("session mounts = %#v", pod.Spec.Containers[0].VolumeMounts) + } +} + +func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + session.Finalizers = []string{clusterv1alpha1.SessionFinalizer} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: "team"}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: "team"}} + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() + if err := c.Delete(context.Background(), session); err != nil { + t.Fatal(err) + } + r := &controllers.SessionReconciler{Client: c, Scheme: scheme, RuntimeImage: "registry.example/session@sha256:deadbeef"} + reconcileMany(t, 3, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + assertObjectCounts(t, c, 0, 0) + var gone clusterv1alpha1.T4Session + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &gone); !apierrors.IsNotFound(err) { + t.Fatalf("session finalizer removed before cleanup completed: %v", err) + } +} + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, storagev1.AddToScheme, clusterv1alpha1.AddToScheme} { + if err := add(scheme); err != nil { + t.Fatal(err) + } + } + return scheme +} + +func testHost() *clusterv1alpha1.T4ClusterHost { + return &clusterv1alpha1.T4ClusterHost{ + ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team", UID: "host-uid"}, + Spec: clusterv1alpha1.T4ClusterHostSpec{StorageClassName: "portable-rwx", RuntimeProfiles: []string{"default"}}, + } +} + +func rwxStorageClass() *storagev1.StorageClass { + return &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "portable-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, + Provisioner: "example.invalid/csi", + } +} + +func testWorkspace(policy clusterv1alpha1.RetentionPolicy) *clusterv1alpha1.T4Workspace { + return &clusterv1alpha1.T4Workspace{ + ObjectMeta: metav1.ObjectMeta{Name: "workspace-a", Namespace: "team", Generation: 3}, + Spec: clusterv1alpha1.T4WorkspaceSpec{ + HostRef: "host-a", DisplayName: "Workspace A", Owner: "team-a", Size: apiresource.MustParse("10Gi"), RetentionPolicy: policy, + }, + } +} + +func testSession() *clusterv1alpha1.T4Session { + return &clusterv1alpha1.T4Session{ + ObjectMeta: metav1.ObjectMeta{Name: "session-a", Namespace: "team", Generation: 2}, + Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a", Title: "Session A", RuntimeProfile: "default", GUIEnabled: true}, + } +} + +func reconcileMany(t *testing.T, count int, reconcile func() error) { + t.Helper() + for i := 0; i < count; i++ { + if err := reconcile(); err != nil { + t.Fatalf("reconcile %d: %v", i+1, err) + } + } +} + +func assertObjectCounts(t *testing.T, c client.Client, wantPods, wantServices int) { + t.Helper() + var pods corev1.PodList + var services corev1.ServiceList + if err := c.List(context.Background(), &pods, client.InNamespace("team")); err != nil { + t.Fatal(err) + } + if err := c.List(context.Background(), &services, client.InNamespace("team")); err != nil { + t.Fatal(err) + } + if len(pods.Items) != wantPods || len(services.Items) != wantServices { + t.Fatalf("pods/services = %d/%d, want %d/%d", len(pods.Items), len(services.Items), wantPods, wantServices) + } +} + +func findCondition(conditions []metav1.Condition, conditionType string) *metav1.Condition { + for i := range conditions { + if conditions[i].Type == conditionType { + return &conditions[i] + } + } + return nil +} + +func contains(values []string, wanted string) bool { + for _, value := range values { + if value == wanted { + return true + } + } + return false +} + +func hasMount(mounts []corev1.VolumeMount, name, path string) bool { + for _, mount := range mounts { + if mount.Name == name && mount.MountPath == path { + return true + } + } + return false +} + +func ptr[T any](value T) *T { return &value } + diff --git a/packages/cluster-operator/go.mod b/packages/cluster-operator/go.mod new file mode 100644 index 00000000..c6c51b71 --- /dev/null +++ b/packages/cluster-operator/go.mod @@ -0,0 +1,14 @@ +module github.com/LycaonLLC/t4-code/packages/cluster-operator + +go 1.24.0 + +require ( + github.com/google/go-cmp v0.7.0 + github.com/stretchr/testify v1.10.0 + k8s.io/api v0.33.2 + k8s.io/apiextensions-apiserver v0.33.2 + k8s.io/apimachinery v0.33.2 + k8s.io/client-go v0.33.2 + sigs.k8s.io/controller-runtime v0.21.0 + sigs.k8s.io/yaml v1.4.0 +) From ad237e3baf64fa37aa8f7d3cd25dacc3e7adeb69 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 14:41:41 +0000 Subject: [PATCH 02/98] test(cluster): define default-off operator client contracts --- apps/desktop/test/target-manager.test.ts | 29 ++ apps/web/test/cluster-mobile.test.ts | 112 +++++ apps/web/test/cluster-operator.test.tsx | 200 +++++++++ e2e/built-web-server.ts | 22 +- e2e/cluster-operator.spec.ts | 395 ++++++++++++++++++ packages/client/test/cluster-operator.test.ts | 161 +++++++ scripts/tailnet-gateway.test.mjs | 55 +++ 7 files changed, 967 insertions(+), 7 deletions(-) create mode 100644 apps/web/test/cluster-mobile.test.ts create mode 100644 apps/web/test/cluster-operator.test.tsx create mode 100644 e2e/cluster-operator.spec.ts create mode 100644 packages/client/test/cluster-operator.test.ts diff --git a/apps/desktop/test/target-manager.test.ts b/apps/desktop/test/target-manager.test.ts index 627f3bf1..2567cfe2 100644 --- a/apps/desktop/test/target-manager.test.ts +++ b/apps/desktop/test/target-manager.test.ts @@ -9,6 +9,7 @@ import { import { commandId, confirmationId, + CLUSTER_OPERATOR_FEATURE, DEVICE_CAPABILITIES, hostId, revision, @@ -224,6 +225,34 @@ async function settlesBeforeTurnLimit(promise: Promise): Promise { } describe("desktop target manager boundaries", () => { + it("requests cluster.operator only after explicit desktop opt-in", async () => { + const defaultTransport = new Transport(); + const defaultRuntime = new DesktopTargetManager({ + cursorStore: new Store(), + localTransportFactory: () => defaultTransport as never, + events: { onEvent: () => {}, onState: () => {}, onError: () => {} }, + }); + await defaultRuntime.connect("local"); + const defaultHello = JSON.parse(defaultTransport.sent[0] ?? "{}") as { + readonly requestedFeatures?: readonly string[]; + }; + expect(defaultHello.requestedFeatures).not.toContain(CLUSTER_OPERATOR_FEATURE); + await defaultRuntime.close(); + + const enabledTransport = new Transport(); + const enabledRuntime = new DesktopTargetManager({ + cursorStore: new Store(), + clusterOperatorEnabled: true, + localTransportFactory: () => enabledTransport as never, + events: { onEvent: () => {}, onState: () => {}, onError: () => {} }, + }); + await enabledRuntime.connect("local"); + const enabledHello = JSON.parse(enabledTransport.sent[0] ?? "{}") as { + readonly requestedFeatures?: readonly string[]; + }; + expect(enabledHello.requestedFeatures).toContain(CLUSTER_OPERATOR_FEATURE); + await enabledRuntime.close(); + }); it("lists and connects named local profiles through profile-scoped transports", async () => { const transports: Transport[] = []; const requestedProfiles: string[] = []; diff --git a/apps/web/test/cluster-mobile.test.ts b/apps/web/test/cluster-mobile.test.ts new file mode 100644 index 00000000..4854280e --- /dev/null +++ b/apps/web/test/cluster-mobile.test.ts @@ -0,0 +1,112 @@ +import { CLUSTER_OPERATOR_FEATURE } from "@t4-code/protocol"; +import type { OmpClient, OmpClientOptions } from "@t4-code/client"; +import { afterEach, describe, expect, it } from "vite-plus/test"; + +import { createBrowserShellPort, detectBackend } from "../src/platform/browser-shell-port.ts"; +import { parseTailnetBackend } from "../src/platform/native-mobile-backend.ts"; + +const originalDocument = globalThis.document; +const originalWindow = globalThis.window; + +function backendScript(value: unknown): void { + Object.defineProperty(globalThis, "document", { + configurable: true, + value: { + getElementById: () => ({ textContent: JSON.stringify(value) }), + }, + }); + Object.defineProperty(globalThis, "window", { + configurable: true, + value: { location: { search: "" } }, + }); +} + +function captureClientOptions(): { + readonly options: () => OmpClientOptions; + readonly factory: (value: OmpClientOptions) => OmpClient; +} { + let captured: OmpClientOptions | undefined; + return { + options: () => { + if (captured === undefined) throw new Error("client was not built"); + return captured; + }, + factory: (value) => { + captured = value; + return { + state: "idle", + connect: async () => undefined, + close: async () => undefined, + wake: () => undefined, + onEvent: () => () => undefined, + onState: () => () => undefined, + onError: () => () => undefined, + } as unknown as OmpClient; + }, + }; +} + +afterEach(() => { + Object.defineProperty(globalThis, "document", { configurable: true, value: originalDocument }); + Object.defineProperty(globalThis, "window", { configurable: true, value: originalWindow }); +}); + +describe("mobile cluster target", () => { + it("keeps the saved Tailnet target operator-disabled by default", () => { + expect(parseTailnetBackend("https://operator.tailnet.ts.net")).toEqual({ + version: 3, + endpointKey: "https://operator.tailnet.ts.net#profile=default", + origin: "https://operator.tailnet.ts.net", + profileId: "default", + wsUrl: "wss://operator.tailnet.ts.net/v1/ws", + label: "T4 on operator", + }); + expect( + parseTailnetBackend("https://operator.tailnet.ts.net", "default", true), + ).toMatchObject({ + clusterOperatorEnabled: true, + wsUrl: "wss://operator.tailnet.ts.net/v1/ws", + }); + }); + + it("requests no cluster feature for an ordinary browser/mobile backend", async () => { + backendScript({ wsUrl: "wss://operator.tailnet.ts.net/v1/ws", label: "Operator" }); + const capture = captureClientOptions(); + const shell = createBrowserShellPort({ clientFactory: capture.factory }); + if (shell === null) throw new Error("shell was not created"); + await shell.bootstrap(); + + expect(capture.options().requestedFeatures).not.toContain(CLUSTER_OPERATOR_FEATURE); + expect(capture.options().compatibilityRequestedFeatures).not.toContain( + CLUSTER_OPERATOR_FEATURE, + ); + }); + + it("allows one explicit secure cluster target and rejects insecure or credentialed URLs", async () => { + backendScript({ + wsUrl: "wss://operator.tailnet.ts.net/v1/ws", + label: "Operator", + clusterOperatorEnabled: true, + }); + const capture = captureClientOptions(); + const shell = createBrowserShellPort({ clientFactory: capture.factory }); + if (shell === null) throw new Error("shell was not created"); + await shell.bootstrap(); + + expect(detectBackend()).toMatchObject({ clusterOperatorEnabled: true }); + expect(capture.options().requestedFeatures).toContain(CLUSTER_OPERATOR_FEATURE); + + backendScript({ + wsUrl: "ws://operator.tailnet.ts.net/v1/ws", + label: "Operator", + clusterOperatorEnabled: true, + }); + expect(() => detectBackend()).toThrow(/secure WSS cluster target/u); + backendScript({ + wsUrl: "wss://operator.tailnet.ts.net/v1/ws?token=secret", + label: "Operator", + clusterOperatorEnabled: true, + }); + expect(() => detectBackend()).toThrow(/query|credential/u); + }); +}); diff --git a/apps/web/test/cluster-operator.test.tsx b/apps/web/test/cluster-operator.test.tsx new file mode 100644 index 00000000..391242c9 --- /dev/null +++ b/apps/web/test/cluster-operator.test.tsx @@ -0,0 +1,200 @@ +import { + CI_TRIGGER_CAPABILITY, + CLUSTER_OPERATOR_FEATURE, + hostId, + revision, + sessionId, + type SessionRef, + type WorkspaceInfrastructureProjection, +} from "@t4-code/protocol"; +import { + createProjectionSnapshot, + type DesktopRuntimeController, + type DesktopRuntimeSnapshot, +} from "@t4-code/client"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { + clusterOperatorAvailability, + createClusterSession, + createClusterWorkspace, + runClusterCi, +} from "../src/features/targets/cluster-operator.ts"; +import { deriveWorkspaceData } from "../src/platform/live-workspace.ts"; + +const HOST = "cluster-host"; +const TARGET = "cluster-target"; +const workspace: WorkspaceInfrastructureProjection = { + id: "workspace-a", + displayName: "Release train", + phase: "Ready", + retentionPolicy: "Retain", + storageClass: "t4-workspaces-rwx", + capacity: "20Gi", + accessMode: "ReadWriteMany", + revision: revision("workspace-r2"), + condition: { + type: "StorageReady", + status: "True", + reason: "Bound", + message: "The RWX claim is bound.", + observedGeneration: 2, + }, +}; + +function snapshot(options: { + readonly enabled?: boolean; + readonly connected?: boolean; + readonly features?: readonly string[]; + readonly capabilities?: readonly string[]; + readonly session?: SessionRef; + readonly workspace?: WorkspaceInfrastructureProjection; +} = {}): DesktopRuntimeSnapshot { + const projection = createProjectionSnapshot(); + return { + version: 1, + integration: { kind: "omp", displayName: "OMP", level: "first-party" }, + platform: "linux", + desktopVersion: "test", + startState: "started", + clusterOperatorEnabled: options.enabled ?? false, + targets: new Map([[TARGET, { targetId: TARGET, label: "Cluster", kind: "remote", state: options.connected === false ? "disconnected" : "connected", paired: true }]]), + connections: new Map([[TARGET, options.connected === false ? "disconnected" : "connected"]]), + targetHosts: new Map([[TARGET, HOST]]), + hosts: new Map([[HOST, { + targetId: TARGET, + hostId: HOST, + ompVersion: "17.0.5", + ompBuild: "8476f445", + appserverVersion: "1", + appserverBuild: "test", + epoch: "host-epoch", + grantedCapabilities: [...(options.capabilities ?? ["sessions.read", "sessions.manage", "sessions.prompt", "sessions.control", CI_TRIGGER_CAPABILITY])], + grantedFeatures: [...(options.features ?? [CLUSTER_OPERATOR_FEATURE])], + negotiatedLimits: {}, + authentication: "paired", + resumed: false, + }]]), + catalogs: new Map(), + settings: new Map(), + projection: { + ...projection, + workspaces: new Map(options.workspace === undefined ? [] : [[`${HOST}\u0000${options.workspace.id}`, options.workspace]]), + workspaceCursors: new Map(options.workspace === undefined ? [] : [[HOST, { epoch: "workspace-epoch", seq: 2 }]]), + sessionIndex: new Map(options.session === undefined ? [] : [[`${HOST}\u0000${String(options.session.sessionId)}`, options.session]]), + sessionIndexMetadata: new Map(options.session === undefined ? [] : [[HOST, { totalCount: 1, truncated: false }]]), + }, + runtimeErrors: [], + } as DesktopRuntimeSnapshot; +} + +const session: SessionRef = { + hostId: hostId(HOST), + sessionId: sessionId("session-a"), + project: { projectId: "cluster/workspace-a" as never, name: "Release train" }, + revision: revision("session-r3"), + title: "Ship release", + status: "active", + updatedAt: "2026-07-20T12:00:00.000Z", + liveState: { + phase: "running", + cluster: { + workspaceId: workspace.id, + infrastructurePhase: "Running", + gui: { state: "Ready", previewId: "preview-a" }, + }, + ci: { + provider: "woodpecker", + correlation: "exact", + repositoryId: "repo-a", + branch: "main", + ref: "refs/heads/main", + commit: "0123456789abcdef", + pipelineNumber: 42, + status: "running", + currentStage: "verify", + startedAt: "2026-07-20T12:01:00.000Z", + deepLink: "https://ci.tailnet.ts.net/repos/repo-a/pipeline/42", + }, + }, +}; + +describe("cluster operator presentation", () => { + it("fails closed with exact disabled, transport, feature, capability, and revision reasons", () => { + expect(clusterOperatorAvailability(snapshot(), TARGET, "read")).toEqual({ + enabled: false, + reason: "Cluster operator is disabled in this app.", + }); + expect(clusterOperatorAvailability(snapshot({ enabled: true, connected: false }), TARGET, "read")).toEqual({ + enabled: false, + reason: "Reconnect this host to inspect cluster workspaces.", + }); + expect(clusterOperatorAvailability(snapshot({ enabled: true, features: [] }), TARGET, "read")).toEqual({ + enabled: false, + reason: "This host does not advertise cluster operator support.", + }); + expect(clusterOperatorAvailability(snapshot({ enabled: true, capabilities: [] }), TARGET, "read")).toEqual({ + enabled: false, + reason: "This host did not grant session read access.", + }); + expect(clusterOperatorAvailability(snapshot({ enabled: true, capabilities: ["sessions.read"] }), TARGET, "manage")).toEqual({ + enabled: false, + reason: "This host did not grant workspace and session management.", + }); + expect(clusterOperatorAvailability(snapshot({ enabled: true, capabilities: ["sessions.read", "sessions.manage"] }), TARGET, "ci", revision("session-r3"))).toEqual({ + enabled: false, + reason: "This host did not grant CI trigger access.", + }); + expect(clusterOperatorAvailability(snapshot({ enabled: true }), TARGET, "ci")).toEqual({ + enabled: false, + reason: "Waiting for the latest session revision.", + }); + }); + + it("derives infrastructure, CI, and GUI truth from canonical projections", () => { + const data = deriveWorkspaceData(snapshot({ enabled: true, workspace, session })); + + expect(data.clusterWorkspaces).toEqual([{ hostId: HOST, targetId: TARGET, infrastructure: workspace }]); + expect(data.sessions[0]).toMatchObject({ + cluster: session.liveState?.cluster, + ci: session.liveState?.ci, + }); + expect(data.sessions[0]?.ci?.currentStage).toBe("verify"); + expect(data.sessions[0]?.cluster?.gui).toEqual({ state: "Ready", previewId: "preview-a" }); + }); + + it("sends only allowlisted workspace, session, and CI arguments", async () => { + const command = vi.fn(async () => ({ accepted: true, result: {} })); + const controller = { command, getSnapshot: () => snapshot({ enabled: true, workspace, session }) } as unknown as DesktopRuntimeController; + + await createClusterWorkspace(controller, TARGET, HOST, { + displayName: "Release train", + retentionPolicy: "Retain", + capacity: "20Gi", + storageClass: "t4-workspaces-rwx", + repository: { repositoryId: "repo-a", ref: "refs/heads/main", commit: "0123456789abcdef" }, + }); + await createClusterSession(controller, TARGET, HOST, { + workspaceId: workspace.id, + title: "Ship release", + runtimeProfile: "default", + guiEnabled: true, + ci: { provider: "woodpecker", repositoryId: "repo-a", ref: "refs/heads/main", commit: "0123456789abcdef" }, + }); + await runClusterCi(controller, TARGET, HOST, "session-a", revision("session-r3"), { + provider: "woodpecker", + action: "run", + repositoryId: "repo-a", + ref: "refs/heads/main", + commit: "0123456789abcdef", + }); + + expect(command.mock.calls.map(([, intent]) => intent)).toEqual([ + { hostId: HOST, command: "workspace.create", args: expect.objectContaining({ displayName: "Release train", capacity: "20Gi" }) }, + { hostId: HOST, command: "session.create", args: expect.objectContaining({ workspaceId: workspace.id, guiEnabled: true }) }, + { hostId: HOST, sessionId: "session-a", command: "ci.run", expectedRevision: "session-r3", args: expect.objectContaining({ provider: "woodpecker", action: "run", repositoryId: "repo-a" }) }, + ]); + const serialized = JSON.stringify(command.mock.calls); + expect(serialized).not.toMatch(/token|secret|kubeconfig|namespace|image|url/iu); + }); +}); diff --git a/e2e/built-web-server.ts b/e2e/built-web-server.ts index d4661d1f..0df03bf9 100644 --- a/e2e/built-web-server.ts +++ b/e2e/built-web-server.ts @@ -17,11 +17,19 @@ const MIME_TYPES: Readonly> = { ".woff2": "font/woff2", }; -function injectBackend(html: string): string { - const payload = JSON.stringify({ - wsUrl: "ws://127.0.0.1:1/v1/ws", - label: "PWA fixture backend", - }); +interface BuiltWebBackend { + readonly wsUrl: string; + readonly label: string; + readonly clusterOperatorEnabled?: true; +} + +const DEFAULT_BACKEND: BuiltWebBackend = { + wsUrl: "ws://127.0.0.1:1/v1/ws", + label: "PWA fixture backend", +}; + +function injectBackend(html: string, backend: BuiltWebBackend): string { + const payload = JSON.stringify(backend); const tag = ``; if (!html.includes("")) throw new Error("web dist index is missing "); return html.replace("", `${tag}`); @@ -32,7 +40,7 @@ export class BuiltWebServer { private readonly server: Server; private port = 0; - constructor() { + constructor(private readonly backend: BuiltWebBackend = DEFAULT_BACKEND) { this.server = createServer((request, response) => { void this.handle(request.url ?? "/", request.method ?? "GET") .then(({ body, contentType, status }) => { @@ -96,7 +104,7 @@ export class BuiltWebServer { } if (pathname === "/" || pathname === "/index.html") { return { - body: injectBackend(await readFile(resolve(WEB_DIST, "index.html"), "utf8")), + body: injectBackend(await readFile(resolve(WEB_DIST, "index.html"), "utf8"), this.backend), contentType: MIME_TYPES[".html"]!, status: 200, }; diff --git a/e2e/cluster-operator.spec.ts b/e2e/cluster-operator.spec.ts new file mode 100644 index 00000000..cad284cd --- /dev/null +++ b/e2e/cluster-operator.spec.ts @@ -0,0 +1,395 @@ +import { type AddressInfo } from "node:net"; +import { expect, test, type Page } from "@playwright/test"; +import WebSocket, { WebSocketServer } from "ws"; + +import { BuiltWebServer } from "./built-web-server.ts"; + +const PUBLIC_WSS = "wss://operator-fixture.tailnet.ts.net/v1/ws"; +const HOST = "cluster-host"; +const SESSION = "cluster-session"; +const VIEW = `${HOST}/${SESSION}`; +const WORKSPACE = { + id: "workspace-a", + displayName: "Release train", + phase: "Ready", + retentionPolicy: "Retain", + storageClass: "t4-workspaces-rwx", + capacity: "20Gi", + accessMode: "ReadWriteMany", + revision: "workspace-r2", + condition: { + type: "StorageReady", + status: "True", + reason: "Bound", + message: "The RWX claim is bound.", + observedGeneration: 2, + }, +} as const; + +function sessionRef(ciStatus: "queued" | "running" | "success" = "running") { + return { + hostId: HOST, + sessionId: SESSION, + project: { projectId: "cluster/workspace-a", name: "Release train" }, + revision: "session-r3", + title: "Ship release", + status: "active", + updatedAt: "2026-07-20T12:00:00.000Z", + model: "gpt-proxy/gpt-5.6-sol", + liveState: { + phase: "running", + cluster: { + workspaceId: WORKSPACE.id, + infrastructurePhase: "Running", + gui: { state: "Ready", previewId: "preview-a" }, + }, + ci: { + provider: "woodpecker", + correlation: "exact", + repositoryId: "repo-a", + branch: "main", + ref: "refs/heads/main", + commit: "0123456789abcdef", + pipelineNumber: 42, + status: ciStatus, + currentStage: ciStatus === "success" ? "complete" : "verify", + startedAt: "2026-07-20T12:01:00.000Z", + deepLink: "https://ci.tailnet.ts.net/repos/repo-a/pipeline/42", + }, + }, + }; +} + +class OperatorWireFixture { + readonly commands: Array> = []; + readonly hellos: Array> = []; + private readonly server = new WebSocketServer({ host: "127.0.0.1", port: 0, path: "/v1/ws" }); + private workspaceSeq = 2; + private ciStatus: "queued" | "running" | "success" = "running"; + + get url(): string { + const address = this.server.address() as AddressInfo | null; + if (address === null) throw new Error("operator fixture is not listening"); + return `ws://127.0.0.1:${address.port}/v1/ws`; + } + + async start(): Promise { + if (this.server.address() !== null) return; + await new Promise((resolve, reject) => { + this.server.once("listening", resolve); + this.server.once("error", reject); + }); + this.server.on("connection", (socket) => { + socket.on("message", (data) => this.receive(socket, JSON.parse(data.toString()) as Record)); + }); + } + + async stop(): Promise { + for (const socket of this.server.clients) socket.terminate(); + await new Promise((resolve) => this.server.close(() => resolve())); + } + + disconnectClients(): void { + for (const socket of this.server.clients) socket.close(1012, "fixture restart"); + } + + private send(socket: WebSocket, frame: unknown): void { + socket.send(JSON.stringify(frame)); + } + + private response(socket: WebSocket, frame: Record, result: unknown): void { + this.send(socket, { + v: "omp-app/1", + type: "response", + requestId: frame.requestId, + commandId: frame.commandId, + hostId: HOST, + ...(frame.sessionId === undefined ? {} : { sessionId: frame.sessionId }), + command: frame.command, + ok: true, + result, + }); + } + + private receive(socket: WebSocket, frame: Record): void { + if (frame.type === "hello") { + this.hellos.push(frame); + this.send(socket, { + v: "omp-app/1", + type: "welcome", + selectedProtocol: "omp-app/1", + hostId: HOST, + ompVersion: "17.0.5", + ompBuild: "8476f4451ed95c5d5401785d279a93d3c659fac4", + appserverVersion: "cluster-fixture", + appserverBuild: "redacted", + epoch: "cluster-epoch-1", + grantedCapabilities: [ + "sessions.read", + "sessions.manage", + "sessions.prompt", + "sessions.control", + "preview.read", + "preview.control", + "preview.input", + "ci.trigger", + ], + grantedFeatures: ["resume", "host.watch", "cluster.operator", "preview.control"], + negotiatedLimits: {}, + authentication: "paired", + resumed: this.hellos.length > 1, + }); + return; + } + if (frame.type !== "command") return; + this.commands.push(frame); + switch (frame.command) { + case "session.list": + this.response(socket, frame, { + cursor: { epoch: "session-index-1", seq: 3 }, + sessions: [sessionRef(this.ciStatus)], + totalCount: 1, + truncated: false, + }); + return; + case "host.watch": + this.response(socket, frame, { watching: true, cursor: { epoch: "session-index-1", seq: 3 } }); + return; + case "workspace.list": + this.response(socket, frame, { + cursor: { epoch: "workspace-index-1", seq: this.workspaceSeq }, + workspaces: [WORKSPACE], + }); + this.send(socket, { + v: "omp-app/1", + type: "workspace.state", + hostId: HOST, + workspaceId: WORKSPACE.id, + cursor: { epoch: "workspace-index-1", seq: this.workspaceSeq }, + revision: WORKSPACE.revision, + upsert: WORKSPACE, + }); + return; + case "session.attach": + this.response(socket, frame, { attached: true, cursor: { epoch: "transcript-1", seq: 1 } }); + this.send(socket, { + v: "omp-app/1", + type: "snapshot", + hostId: HOST, + sessionId: SESSION, + cursor: { epoch: "transcript-1", seq: 1 }, + revision: "session-r3", + entries: [], + }); + this.send(socket, { + v: "omp-app/1", + type: "agent", + hostId: HOST, + sessionId: SESSION, + agentId: "Main", + state: "running", + progress: 0.4, + detail: { kind: "main", title: "Main", startedAt: "2026-07-20T12:00:00.000Z" }, + }); + this.send(socket, { + v: "omp-app/1", + type: "agent", + hostId: HOST, + sessionId: SESSION, + agentId: "WorkerA", + state: "running", + progress: 0.7, + detail: { + parentId: "Main", + title: "WorkerA", + description: "Verify CI and wait for peer reply", + currentTool: "irc.wait", + evidence: "Parked peer revived; owner-scoped job still running", + startedAt: "2026-07-20T12:00:10.000Z", + }, + }); + this.send(socket, { + v: "omp-app/1", + type: "preview.state", + hostId: HOST, + sessionId: SESSION, + previewId: "preview-a", + state: "ready", + url: "https://127.0.0.1:4173/", + revision: "preview-r1", + cursor: { epoch: "preview-1", seq: 1 }, + title: "Session GUI", + canGoBack: false, + canGoForward: false, + viewport: { width: 1280, height: 720, deviceScaleFactor: 1 }, + availableActions: ["navigate", "capture", "click", "fill", "type", "press", "scroll"], + authority: { + id: "omp-session", + label: "Session", + kind: "isolated-session", + requiresExplicitOptIn: false, + }, + }); + return; + case "session.steer": + case "session.prompt": + this.response(socket, frame, { accepted: true }); + return; + case "preview.policy.check": + this.response(socket, frame, { allowed: true }); + return; + case "preview.lease.acquire": + this.response(socket, frame, { + previewId: "preview-a", + leaseId: "preview-lease-a", + expiresAt: Date.now() + 30_000, + }); + return; + case "preview.lease.release": + this.response(socket, frame, { previewId: "preview-a", released: true }); + return; + case "preview.type": + case "preview.fill": + case "preview.press": + case "preview.scroll": + this.response(socket, frame, { accepted: true }); + return; + case "ci.run": + this.ciStatus = "queued"; + this.response(socket, frame, { + provider: "woodpecker", + correlation: "exact", + pipelineNumber: 43, + status: "queued", + }); + return; + default: + this.response(socket, frame, {}); + } + } +} + +let wire: OperatorWireFixture; +let web: BuiltWebServer; + +async function routeFixtureWss(page: Page): Promise { + await page.addInitScript( + ({ publicUrl, routedUrl }) => { + const NativeWebSocket = window.WebSocket; + class RoutedWebSocket extends NativeWebSocket { + constructor(url: string | URL, protocols?: string | string[]) { + super(String(url) === publicUrl ? routedUrl : url, protocols); + } + } + Object.defineProperties(RoutedWebSocket, { + CONNECTING: { value: NativeWebSocket.CONNECTING }, + OPEN: { value: NativeWebSocket.OPEN }, + CLOSING: { value: NativeWebSocket.CLOSING }, + CLOSED: { value: NativeWebSocket.CLOSED }, + }); + window.WebSocket = RoutedWebSocket; + }, + { publicUrl: PUBLIC_WSS, routedUrl: wire.url }, + ); +} + +async function openSession(page: Page, mobile = false): Promise { + await routeFixtureWss(page); + await page.goto(web.url, { waitUntil: "domcontentloaded" }); + if (mobile) await page.getByRole("button", { name: "Show session list" }).click(); + const row = page.locator(`[data-session-row="${VIEW}"]`); + await expect(row).toBeVisible(); + await row.click(); + await expect(page).toHaveURL(/#\/sessions\/cluster-host%2Fcluster-session|#\/sessions\/cluster-host\/cluster-session/u); +} + +test.describe("OMP/T4 cluster GUI boundaries", () => { + test.describe.configure({ mode: "serial" }); + + test.beforeAll(async () => { + wire = new OperatorWireFixture(); + await wire.start(); + web = new BuiltWebServer({ + wsUrl: PUBLIC_WSS, + label: "Redacted cluster fixture", + clusterOperatorEnabled: true, + }); + await web.start(); + }); + + test.afterAll(async () => { + await web?.stop(); + await wire?.stop(); + }); + + test("desktop attach, steer, and reconnect preserve canonical cursors without duplicate rows", async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await openSession(page); + const composer = page.getByRole("textbox", { name: "Message the session" }); + await composer.fill("Keep the current approach"); + await page.getByRole("button", { name: "Steer", exact: true }).click(); + await expect.poll(() => wire.commands.some((frame) => frame.command === "session.steer")).toBe(true); + + const helloCount = wire.hellos.length; + wire.disconnectClients(); + await expect.poll(() => wire.hellos.length).toBeGreaterThan(helloCount); + await expect(page.locator(`[data-session-row="${VIEW}"]`)).toHaveCount(1); + expect(wire.hellos.at(-1)?.requestedFeatures).toContain("cluster.operator"); + }); + + test("Agent View keeps exact OMP ids, parentage, attention, and progress alongside exact CI state", async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 900 }); + await openSession(page); + await page.goto(`${web.url}#/agents`); + await expect(page.getByRole("heading", { name: "Agent View" })).toBeVisible(); + await expect(page.getByRole("heading", { name: "Main", exact: true })).toBeVisible(); + await expect(page.getByRole("heading", { name: "WorkerA", exact: true })).toBeVisible(); + await expect(page.getByText("Parent: Main", { exact: false })).toBeVisible(); + await expect(page.getByRole("progressbar", { name: "70% done" })).toBeVisible(); + await expect(page.getByText("Parked peer revived; owner-scoped job still running", { exact: true })).toBeVisible(); + await expect(page.getByText("verify", { exact: true }).first()).toBeVisible(); + await expect(page.getByText("running", { exact: true }).first()).toBeVisible(); + }); + + test("phone uses one secure WSS target and reconnect replay does not duplicate workspace or session rows", async ({ page }) => { + await page.emulateMedia({ reducedMotion: "reduce" }); + await page.setViewportSize({ width: 390, height: 844 }); + await openSession(page, true); + expect(wire.hellos.at(-1)?.requestedFeatures).toContain("cluster.operator"); + const helloCount = wire.hellos.length; + wire.disconnectClients(); + await expect.poll(() => wire.hellos.length).toBeGreaterThan(helloCount); + await page.getByRole("button", { name: "Show session list" }).click(); + await expect(page.locator(`[data-session-row="${VIEW}"]`)).toHaveCount(1); + await page.goto(`${web.url}#/hosts`); + await expect(page.locator('[data-cluster-workspace-id="workspace-a"]')).toHaveCount(1); + await expect(page.getByText("t4-workspaces-rwx", { exact: true })).toBeVisible(); + await expect(page.getByRole("heading", { name: "Cluster workspaces" })).toBeVisible(); + const createWorkspace = page.getByRole("button", { name: "Create cluster workspace" }); + await expect(createWorkspace).toBeVisible(); + const createBox = await createWorkspace.boundingBox(); + expect(createBox).not.toBeNull(); + expect(createBox!.height).toBeGreaterThanOrEqual(43.99); + expect( + await page.evaluate(() => + getComputedStyle(document.documentElement).getPropertyValue("--motion-duration-fast").trim(), + ), + ).toBe("0ms"); + }); + + test("phone Browser Preview gates input through lease/revision and denies cross-session GUI routes", async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await openSession(page, true); + await page.goto(`${web.url}#/sessions/${encodeURIComponent(VIEW)}/preview`); + await expect(page.getByRole("heading", { name: "Browser preview" })).toBeVisible(); + await page.getByRole("textbox", { name: "Text" }).fill("hello from phone"); + await page.getByRole("button", { name: "Type", exact: true }).click(); + await expect.poll(() => wire.commands.some((frame) => frame.command === "preview.lease.acquire")).toBe(true); + await expect.poll(() => wire.commands.some((frame) => frame.command === "preview.type")).toBe(true); + + await page.setViewportSize({ width: 844, height: 390 }); + await expect(page.getByRole("heading", { name: "Browser preview" })).toBeVisible(); + await page.goto(`${web.url}#/sessions/${encodeURIComponent(`${HOST}/other-session`)}/preview`); + await expect(page.getByRole("heading", { name: "Browser preview" })).toHaveCount(0); + }); +}); diff --git a/packages/client/test/cluster-operator.test.ts b/packages/client/test/cluster-operator.test.ts new file mode 100644 index 00000000..fd05817c --- /dev/null +++ b/packages/client/test/cluster-operator.test.ts @@ -0,0 +1,161 @@ +import { + CI_TRIGGER_CAPABILITY, + CLUSTER_OPERATOR_FEATURE, + hostId, + revision, + type WorkspaceStateFrame, +} from "@t4-code/protocol"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { bootstrapDesktopHost } from "../src/desktop-runtime-bootstrap.ts"; +import { + DEFAULT_CLUSTER_OPERATOR_ENABLED, + applyPublicFrame, + clusterOperatorRequestedFeatures, + createProjectionSnapshot, + type ProjectionFrame, +} from "../src/index.ts"; + +const HOST = hostId("cluster-host"); +const welcome = { + hostId: HOST, + grantedCapabilities: ["sessions.read", "sessions.manage", CI_TRIGGER_CAPABILITY], + grantedFeatures: [CLUSTER_OPERATOR_FEATURE, "host.watch"], +} as never; + +function workspaceState( + seq: number, + phase: "Pending" | "Ready" | "Failed" | "Terminating" | "Unknown", + workspaceId = "workspace-a", +): WorkspaceStateFrame { + return { + v: "omp-app/1", + type: "workspace.state", + hostId: HOST, + workspaceId, + cursor: { epoch: "workspace-epoch", seq }, + revision: revision(`workspace-r${seq}`), + upsert: { + id: workspaceId, + displayName: "Operator workspace", + phase, + retentionPolicy: "Retain", + capacity: "20Gi", + storageClass: "rwx", + accessMode: "ReadWriteMany", + revision: revision(`workspace-r${seq}`), + }, + } as WorkspaceStateFrame; +} + +describe("cluster operator client contract", () => { + it("is source-default-off and requests the one feature only after explicit opt-in", () => { + expect(DEFAULT_CLUSTER_OPERATOR_ENABLED).toBe(false); + expect(clusterOperatorRequestedFeatures(["resume", CLUSTER_OPERATOR_FEATURE])).toEqual([ + "resume", + ]); + expect( + clusterOperatorRequestedFeatures(["resume", CLUSTER_OPERATOR_FEATURE], true), + ).toEqual(["resume", CLUSTER_OPERATOR_FEATURE]); + }); + + it("does not issue cluster bootstrap commands while the local option is off", async () => { + const issue = vi.fn(async () => ({ + accepted: true, + result: { cursor: { epoch: "session-epoch", seq: 1 }, sessions: [] }, + })) as never; + + await bootstrapDesktopHost({ targetId: "cluster", frame: welcome, issue }); + + expect(issue).toHaveBeenCalledTimes(2); + expect(issue.mock.calls.map(([intent]) => intent.command)).toEqual([ + "session.list", + "host.watch", + ]); + expect(issue).not.toHaveBeenCalledWith( + expect.objectContaining({ command: "workspace.list" }), + ); + }); + + it("bootstraps cluster workspace state only when enabled and advertised", async () => { + const commands: string[] = []; + const issue = vi.fn(async (intent: { readonly command: string }) => { + commands.push(intent.command); + return { + accepted: true, + result: + intent.command === "session.list" + ? { cursor: { epoch: "session-epoch", seq: 1 }, sessions: [] } + : intent.command === "workspace.list" + ? { + cursor: { epoch: "workspace-epoch", seq: 4 }, + workspaces: [workspaceState(4, "Ready").upsert], + } + : {}, + }; + }) as never; + + await bootstrapDesktopHost({ + targetId: "cluster", + frame: welcome, + issue, + clusterOperatorEnabled: true, + }); + + expect(commands).toEqual(["session.list", "host.watch", "workspace.list"]); + }); + + it("keeps workspace cursors independent and drops duplicate replay", () => { + let state = createProjectionSnapshot(); + state = applyPublicFrame(state, workspaceState(1, "Pending") as ProjectionFrame); + const first = state.workspaces.get(`${String(HOST)}\u0000workspace-a`); + expect(first?.phase).toBe("Pending"); + expect(state.workspaceCursors.get(String(HOST))).toEqual({ + epoch: "workspace-epoch", + seq: 1, + }); + + state = applyPublicFrame(state, workspaceState(1, "Failed") as ProjectionFrame); + expect(state.workspaces.get(`${String(HOST)}\u0000workspace-a`)).toBe(first); + + state = applyPublicFrame( + state, + { + v: "omp-app/1", + type: "sessions", + hostId: HOST, + cursor: { epoch: "session-epoch", seq: 900 }, + sessions: [], + totalCount: 0, + truncated: false, + } as ProjectionFrame, + ); + state = applyPublicFrame(state, workspaceState(2, "Ready") as ProjectionFrame); + expect(state.workspaces.get(`${String(HOST)}\u0000workspace-a`)?.phase).toBe("Ready"); + expect(state.workspaceCursors.get(String(HOST))?.seq).toBe(2); + }); + + it("applies exact workspace removals without disturbing another host", () => { + const hostB = hostId("cluster-host-b"); + let state = applyPublicFrame(createProjectionSnapshot(), workspaceState(1, "Ready") as ProjectionFrame); + state = applyPublicFrame( + state, + { ...workspaceState(1, "Ready", "workspace-b"), hostId: hostB } as ProjectionFrame, + ); + state = applyPublicFrame( + state, + { + v: "omp-app/1", + type: "workspace.state", + hostId: HOST, + workspaceId: "workspace-a", + cursor: { epoch: "workspace-epoch", seq: 2 }, + revision: revision("workspace-r2"), + remove: "workspace-a", + } as WorkspaceStateFrame as ProjectionFrame, + ); + + expect(state.workspaces.has(`${String(HOST)}\u0000workspace-a`)).toBe(false); + expect(state.workspaces.has(`${String(hostB)}\u0000workspace-b`)).toBe(true); + }); +}); diff --git a/scripts/tailnet-gateway.test.mjs b/scripts/tailnet-gateway.test.mjs index 7373e33f..d36810e2 100644 --- a/scripts/tailnet-gateway.test.mjs +++ b/scripts/tailnet-gateway.test.mjs @@ -12,6 +12,7 @@ import { CAPACITOR_NATIVE_ORIGINS, cacheControlForStaticPath, injectBackendConfig, + normalizeClusterWebSocketUrl, normalizeAllowedOrigin, normalizeDeploymentIdentity, normalizeNativeAllowedOrigins, @@ -164,6 +165,60 @@ test("backend injection is explicit, credential-free, and script-safe", () => { assert.doesNotMatch(injected, /token|password|credential/iu); }); +test("cluster gateway configuration is default-off and adds no implicit target", () => { + const options = optionsFromEnvironment({ + T4_ALLOWED_ORIGIN: ALLOWED_ORIGIN, + T4_DEPLOYMENT_IDENTITY: DEPLOYMENT_IDENTITY, + XDG_RUNTIME_DIR: "/run/user/1000", + }); + assert.equal(options.clusterOperatorEnabled, false); + assert.equal(options.clusterWsUrl, undefined); + + const injected = injectBackendConfig("", { + allowedOrigin: ALLOWED_ORIGIN, + label: "Ordinary host", + }); + const payload = JSON.parse(/\n`; return `${indexHtml.slice(0, offset)}${script}${indexHtml.slice(offset)}`; @@ -202,8 +228,8 @@ export function safeStaticPath(webRoot, pathname) { return candidate === root || candidate.startsWith(`${root}${sep}`) ? candidate : undefined; } -function gatewayCsp(allowedOrigin) { - const websocketOrigin = new URL(websocketUrlForOrigin(allowedOrigin)).origin; +function gatewayCsp(allowedOrigin, clusterWsUrl) { + const websocketOrigin = new URL(clusterWsUrl ?? websocketUrlForOrigin(allowedOrigin)).origin; return [ "default-src 'self'", "script-src 'self'", @@ -219,8 +245,8 @@ function gatewayCsp(allowedOrigin) { ].join("; "); } -function applySecurityHeaders(response, allowedOrigin) { - response.setHeader("Content-Security-Policy", gatewayCsp(allowedOrigin)); +function applySecurityHeaders(response, allowedOrigin, clusterWsUrl) { + response.setHeader("Content-Security-Policy", gatewayCsp(allowedOrigin, clusterWsUrl)); response.setHeader("Permissions-Policy", "camera=(), geolocation=(), microphone=(), payment=(), usb=()"); response.setHeader("Referrer-Policy", "no-referrer"); response.setHeader("X-Content-Type-Options", "nosniff"); @@ -491,6 +517,11 @@ export async function startTailnetGateway(input) { nativeAllowedOrigins: normalizeNativeAllowedOrigins(input.nativeAllowedOrigins), label: input.label ?? "OMP on this Tailnet host", deploymentIdentity: normalizeDeploymentIdentity(input.deploymentIdentity), + clusterOperatorEnabled: input.clusterOperatorEnabled === true, + clusterWsUrl: + input.clusterOperatorEnabled === true + ? normalizeClusterWebSocketUrl(input.clusterWsUrl) + : undefined, heartbeatIntervalMs: input.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS, profiles: normalizeProfileRoutes(input.profileRoutes ?? input.profiles ?? []), startProfiles: input.startProfiles === true || input.enableProfileStarts === true, @@ -527,7 +558,7 @@ export async function startTailnetGateway(input) { }); const server = createServer((request, response) => { void (async () => { - applySecurityHeaders(response, options.allowedOrigin); + applySecurityHeaders(response, options.allowedOrigin, options.clusterWsUrl); const pathname = requestPath(request.url); if (pathname === "/healthz") { const [web, resolvedAppSocket] = await Promise.all([ @@ -657,6 +688,17 @@ export function optionsFromEnvironment(environment = process.env) { throw new Error("T4_PROFILE_ROUTES must be valid JSON"); } } + const clusterOperatorEnabled = environment.T4_CLUSTER_OPERATOR_ENABLED === "true"; + if ( + environment.T4_CLUSTER_OPERATOR_ENABLED !== undefined && + environment.T4_CLUSTER_OPERATOR_ENABLED !== "false" && + !clusterOperatorEnabled + ) { + throw new Error("T4_CLUSTER_OPERATOR_ENABLED must be true or false"); + } + const clusterWsUrl = clusterOperatorEnabled + ? normalizeClusterWebSocketUrl(environment.T4_CLUSTER_WS_URL) + : undefined; return { webRoot: environment.T4_WEB_ROOT ?? resolve(scriptDirectory, "..", "apps", "web", "dist"), appSocket: environment.T4_APP_SERVER_SOCKET ?? defaultSocketPath(environment), @@ -669,6 +711,8 @@ export function optionsFromEnvironment(environment = process.env) { : environment.T4_NATIVE_ALLOWED_ORIGINS.split(","), label: environment.T4_HOST_LABEL ?? "OMP on this Tailnet host", deploymentIdentity: environment.T4_DEPLOYMENT_IDENTITY, + clusterOperatorEnabled, + clusterWsUrl, profileRoutes, startProfiles: environment.T4_ENABLE_PROFILE_STARTS === "1", }; From a2a6c5d04a08e1ac95a3184eb841a53a0c1e3685 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 16:01:22 +0000 Subject: [PATCH 11/98] ci(cluster): hand chart contracts to Helm stage --- .woodpecker.yml | 8 ++++++-- .../cluster-ci/cluster-ci-contract.test.mjs | 20 ++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index a86bbb22..0e9bac05 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -202,7 +202,9 @@ steps: image: mirror.gcr.io/library/golang:1.25.1-bookworm directory: packages/cluster-operator commands: - - go test ./... + - go test ./api/... ./controllers/... ./cmd/... + - mkdir -p ../../artifacts/cluster-proof + - CGO_ENABLED=0 go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -219,8 +221,10 @@ steps: cluster-chart-tests: depends_on: [cluster-operator-tests] image: mirror.gcr.io/alpine/helm:3.19.0 + directory: packages/cluster-operator/charttests commands: - - helm lint deploy/charts/t4-cluster + - helm lint ../../../deploy/charts/t4-cluster + - ../../../artifacts/cluster-proof/chart-contract.test backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted diff --git a/scripts/cluster-ci/cluster-ci-contract.test.mjs b/scripts/cluster-ci/cluster-ci-contract.test.mjs index 4c2d40f6..6bcf9d07 100644 --- a/scripts/cluster-ci/cluster-ci-contract.test.mjs +++ b/scripts/cluster-ci/cluster-ci-contract.test.mjs @@ -210,9 +210,23 @@ test("Woodpecker keeps upstream gates and serializes bounded cluster publication steps["android-debug"].commands.includes("pnpm --filter @t4-code/mobile check:android:debug"), ); assert.ok(steps["cluster-ci-contracts"].commands.includes("pnpm test:cluster:ci")); - assert.ok(steps["cluster-server-tests"].commands.includes("pnpm --filter @t4-code/cluster-server test")); - assert.ok(steps["cluster-operator-tests"].commands.includes("go test ./...")); - assert.ok(steps["cluster-chart-tests"].commands.includes("helm lint deploy/charts/t4-cluster")); + assert.ok( + steps["cluster-operator-tests"].commands.includes("go test ./api/... ./controllers/... ./cmd/..."), + ); + assert.ok( + steps["cluster-operator-tests"].commands.some((command) => + command.includes("go test -c ./charttests"), + ), + ); + assert.ok( + steps["cluster-chart-tests"].commands.includes("helm lint ../../../deploy/charts/t4-cluster"), + ); + assert.ok( + steps["cluster-chart-tests"].commands.some((command) => command.endsWith("chart-contract.test")), + ); + assert.ok( + steps["cluster-server-tests"].commands.includes("pnpm --filter @t4-code/cluster-server test"), + ); assert.equal(JSON.stringify(pipeline).includes("from_secret"), false); assert.deepEqual(steps["harbor-auth"].depends_on, ["cluster-chart-tests", "android-debug"]); assert.equal( From bb5b0d36bf2ea8ae43067f2e90f1bc2e7df1ae3a Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 16:05:20 +0000 Subject: [PATCH 12/98] feat(cluster): add stateless cluster gateway --- packages/cluster-server/package.json | 27 ++ packages/cluster-server/src/config.ts | 133 +++++ packages/cluster-server/src/gateway.ts | 355 ++++++++++++++ packages/cluster-server/src/index.ts | 13 + .../cluster-server/src/kubernetes-client.ts | 285 +++++++++++ .../src/kubernetes-projection.ts | 455 ++++++++++++++++++ .../cluster-server/src/kubernetes-runner.ts | 120 +++++ packages/cluster-server/src/main.ts | 97 ++++ packages/cluster-server/src/observability.ts | 117 +++++ .../cluster-server/src/pod-host-router.ts | 131 +++++ packages/cluster-server/src/server.ts | 117 +++++ .../src/session-authority-runner.ts | 100 ++++ .../cluster-server/src/session-host-main.ts | 114 +++++ .../cluster-server/src/session-host-policy.ts | 95 ++++ packages/cluster-server/src/woodpecker.ts | 210 ++++++++ packages/cluster-server/test/config.test.ts | 34 ++ packages/cluster-server/tsconfig.json | 8 + packages/host-service/src/remote/listener.ts | 20 +- packages/host-service/src/remote/types.ts | 2 + packages/host-service/src/server.ts | 10 +- packages/host-wire/src/additive.ts | 4 + packages/host-wire/src/capabilities.ts | 2 + packages/host-wire/src/cluster.ts | 361 ++++++++++++++ packages/host-wire/src/command.ts | 34 +- packages/host-wire/src/index.ts | 1 + packages/host-wire/src/session-index.ts | 11 + packages/protocol/src/server-event.ts | 1 + 27 files changed, 2846 insertions(+), 11 deletions(-) create mode 100644 packages/cluster-server/package.json create mode 100644 packages/cluster-server/src/config.ts create mode 100644 packages/cluster-server/src/gateway.ts create mode 100644 packages/cluster-server/src/index.ts create mode 100644 packages/cluster-server/src/kubernetes-client.ts create mode 100644 packages/cluster-server/src/kubernetes-projection.ts create mode 100644 packages/cluster-server/src/kubernetes-runner.ts create mode 100755 packages/cluster-server/src/main.ts create mode 100644 packages/cluster-server/src/observability.ts create mode 100644 packages/cluster-server/src/pod-host-router.ts create mode 100644 packages/cluster-server/src/server.ts create mode 100644 packages/cluster-server/src/session-authority-runner.ts create mode 100755 packages/cluster-server/src/session-host-main.ts create mode 100644 packages/cluster-server/src/session-host-policy.ts create mode 100644 packages/cluster-server/src/woodpecker.ts create mode 100644 packages/cluster-server/test/config.test.ts create mode 100644 packages/cluster-server/tsconfig.json create mode 100644 packages/host-wire/src/cluster.ts diff --git a/packages/cluster-server/package.json b/packages/cluster-server/package.json new file mode 100644 index 00000000..537956da --- /dev/null +++ b/packages/cluster-server/package.json @@ -0,0 +1,27 @@ +{ + "name": "@t4-code/cluster-server", + "version": "0.1.30", + "private": true, + "type": "module", + "description": "Stateless Kubernetes-backed omp-app/1 cluster gateway", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { ".": { "types": "./src/index.ts", "import": "./src/index.ts" }, "./*": { "types": "./src/*.ts", "import": "./src/*.ts" } }, + "scripts": { + "build": "bun run typecheck", + "check": "bun run typecheck", + "test": "bun test test", + "typecheck": "tsgo -p tsconfig.json --noEmit" + }, + "dependencies": { + "@t4-code/host-service": "workspace:*", + "@t4-code/host-wire": "workspace:*" + }, + "devDependencies": { + "@types/bun": "1.3.5", + "@types/node": "24.12.4", + "typescript": "~6.0.3", + "vite-plus": "0.2.2" + }, + "engines": { "bun": ">=1.3.14" } +} diff --git a/packages/cluster-server/src/config.ts b/packages/cluster-server/src/config.ts new file mode 100644 index 00000000..958680fa --- /dev/null +++ b/packages/cluster-server/src/config.ts @@ -0,0 +1,133 @@ +import { readFile } from "node:fs/promises"; +import { isIP } from "node:net"; + +const DNS_LABEL = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; +export interface ClusterServerConfig { + readonly namespace: string; + readonly podName: string; + readonly epoch: string; + readonly hostName: string; + readonly gatewayPort: number; + readonly adminPort: number; + readonly kubernetesBaseUrl: string; + readonly kubernetesTokenPath: string; + readonly kubernetesCaPath: string; + readonly internalToken: string; + readonly trustedProxyAddresses: readonly string[]; + readonly trustedProxyCidrs: readonly string[]; + readonly woodpecker?: { + readonly baseUrl: string; + readonly token: string; + readonly repositories: Readonly>; + }; +} +function required(env: Readonly>, name: string): string { + const value = env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} +function dns(value: string, name: string): string { + if (!DNS_LABEL.test(value)) throw new Error(`${name} is invalid`); + return value; +} +function port(value: string | undefined, fallback: number, name: string): number { + const result = Number(value ?? fallback); + if (!Number.isSafeInteger(result) || result < 1 || result > 65_535) throw new Error(`${name} is invalid`); + return result; +} +function repositories(value: string): Readonly> { + if (new TextEncoder().encode(value).byteLength > 65_536) throw new Error("T4_WOODPECKER_REPOSITORIES exceeds limit"); + let input: unknown; + try { input = JSON.parse(value); } catch { throw new Error("T4_WOODPECKER_REPOSITORIES is invalid JSON"); } + if (!input || typeof input !== "object" || Array.isArray(input) || Object.keys(input).length > 128) + throw new Error("T4_WOODPECKER_REPOSITORIES is invalid"); + const output: Record = {}; + for (const [id, raw] of Object.entries(input as Record)) { + if (!raw || typeof raw !== "object" || Array.isArray(raw) || Object.keys(raw).length !== 1 || typeof (raw as Record).slug !== "string") + throw new Error("T4_WOODPECKER_REPOSITORIES entry is invalid"); + output[id] = { slug: (raw as { slug: string }).slug }; + } + return Object.freeze(output); +} +function proxyAddresses(value: string | undefined): readonly string[] { + if (!value) return []; + const values = [...new Set(value.split(",").map(item => item.trim()))]; + if (values.length > 64 || values.some(item => !item || isIP(item) === 0)) + throw new Error("T4_CLUSTER_TRUSTED_PROXY_ADDRESSES is invalid"); + return Object.freeze(values); +} +function ipv6Value(value: string): bigint { + const halves = value.split("::"); + if (halves.length > 2) throw new Error("invalid IPv6 address"); + const left = halves[0] ? halves[0].split(":") : []; + const right = halves[1] ? halves[1].split(":") : []; + const words = [...left, ...Array(8 - left.length - right.length).fill("0"), ...right]; + if (words.length !== 8) throw new Error("invalid IPv6 address"); + return words.reduce((result, word) => (result << 16n) | BigInt(Number.parseInt(word, 16)), 0n); +} +function canonicalCidr(value: string): string { + const pieces = value.split("/"); + if (pieces.length !== 2 || !/^(?:0|[1-9][0-9]*)$/u.test(pieces[1]!)) + throw new Error("T4_CLUSTER_TRUSTED_PROXY_CIDRS is invalid"); + const address = pieces[0]!; + const family = isIP(address); + const prefix = Number(pieces[1]); + const width = family === 4 ? 32 : family === 6 ? 128 : 0; + if (prefix > width) throw new Error("T4_CLUSTER_TRUSTED_PROXY_CIDRS is invalid"); + const canonicalAddress = family === 4 + ? address.split(".").map(Number).join(".") + : new URL(`http://[${address}]/`).hostname.slice(1, -1); + if (canonicalAddress !== address.toLowerCase()) throw new Error("T4_CLUSTER_TRUSTED_PROXY_CIDRS must use canonical network addresses"); + const numeric = family === 4 + ? address.split(".").map(Number).reduce((result, octet) => (result << 8n) | BigInt(octet), 0n) + : ipv6Value(address); + const hostBits = BigInt(width - prefix); + if (hostBits > 0n && (numeric & ((1n << hostBits) - 1n)) !== 0n) + throw new Error("T4_CLUSTER_TRUSTED_PROXY_CIDRS must use canonical network addresses"); + return `${canonicalAddress}/${prefix}`; +} +function proxyCidrs(value: string | undefined): readonly string[] { + if (!value) return []; + const values = [...new Set(value.split(",").map(item => canonicalCidr(item.trim())))]; + if (values.length > 64) throw new Error("T4_CLUSTER_TRUSTED_PROXY_CIDRS exceeds limit"); + return Object.freeze(values); +} + + +export function clusterServerConfigFromEnv(env: Readonly>): ClusterServerConfig { + const namespace = dns(required(env, "POD_NAMESPACE"), "POD_NAMESPACE"); + const podName = dns(required(env, "POD_NAME"), "POD_NAME"); + const podUid = required(env, "POD_UID"); + if (!/^[A-Za-z0-9-]{8,128}$/u.test(podUid)) throw new Error("POD_UID is invalid"); + const serviceHost = required(env, "KUBERNETES_SERVICE_HOST"); + const servicePort = port(env.KUBERNETES_SERVICE_PORT_HTTPS ?? env.KUBERNETES_SERVICE_PORT, 443, "KUBERNETES_SERVICE_PORT"); + const internalToken = required(env, "T4_CLUSTER_INTERNAL_TOKEN"); + if (new TextEncoder().encode(internalToken).byteLength < 32 || internalToken.length > 16_384) throw new Error("T4_CLUSTER_INTERNAL_TOKEN is invalid"); + const woodpeckerValues = [env.T4_WOODPECKER_BASE_URL, env.T4_WOODPECKER_TOKEN, env.T4_WOODPECKER_REPOSITORIES]; + if (woodpeckerValues.some(Boolean) && !woodpeckerValues.every(Boolean)) throw new Error("Woodpecker configuration must be complete"); + return { + namespace, + podName, + epoch: `replica:${podUid}`, + hostName: dns(required(env, "T4_CLUSTER_HOST_NAME"), "T4_CLUSTER_HOST_NAME"), + gatewayPort: port(env.T4_CLUSTER_SERVER_PORT, 8080, "T4_CLUSTER_SERVER_PORT"), + adminPort: port(env.T4_CLUSTER_ADMIN_PORT, 9090, "T4_CLUSTER_ADMIN_PORT"), + trustedProxyAddresses: proxyAddresses(env.T4_CLUSTER_TRUSTED_PROXY_ADDRESSES), + trustedProxyCidrs: proxyCidrs(env.T4_CLUSTER_TRUSTED_PROXY_CIDRS), + kubernetesBaseUrl: `https://${serviceHost}:${servicePort}`, + kubernetesTokenPath: env.T4_KUBERNETES_TOKEN_PATH ?? "/var/run/secrets/kubernetes.io/serviceaccount/token", + kubernetesCaPath: env.T4_KUBERNETES_CA_PATH ?? "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + internalToken, + ...(woodpeckerValues.every(Boolean) ? { + woodpecker: { + baseUrl: woodpeckerValues[0]!, token: woodpeckerValues[1]!, repositories: repositories(woodpeckerValues[2]!), + }, + } : {}), + }; +} +export async function loadKubernetesCredentials(config: ClusterServerConfig): Promise<{ token: string; ca: string }> { + const [token, ca] = await Promise.all([readFile(config.kubernetesTokenPath, "utf8"), readFile(config.kubernetesCaPath, "utf8")]); + if (!token.trim() || token.length > 16_384) throw new Error("Kubernetes token file is invalid"); + if (!ca.includes("BEGIN CERTIFICATE") || ca.length > 1024 * 1024) throw new Error("Kubernetes CA file is invalid"); + return { token: token.trim(), ca }; +} diff --git a/packages/cluster-server/src/gateway.ts b/packages/cluster-server/src/gateway.ts new file mode 100644 index 00000000..c853f13d --- /dev/null +++ b/packages/cluster-server/src/gateway.ts @@ -0,0 +1,355 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + CI_TRIGGER_CAPABILITY, + CLUSTER_OPERATOR_FEATURE, + commandId, + decodeClientFrame, + parseBounded, + requiredCapability, + type CiRunArguments, + type ConfirmFrame, + type ClientFrame, + type ClusterSessionCreateArguments, + type ClusterWorkspaceCreateArguments, + type CommandFrame, + type ResultFrame, + type ServerFrame, +} from "@t4-code/host-wire"; +import type { Revision } from "@t4-code/host-wire"; +import { IdempotencyStore, type CommandOutcome } from "@t4-code/host-service"; +import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; +import { + rewriteClientAddress, + rewriteServerAddress, + type PodHostConnection, + type PodHostConnector, + type PodHostRoute, +} from "./pod-host-router.ts"; +import type { CiProvider } from "./woodpecker.ts"; + +const GATEWAY_CAPABILITIES = [ + "sessions.read", "sessions.prompt", "sessions.control", "sessions.manage", "agents.control", + "bash.run", "files.read", "files.write", "files.list", "files.diff", + "preview.read", "preview.control", "preview.input", "term.open", "term.input", "term.resize", CI_TRIGGER_CAPABILITY, +] as const; +const GATEWAY_FEATURES = [ + "resume", "session.state", "session.delta", "session.observer", "controller.lease", "prompt.lease", "prompt.images", + "transcript.images", "transcript.search", "transcript.page", "agent.lifecycle", "agent.progress", "agent.event", + "agent.transcript", "terminal.io", "files.list", "files.diff", "preview.control", CLUSTER_OPERATOR_FEATURE, +] as const; + +export interface GatewayClient { + send(frame: ServerFrame): void; + close(code?: number, reason?: string): void; +} +export interface GatewayMutationBackend { + createWorkspace(commandId: string, args: ClusterWorkspaceCreateArguments, principal: string): Promise<{ id: string; revision: string }>; + createSession(commandId: string, args: ClusterSessionCreateArguments, principal: string): Promise<{ sessionId: string; revision: string }>; + deleteSession(commandId: string, sessionId: string, principal: string): Promise<{ deleted: true }>; +} +export interface ClusterGatewayOptions { + readonly projection: ClusterInfrastructureProjection; + readonly connector: PodHostConnector; + readonly mutations: GatewayMutationBackend; + readonly internalToken: string; + readonly ciProvider?: CiProvider; + readonly appserverVersion?: string; + readonly appserverBuild?: string; +} +export interface GatewayConnection { + receive(frame: unknown): Promise; + close(): void; + drain(): void; +} + +function errorResult(command: CommandFrame, code: string, message: string): ResultFrame { + return { + v: "omp-app/1", type: "response", requestId: command.requestId, commandId: command.commandId, + hostId: command.hostId, ...(command.sessionId ? { sessionId: command.sessionId } : {}), ok: false, + error: { code, message }, + }; +} +function successResult(command: CommandFrame, result: unknown): ResultFrame { + return { + v: "omp-app/1", type: "response", requestId: command.requestId, commandId: command.commandId, + hostId: command.hostId, ...(command.sessionId ? { sessionId: command.sessionId } : {}), ok: true, + command: command.command, result, + }; +} +function replayForRequest(frame: ServerFrame, command: CommandFrame): ServerFrame { + return frame.type === "response" ? { ...frame, requestId: command.requestId, commandId: command.commandId } : frame; +} + +interface PendingConfirmation { readonly command: CommandFrame; readonly expiresAt: number; } +interface RoutedPodConnection { readonly route: PodHostRoute; readonly pending: Promise; } + +function validPrincipal(value: string): boolean { + return value.length > 0 && new TextEncoder().encode(value).byteLength <= 256 && !/[\u0000-\u001f\u007f]/u.test(value) && value === value.trim(); +} +export class ClusterGateway { + readonly #projection: ClusterInfrastructureProjection; + readonly #connector: PodHostConnector; + readonly #mutations: GatewayMutationBackend; + readonly #ci?: CiProvider; + readonly #idempotency = new IdempotencyStore({ maxCompletedEntries: 1_024, completedTtlMs: 5 * 60_000 }); + readonly #connections = new Set(); + readonly #version: string; + readonly #build: string; + #draining = false; + + constructor(options: ClusterGatewayOptions) { + this.#projection = options.projection; + this.#connector = options.connector; + this.#mutations = options.mutations; + this.#ci = options.ciProvider; + this.#version = options.appserverVersion ?? "0.1.30"; + this.#build = options.appserverBuild ?? "cluster"; + if (options.internalToken.length < 32) throw new Error("cluster internal token is invalid"); + } + get connectionCount(): number { return this.#connections.size; } + get draining(): boolean { return this.#draining; } + beginDrain(): void { + this.#draining = true; + for (const connection of this.#connections) connection.drain(); + } + connect(client: GatewayClient, principal: string): GatewayConnection { + if (!validPrincipal(principal)) { + client.close(1008, "authenticated gateway identity required"); + return { receive: async () => undefined, close: () => undefined, drain: () => undefined }; + } + if (this.#draining) { + client.close(1012, "cluster server draining"); + return { receive: async () => undefined, close: () => undefined, drain: () => undefined }; + } + let helloReceived = false; + let operatorEnabled = false; + let grantedCapabilities = new Set(); + let grantedFeatures = new Set(); + let unsubscribeWorkspaces: (() => void) | undefined; + let unsubscribeSessions: (() => void) | undefined; + const challenges = new Map(); + const upstream = new Map(); + const previewOwners = new Map(); + let closed = false; + const sendSessions = (): void => { + const sessions = this.#projection.sessionRefs(principal); + client.send({ + v: "omp-app/1", type: "sessions", hostId: this.#projection.hostId, + cursor: this.#projection.sessionCursor, sessions, + totalCount: sessions.length, truncated: false, + }); + for (const [session, cached] of upstream) { + const current = this.#projection.sessionRoute(session, principal); + if (current?.url === cached.route.url && current.upstreamSessionId === cached.route.upstreamSessionId) continue; + upstream.delete(session); + void cached.pending.then(connection => connection.close(1001, "session route changed"), () => undefined); + } + }; + const close = (): void => { + if (closed) return; + closed = true; + unsubscribeWorkspaces?.(); + unsubscribeSessions?.(); + challenges.clear(); + for (const cached of upstream.values()) void cached.pending.then(value => value.close(1001, "gateway client closed"), () => undefined); + upstream.clear(); + this.#connections.delete(connection); + }; + const idempotent = async (command: CommandFrame, operation: () => Promise): Promise => { + const scopedId = commandId(createHash("sha256").update(principal).update("\u0000").update(command.commandId).digest("base64url")); + const state = this.#idempotency.begin(scopedId, command); + if (state.kind === "conflict") { client.send(errorResult(command, "idempotency_conflict", "command id was reused with another payload")); return; } + if (state.kind === "replay" || state.kind === "pending") { + const outcome = state.kind === "replay" ? state.outcome : await state.outcome; + client.send(replayForRequest(outcome.frame, command)); + return; + } + let output: ServerFrame; + try { output = await operation(); } + catch { output = errorResult(command, "UPSTREAM_UNAVAILABLE", "cluster operation failed"); } + const outcome: CommandOutcome = { frame: output }; + this.#idempotency.complete(scopedId, command, outcome); + client.send(output); + }; + const route = async (frame: ClientFrame): Promise => { + const clusterSessionId = "sessionId" in frame && typeof frame.sessionId === "string" ? frame.sessionId : undefined; + if (!clusterSessionId) { if (frame.type === "command") client.send(errorResult(frame, "INVALID_FRAME", "session route is required")); return; } + const selected = this.#projection.sessionRoute(clusterSessionId, principal); + if (!selected) { if (frame.type === "command") client.send(errorResult(frame, "NOT_AUTHORIZED", "session is unavailable for this identity")); return; } + let cached = upstream.get(clusterSessionId); + if (cached && (cached.route.url !== selected.url || cached.route.upstreamSessionId !== selected.upstreamSessionId)) { + upstream.delete(clusterSessionId); + void cached.pending.then(connection => connection.close(1001, "session route changed"), () => undefined); + cached = undefined; + } + if (!cached) { + let pending: Promise; + pending = this.#connector.connect( + selected, + upstreamFrame => { + if (upstreamFrame.type === "sessions") return; + const current = this.#projection.sessionRoute(clusterSessionId, principal); + if (current?.url !== selected.url || current.upstreamSessionId !== selected.upstreamSessionId) return; + const previewId = (upstreamFrame as unknown as Record).previewId; + if (upstreamFrame.type.startsWith("preview.") && typeof previewId === "string") previewOwners.set(previewId, clusterSessionId); + client.send(rewriteServerAddress(upstreamFrame, selected, this.#projection.hostId)); + }, + () => { if (upstream.get(clusterSessionId)?.pending === pending) upstream.delete(clusterSessionId); }, + ); + cached = { route: selected, pending }; + upstream.set(clusterSessionId, cached); + pending.catch(() => { if (upstream.get(clusterSessionId)?.pending === pending) upstream.delete(clusterSessionId); }); + } + try { + const socket = await cached.pending; + socket.send(rewriteClientAddress(frame, selected, socket.hostId ?? "upstream")); + } catch { + if (frame.type === "command") client.send(errorResult(frame, "UPSTREAM_UNAVAILABLE", "session pod host connection failed")); + } + }; + const confirm = async (frame: ConfirmFrame): Promise => { + const pending = challenges.get(String(frame.confirmationId)); + if (!pending) return false; + const command = { ...pending.command, requestId: frame.requestId } as CommandFrame; + if (pending.expiresAt < Date.now() || pending.command.commandId !== frame.commandId || pending.command.hostId !== frame.hostId || pending.command.sessionId !== frame.sessionId) { + challenges.delete(String(frame.confirmationId)); + client.send(errorResult(command, "confirmation_invalid", "confirmation is invalid or expired")); + return true; + } + challenges.delete(String(frame.confirmationId)); + if (frame.decision === "deny") { + client.send(errorResult(command, "confirmation_denied", "command was denied")); + return true; + } + await idempotent(command, async () => { + const session = command.sessionId!; + if (!this.#projection.ownsSession(session, principal)) return errorResult(command, "NOT_AUTHORIZED", "session is unavailable for this identity"); + if (this.#projection.sessionRevision(session, principal) !== command.expectedRevision) + return errorResult(command, "stale_revision", "session revision changed before deletion"); + return successResult(command, await this.#mutations.deleteSession(command.commandId, session, principal)); + }); + return true; + }; + const receive = async (input: unknown): Promise => { + if (closed) return; + let frame: ClientFrame; + try { frame = decodeClientFrame(typeof input === "string" || input instanceof Uint8Array ? parseBounded(input) : input); } + catch { client.close(1002, "invalid omp-app frame"); close(); return; } + if (frame.type === "hello") { + if (helloReceived) { client.close(1002, "duplicate hello"); close(); return; } + helloReceived = true; + operatorEnabled = frame.requestedFeatures.includes(CLUSTER_OPERATOR_FEATURE); + const requestedCapabilities = new Set(frame.capabilities?.client ?? []); + grantedCapabilities = new Set(GATEWAY_CAPABILITIES.filter(capability => requestedCapabilities.has(capability) && (capability !== CI_TRIGGER_CAPABILITY || this.#ci !== undefined))); + const requestedFeatures = new Set(frame.requestedFeatures); + grantedFeatures = new Set(GATEWAY_FEATURES.filter(feature => requestedFeatures.has(feature) && (feature !== CLUSTER_OPERATOR_FEATURE || operatorEnabled))); + client.send({ + v: "omp-app/1", type: "welcome", selectedProtocol: "omp-app/1", hostId: this.#projection.hostId, + ompVersion: "17.0.5", ompBuild: "8476f4451ed95c5d5401785d279a93d3c659fac4", + appserverVersion: this.#version, appserverBuild: this.#build, epoch: this.#projection.epoch, + grantedCapabilities: [...grantedCapabilities], grantedFeatures: [...grantedFeatures], + negotiatedLimits: { maxPayloadLength: 1_048_576, maxWorkspaces: 256, maxSessions: 1_000, workspaceReplayFrames: 512 }, + authentication: "paired", + resumed: frame.savedCursors.some(saved => saved.hostId === this.#projection.hostId && saved.cursor.epoch === this.#projection.epoch), + }); + if (!operatorEnabled || !grantedCapabilities.has("sessions.read")) return; + sendSessions(); + unsubscribeWorkspaces = this.#projection.subscribe(value => client.send(value), this.#projection.workspaceCursor, principal); + unsubscribeSessions = this.#projection.subscribeSessions(sendSessions); + return; + } + if (!helloReceived) { client.close(1002, "hello required"); close(); return; } + if (frame.type === "ping") { client.send({ v: "omp-app/1", type: "pong", nonce: frame.nonce, timestamp: frame.timestamp }); return; } + if (frame.type === "confirm" && await confirm(frame)) return; + if (frame.type !== "command") { + if (!operatorEnabled || !("hostId" in frame) || frame.hostId !== this.#projection.hostId) return; + if (frame.type === "terminal.input" && (!grantedFeatures.has("terminal.io") || !grantedCapabilities.has("term.input"))) return; + if (frame.type === "terminal.resize" && (!grantedFeatures.has("terminal.io") || !grantedCapabilities.has("term.resize"))) return; + if (frame.type === "terminal.close" && (!grantedFeatures.has("terminal.io") || !grantedCapabilities.has("term.open"))) return; + if (frame.type !== "confirm" && !frame.type.startsWith("terminal.")) return; + await route(frame); + return; + } + if (!operatorEnabled) { client.send(errorResult(frame, "UNSUPPORTED_FEATURE", "cluster.operator was not negotiated")); return; } + if (frame.hostId !== this.#projection.hostId) { client.send(errorResult(frame, "NOT_FOUND", "cluster host was not found")); return; } + const capability = requiredCapability(frame.command); + if (!capability || !grantedCapabilities.has(capability)) { client.send(errorResult(frame, "NOT_AUTHORIZED", "command capability was not granted")); return; } + if (frame.command === "workspace.list") { client.send(successResult(frame, this.#projection.workspaceList(principal))); return; } + if (frame.command === "workspace.create") { + await idempotent(frame, async () => { + const args = frame.args as unknown as ClusterWorkspaceCreateArguments; + const created = await this.#mutations.createWorkspace(frame.commandId, args, principal); + return successResult(frame, { workspace: { + id: created.id, displayName: args.displayName, phase: "Pending", retentionPolicy: args.retentionPolicy, + ...(args.storageClass ? { storageClass: args.storageClass } : {}), capacity: args.capacity, + accessMode: "ReadWriteMany", revision: created.revision, + } }); + }); + return; + } + if (frame.command === "session.create") { + const args = frame.args as unknown as ClusterSessionCreateArguments; + if (!this.#projection.ownsWorkspace(args.workspaceId, principal)) { client.send(errorResult(frame, "NOT_AUTHORIZED", "workspace is unavailable for this identity")); return; } + await idempotent(frame, async () => { + const created = await this.#mutations.createSession(frame.commandId, args, principal); + const session = await this.#projection.waitForSessionAuthority(created.sessionId); + if (!this.#projection.ownsSession(created.sessionId, principal)) + return errorResult(frame, "NOT_AUTHORIZED", "session is unavailable for this identity"); + return successResult(frame, { session }); + }); + return; + } + if (frame.command === "session.delete") { + if (!frame.sessionId || !this.#projection.ownsSession(frame.sessionId, principal)) { client.send(errorResult(frame, "NOT_AUTHORIZED", "session is unavailable for this identity")); return; } + if (this.#projection.sessionRevision(frame.sessionId, principal) !== frame.expectedRevision) { client.send(errorResult(frame, "stale_revision", "session revision changed before confirmation")); return; } + if (frame.confirmationId !== undefined) { client.send(errorResult(frame, "confirmation_invalid", "command confirmation must use a confirm frame")); return; } + for (const [id, pending] of challenges) if (pending.expiresAt < Date.now()) challenges.delete(id); + if (challenges.size >= 5) { client.send(errorResult(frame, "confirmation_unavailable", "confirmation capacity exceeded")); return; } + const confirmationId = randomUUID(); + const expiresAt = Date.now() + 60_000; + challenges.set(confirmationId, { command: frame, expiresAt }); + client.send({ + v: "omp-app/1", type: "confirmation", confirmationId: confirmationId as never, + commandId: frame.commandId, hostId: this.#projection.hostId, sessionId: frame.sessionId, + commandHash: createHash("sha256").update(JSON.stringify({ ...frame, confirmationId: undefined })).digest("hex"), + revision: frame.expectedRevision as Revision, + expiresAt: new Date(expiresAt).toISOString(), summary: "session.delete", + }); + return; + } + if (frame.command === "ci.run") { + await idempotent(frame, async () => { + if (!frame.sessionId || this.#projection.sessionRevision(frame.sessionId, principal) !== frame.expectedRevision) + return errorResult(frame, "stale_revision", "session revision changed before CI trigger"); + if (!this.#ci) return errorResult(frame, "UNSUPPORTED_FEATURE", "CI provider is unavailable"); + const args = frame.args as unknown as CiRunArguments; + const allowed = this.#projection.sessionCiSelection(frame.sessionId, principal); + if (!allowed || allowed.repositoryId !== args.repositoryId || allowed.ref !== args.ref || allowed.commit !== args.commit) + return errorResult(frame, "NOT_AUTHORIZED", "CI correlation is not declared by this session"); + return successResult(frame, await this.#ci.run({ sessionId: frame.sessionId, repositoryId: args.repositoryId, ref: args.ref, commit: args.commit })); + }); + return; + } + if (frame.command.startsWith("preview.")) { + const preview = typeof frame.args.previewId === "string" ? frame.args.previewId : undefined; + if (preview && previewOwners.has(preview) && previewOwners.get(preview) !== frame.sessionId) { + client.send(errorResult(frame, "NOT_AUTHORIZED", "preview belongs to another session")); + return; + } + } + await route(frame); + }; + const connection: GatewayConnection = { + receive, + close, + drain: () => { + if (closed) return; + client.send({ v: "omp-app/1", type: "bye", code: "server_restart", reason: "cluster server draining", retryable: true }); + client.close(1012, "cluster server draining"); + close(); + }, + }; + this.#connections.add(connection); + return connection; + } +} diff --git a/packages/cluster-server/src/index.ts b/packages/cluster-server/src/index.ts new file mode 100644 index 00000000..50b04ea9 --- /dev/null +++ b/packages/cluster-server/src/index.ts @@ -0,0 +1,13 @@ +export * from "./config.ts"; +export * from "./gateway.ts"; +export * from "./kubernetes-client.ts"; +export * from "./kubernetes-projection.ts"; +export * from "./kubernetes-runner.ts"; +export * from "./observability.ts"; +export * from "./pod-host-router.ts"; +export * from "./server.ts"; +export * from "./session-host-policy.ts"; +export * from "./session-authority-runner.ts"; +export * from "./woodpecker.ts"; +export { runClusterServer } from "./main.ts"; +export { runSessionHost } from "./session-host-main.ts"; diff --git a/packages/cluster-server/src/kubernetes-client.ts b/packages/cluster-server/src/kubernetes-client.ts new file mode 100644 index 00000000..faf9479f --- /dev/null +++ b/packages/cluster-server/src/kubernetes-client.ts @@ -0,0 +1,285 @@ +import { createHash } from "node:crypto"; +import type { + ClusterSessionCreateArguments, + ClusterWorkspaceCreateArguments, +} from "@t4-code/host-wire"; +import { + CLUSTER_MAX_SESSIONS, + CLUSTER_MAX_WORKSPACES, + type InfrastructureList, + type KubernetesResource, + type KubernetesWatchEvent, +} from "./kubernetes-projection.ts"; + +const API_PREFIX = "/apis/cluster.t4.dev/v1alpha1"; +const MAX_WATCH_LINE_BYTES = 1024 * 1024; + +export interface KubernetesApiClientOptions { + readonly baseUrl: string; + readonly namespace: string; + readonly token: string; + readonly ca?: string; + readonly fetch?: typeof globalThis.fetch; +} +interface KubernetesList { + readonly metadata?: { readonly resourceVersion?: string }; + readonly items?: readonly KubernetesResource[]; +} + +function canonical(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== "object") return value; + return Object.fromEntries(Object.keys(value as Record).sort().map(key => [key, canonical((value as Record)[key])])); +} +export function semanticResourceHash(value: unknown): string { + return `sha256:${createHash("sha256").update(JSON.stringify(canonical(value))).digest("hex")}`; +} +function resourceName(prefix: "workspace" | "session", commandId: string): string { + const digest = createHash("sha256").update(commandId, "utf8").digest("hex").slice(0, 16); + return `${prefix}-${digest}`; +} +function safeNamespace(value: string): string { + if (!/^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u.test(value)) throw new Error("Kubernetes namespace is invalid"); + return value; +} +function exactHttpsBase(value: string): string { + const url = new URL(value); + if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash) throw new Error("Kubernetes base URL must use HTTPS"); + return url.href.replace(/\/$/u, ""); +} +function object(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +export class KubernetesApiError extends Error { + constructor(readonly status: number, message: string, readonly body?: unknown) { + super(message); + this.name = "KubernetesApiError"; + } +} + +export class KubernetesApiClient { + readonly baseUrl: string; + readonly namespace: string; + readonly #token: string; + readonly #ca?: string; + readonly #fetch: typeof globalThis.fetch; + + constructor(options: KubernetesApiClientOptions) { + this.baseUrl = exactHttpsBase(options.baseUrl); + this.namespace = safeNamespace(options.namespace); + if (!options.token || options.token.length > 16_384) throw new Error("Kubernetes service account token is invalid"); + this.#token = options.token; + this.#ca = options.ca; + this.#fetch = options.fetch ?? globalThis.fetch; + } + + async listInfrastructure(hostName?: string, signal?: AbortSignal): Promise { + const [hosts, workspaces, sessions] = await Promise.all([ + this.list("t4clusterhosts", 256, signal), + this.list("t4workspaces", CLUSTER_MAX_WORKSPACES, signal), + this.list("t4sessions", CLUSTER_MAX_SESSIONS, signal), + ]); + const host = hostName ? hosts.items.find(value => value.metadata.name === hostName) : hosts.items[0]; + if (!host) throw new Error("T4ClusterHost is unavailable"); + if (hosts.items.length > 1 && !hostName) throw new Error("T4ClusterHost selection is ambiguous"); + const belongsToHost = (resource: KubernetesResource): boolean => object(resource.spec).hostRef === host.metadata.name; + return { + host, + workspaces: workspaces.items.filter(belongsToHost), + sessions: sessions.items.filter(belongsToHost), + resourceVersion: sessions.resourceVersion || workspaces.resourceVersion || hosts.resourceVersion, + resourceVersions: { t4clusterhosts: hosts.resourceVersion, t4workspaces: workspaces.resourceVersion, t4sessions: sessions.resourceVersion }, + }; + } + + async list(resource: string, limit: number, signal?: AbortSignal): Promise<{ items: KubernetesResource[]; resourceVersion: string }> { + const response = object(await this.request(`${this.#collection(resource)}?limit=${limit}`, { signal })); + const metadata = object(response.metadata); + const items = Array.isArray(response.items) ? response.items as KubernetesResource[] : []; + if (items.length > limit || typeof metadata.continue === "string" && metadata.continue.length > 0) + throw new Error(`${resource} list exceeds limit`); + return { items, resourceVersion: typeof metadata.resourceVersion === "string" ? metadata.resourceVersion : "0" }; + } + + async create(resource: string, body: unknown, signal?: AbortSignal): Promise { + return await this.request(this.#collection(resource), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + signal, + }) as KubernetesResource; + } + + async get(resource: string, name: string, signal?: AbortSignal): Promise { + return await this.request(`${this.#collection(resource)}/${encodeURIComponent(name)}`, { signal }) as KubernetesResource; + } + + async delete(resource: string, name: string, preconditions: { readonly uid: string; readonly resourceVersion: string }, signal?: AbortSignal): Promise { + return await this.request(`${this.#collection(resource)}/${encodeURIComponent(name)}`, { + method: "DELETE", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ apiVersion: "v1", kind: "DeleteOptions", propagationPolicy: "Foreground", preconditions }), + signal, + }); + } + + async watch( + resource: string, + resourceVersion: string, + onEvent: (event: KubernetesWatchEvent) => void, + signal: AbortSignal, + onStarted?: () => void, + ): Promise { + const query = new URLSearchParams({ watch: "1", allowWatchBookmarks: "true", resourceVersion, timeoutSeconds: "300" }); + const response = await this.#raw(`${this.#collection(resource)}?${query}`, { signal }); + if (!response.ok) throw new KubernetesApiError(response.status, `Kubernetes watch failed with ${response.status}`); + if (!response.body) throw new Error("Kubernetes watch body is unavailable"); + const reader = response.body.getReader(); + onStarted?.(); + const decoder = new TextDecoder("utf-8", { fatal: true }); + let buffered = ""; + for (;;) { + const chunk = await reader.read(); + if (chunk.done) break; + buffered += decoder.decode(chunk.value, { stream: true }); + if (new TextEncoder().encode(buffered).byteLength > MAX_WATCH_LINE_BYTES) throw new Error("Kubernetes watch frame exceeds limit"); + for (;;) { + const newline = buffered.indexOf("\n"); + if (newline < 0) break; + const line = buffered.slice(0, newline); + buffered = buffered.slice(newline + 1); + if (!line) continue; + const value = object(JSON.parse(line)); + if (value.type === "BOOKMARK") continue; + if (value.type === "ERROR") throw new KubernetesApiError(Number(object(value.object).code) || 500, "Kubernetes watch error", value.object); + if (value.type === "ADDED" || value.type === "MODIFIED" || value.type === "DELETED") + onEvent({ type: value.type, object: value.object as KubernetesResource }); + } + } + } + + async request(path: string, init: RequestInit = {}): Promise { + const response = await this.#raw(path, init); + const body = await response.json().catch(() => undefined) as unknown; + if (!response.ok) throw new KubernetesApiError(response.status, `Kubernetes API request failed with ${response.status}`, body); + return body; + } + + #collection(resource: string): string { + if (!/^[a-z0-9]+$/u.test(resource)) throw new Error("Kubernetes resource is invalid"); + return `${API_PREFIX}/namespaces/${this.namespace}/${resource}`; + } + #raw(path: string, init: RequestInit): Promise { + const headers = new Headers(init.headers); + headers.set("accept", "application/json"); + headers.set("authorization", `Bearer ${this.#token}`); + const request = { ...init, headers } as RequestInit & { tls?: { ca?: string } }; + if (this.#ca) request.tls = { ca: this.#ca }; + return this.#fetch(`${this.baseUrl}${path}`, request); + } +} + +export interface KubernetesGatewayMutationBackendOptions { + readonly client: KubernetesApiClient; + readonly hostRef: string; +} +export class KubernetesGatewayMutationBackend { + readonly #client: KubernetesApiClient; + readonly #hostRef: string; + constructor(options: KubernetesGatewayMutationBackendOptions) { + this.#client = options.client; + this.#hostRef = options.hostRef; + } + + async createWorkspace(commandId: string, args: ClusterWorkspaceCreateArguments, principal: string): Promise<{ id: string; revision: string }> { + this.#validatePrincipal(principal); + const identity = `${principal}\u0000${commandId}`; + const name = resourceName("workspace", identity); + const body = { + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { name, annotations: this.#annotations(commandId, args, principal) }, + spec: { + hostRef: this.#hostRef, + owner: principal, + displayName: args.displayName, + retentionPolicy: args.retentionPolicy, + size: args.capacity, + ...(args.storageClass ? { storageClass: args.storageClass } : {}), + ...(args.repository ? { repository: { id: args.repository.repositoryId, ...(args.repository.ref ? { ref: args.repository.ref } : {}), ...(args.repository.commit ? { commit: args.repository.commit } : {}) } } : {}), + }, + }; + const resource = await this.#createOrRead("t4workspaces", name, body, commandId, semanticResourceHash({ args, principal }), principal); + return { id: resource.metadata.name, revision: resource.metadata.resourceVersion ?? "0" }; + } + + async createSession(commandId: string, args: ClusterSessionCreateArguments, principal: string): Promise<{ sessionId: string; revision: string }> { + this.#validatePrincipal(principal); + const workspace = await this.#client.get("t4workspaces", args.workspaceId); + this.#assertWorkspaceOwner(workspace, principal); + const identity = `${principal}\u0000${commandId}`; + const name = resourceName("session", identity); + const body = { + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Session", + metadata: { name, annotations: this.#annotations(commandId, args, principal) }, + spec: { + hostRef: this.#hostRef, + workspaceRef: args.workspaceId, + ...(args.title ? { title: args.title } : {}), + runtimeProfile: args.runtimeProfile, + gui: { enabled: args.guiEnabled }, + ...(args.ci ? { ci: { repositoryId: args.ci.repositoryId, ref: args.ci.ref, commit: args.ci.commit } } : {}), + }, + }; + const resource = await this.#createOrRead("t4sessions", name, body, commandId, semanticResourceHash({ args, principal }), principal); + return { sessionId: resource.metadata.name, revision: resource.metadata.resourceVersion ?? "0" }; + } + + async deleteSession(_commandId: string, sessionId: string, principal: string): Promise<{ deleted: true }> { + this.#validatePrincipal(principal); + const session = await this.#client.get("t4sessions", sessionId); + if (object(session.spec).hostRef !== this.#hostRef) throw new Error("session belongs to another cluster host"); + const workspaceRef = object(session.spec).workspaceRef; + if (typeof workspaceRef !== "string") throw new Error("session workspace reference is invalid"); + const workspace = await this.#client.get("t4workspaces", workspaceRef); + this.#assertWorkspaceOwner(workspace, principal); + if (!session.metadata.uid || !session.metadata.resourceVersion) throw new Error("session delete precondition is unavailable"); + await this.#client.delete("t4sessions", sessionId, { uid: session.metadata.uid, resourceVersion: session.metadata.resourceVersion }); + return { deleted: true }; + } + + #validatePrincipal(principal: string): void { + if (!principal || new TextEncoder().encode(principal).byteLength > 256 || /[\u0000-\u001f\u007f]/u.test(principal) || principal !== principal.trim()) + throw new Error("gateway principal is invalid"); + } + #assertWorkspaceOwner(workspace: KubernetesResource, principal: string): void { + const spec = object(workspace.spec); + if (workspace.kind !== "T4Workspace" || spec.hostRef !== this.#hostRef || spec.owner !== principal) + throw new Error("workspace is unavailable for this identity"); + } + #annotations(commandId: string, args: unknown, principal: string): Record { + return { + "cluster.t4.dev/command-id": commandId, + "cluster.t4.dev/principal-hash": semanticResourceHash(principal), + "cluster.t4.dev/semantic-hash": semanticResourceHash({ args, principal }), + }; + } + async #createOrRead(resourceType: string, name: string, body: unknown, commandId: string, hash: string, principal: string): Promise { + try { + const created = await this.#client.create(resourceType, body); + return created.metadata ? created : { ...(body as KubernetesResource), metadata: { ...(body as KubernetesResource).metadata, resourceVersion: "0" } }; + } catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 409) throw error; + const existing = await this.#client.get(resourceType, name); + const annotations = existing.metadata.annotations ?? {}; + if ( + annotations["cluster.t4.dev/command-id"] !== commandId || + annotations["cluster.t4.dev/principal-hash"] !== semanticResourceHash(principal) || + annotations["cluster.t4.dev/semantic-hash"] !== hash + ) throw new Error("idempotency conflict for existing Kubernetes resource"); + return existing; + } + } +} diff --git a/packages/cluster-server/src/kubernetes-projection.ts b/packages/cluster-server/src/kubernetes-projection.ts new file mode 100644 index 00000000..751a3ff1 --- /dev/null +++ b/packages/cluster-server/src/kubernetes-projection.ts @@ -0,0 +1,455 @@ +import { + CLUSTER_MAX_WORKSPACES as WIRE_MAX_WORKSPACES, + decodeClusterCondition, + decodeSessionRef, + decodeWorkspaceInfrastructureProjection, + hostId, + revision, + type ClusterCondition, + type Cursor, + type HostId, + type SessionRef, + type WorkspaceInfrastructureProjection, + type WorkspaceListResult, + type WorkspaceStateFrame, +} from "@t4-code/host-wire"; +import type { PodHostEndpoint, PodHostRoute } from "./pod-host-router.ts"; +export const CLUSTER_MAX_WORKSPACES = WIRE_MAX_WORKSPACES; +export const CLUSTER_MAX_SESSIONS = 1_000; +export const CLUSTER_WORKSPACE_REPLAY_FRAMES = 512; + +export interface KubernetesMetadata { + readonly name: string; + readonly uid?: string; + readonly resourceVersion?: string; + readonly generation?: number; + readonly creationTimestamp?: string; + readonly deletionTimestamp?: string; + readonly annotations?: Readonly>; +} +export interface KubernetesResource { + readonly apiVersion?: string; + readonly kind?: string; + readonly metadata: KubernetesMetadata; + readonly spec?: Readonly>; + readonly status?: Readonly>; +} +export interface KubernetesWatchEvent { + readonly type: "ADDED" | "MODIFIED" | "DELETED"; + readonly object: KubernetesResource; +} +export interface InfrastructureList { + readonly host: KubernetesResource; + readonly workspaces: readonly KubernetesResource[]; + readonly sessions: readonly KubernetesResource[]; + readonly resourceVersion: string; + readonly resourceVersions?: Readonly>; +} +export interface ClusterInfrastructureProjectionOptions { + readonly epoch: string; + readonly namespace: string; + readonly maxWorkspaces?: number; + readonly maxSessions?: number; +} + +type SessionListener = () => void; +interface WorkspaceReplayItem { readonly frame: WorkspaceStateFrame; readonly owner?: string; } +interface WorkspaceSubscription { readonly listener: (frame: WorkspaceStateFrame) => void; readonly principal?: string; } + +interface SessionAuthorityWaiter { readonly resolve: (value: SessionRef) => void; readonly reject: (reason: Error) => void; readonly timer: ReturnType; } +function record(value: unknown): Record { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} +function text(value: unknown, fallback = ""): string { + return typeof value === "string" ? value : fallback; +} +function number(value: unknown, fallback = 0): number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : fallback; +} +function firstCondition(status: Readonly>): ClusterCondition | undefined { + if (!Array.isArray(status.conditions) || status.conditions.length === 0) return undefined; + const raw = record(status.conditions[0]); + try { + return decodeClusterCondition({ + type: raw.type, + status: raw.status, + reason: raw.reason, + message: raw.message ?? "", + observedGeneration: raw.observedGeneration ?? status.observedGeneration ?? 0, + }); + } catch { + return undefined; + } +} +function categorical(value: unknown, values: T, fallback: T[number]): T[number] { + return typeof value === "string" && (values as readonly string[]).includes(value) ? value as T[number] : fallback; +} +function resourceRevision(resource: KubernetesResource): string { + return text(resource.metadata.resourceVersion, `generation-${number(resource.metadata.generation)}`); +} +function serviceName(value: unknown): string | undefined { + if (typeof value !== "string" || !/^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u.test(value)) return undefined; + return value; +} +function workspaceOwner(resource: KubernetesResource): string | undefined { + const owner = record(resource.spec).owner; + if (typeof owner !== "string" || owner.length === 0 || new TextEncoder().encode(owner).byteLength > 256 || /[\u0000-\u001f\u007f]/u.test(owner)) return undefined; + return owner; +} + +export function clusterHostIdFromUid(uid: string): HostId { + if (!/^[A-Za-z0-9][A-Za-z0-9-]{0,127}$/u.test(uid)) throw new Error("T4ClusterHost UID is invalid"); + return hostId(`cluster:${uid}`); +} + +export class ClusterInfrastructureProjection { + readonly epoch: string; + readonly namespace: string; + readonly maxWorkspaces: number; + readonly maxSessions: number; + #hostId?: HostId; + #host?: KubernetesResource; + #workspaces = new Map(); + #sessions = new Map(); + #authoritativeSessions = new Map(); + #authorityWaiters = new Map>(); + #versions = new Map(); + #workspaceSequence = 0; + #sessionSequence = 0; + #workspaceListeners = new Set(); + #sessionListeners = new Set(); + #replay: WorkspaceReplayItem[] = []; + #resourceVersion = "0"; + #resourceVersions: Readonly> = {}; + + constructor(options: ClusterInfrastructureProjectionOptions) { + if (!options.epoch || options.epoch.length > 128) throw new Error("replica epoch is invalid"); + if (!/^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u.test(options.namespace)) throw new Error("namespace is invalid"); + this.epoch = options.epoch; + this.namespace = options.namespace; + this.maxWorkspaces = options.maxWorkspaces ?? CLUSTER_MAX_WORKSPACES; + this.maxSessions = options.maxSessions ?? CLUSTER_MAX_SESSIONS; + if (this.maxWorkspaces < 1 || this.maxWorkspaces > CLUSTER_MAX_WORKSPACES) throw new Error("workspace projection limit is invalid"); + if (this.maxSessions < 1 || this.maxSessions > CLUSTER_MAX_SESSIONS) throw new Error("session projection limit is invalid"); + } + + get hostId(): HostId { + if (!this.#hostId) throw new Error("cluster host projection is not synchronized"); + return this.#hostId; + } + get workspaceCursor(): Cursor { return { epoch: this.epoch, seq: this.#workspaceSequence }; } + get sessionCursor(): Cursor { return { epoch: this.epoch, seq: this.#sessionSequence }; } + get resourceVersion(): string { return this.#resourceVersion; } + resourceVersionFor(resource: string): string { return this.#resourceVersions[resource] ?? this.#resourceVersion; } + + replace(input: InfrastructureList): void { + if (input.workspaces.length > this.maxWorkspaces) throw new Error("workspace projection limit exceeded"); + if (input.sessions.length > this.maxSessions) throw new Error("session projection limit exceeded"); + if (input.host.kind !== "T4ClusterHost" || !input.host.metadata.uid) throw new Error("T4ClusterHost identity is missing"); + const selectedHost = input.host.metadata.name; + for (const resource of input.workspaces) + if (resource.kind !== "T4Workspace" || record(resource.spec).hostRef !== selectedHost) throw new Error("workspace belongs to another cluster host"); + for (const resource of input.sessions) + if (resource.kind !== "T4Session" || record(resource.spec).hostRef !== selectedHost) throw new Error("session belongs to another cluster host"); + const nextHostId = clusterHostIdFromUid(input.host.metadata.uid); + if (this.#hostId && this.#hostId !== nextHostId) throw new Error("T4ClusterHost UID changed within a replica epoch"); + const initialized = this.#hostId !== undefined; + const previousWorkspaces = this.#workspaces; + const previousSessions = this.#sessions; + const nextWorkspaces = new Map(input.workspaces.map(resource => [resource.metadata.name, resource])); + const nextSessions = new Map(input.sessions.map(resource => [resource.metadata.name, resource])); + this.#hostId = nextHostId; + this.#host = input.host; + this.#workspaces = nextWorkspaces; + this.#sessions = nextSessions; + for (const name of this.#authoritativeSessions.keys()) if (!nextSessions.has(name)) this.#authoritativeSessions.delete(name); + this.#versions.clear(); + for (const resource of [input.host, ...input.workspaces, ...input.sessions]) + this.#versions.set(`${resource.kind ?? "unknown"}/${resource.metadata.name}`, text(resource.metadata.resourceVersion)); + this.#resourceVersion = input.resourceVersion; + this.#resourceVersions = input.resourceVersions ?? {}; + if (!initialized) { + this.#workspaceSequence = 1; + this.#sessionSequence = 1; + return; + } + const workspaceNames = [...new Set([...previousWorkspaces.keys(), ...nextWorkspaces.keys()])].sort(); + let workspaceAccessChanged = false; + for (const name of workspaceNames) { + const before = previousWorkspaces.get(name); + const after = nextWorkspaces.get(name); + if (before && after && resourceRevision(before) === resourceRevision(after)) continue; + const beforeOwner = before ? workspaceOwner(before) : undefined; + const afterOwner = after ? workspaceOwner(after) : undefined; + if (before && after && beforeOwner !== afterOwner) { + this.#publishWorkspace(before, true, beforeOwner); + this.#publishWorkspace(after, false, afterOwner); + workspaceAccessChanged = true; + } else if (after) { + this.#publishWorkspace(after, false, afterOwner); + workspaceAccessChanged ||= !before; + } else if (before) { + this.#publishWorkspace(before, true, beforeOwner); + workspaceAccessChanged = true; + } + } + const sessionNames = new Set([...previousSessions.keys(), ...nextSessions.keys()]); + const sessionsChanged = [...sessionNames].some(name => { + const before = previousSessions.get(name); + const after = nextSessions.get(name); + return !before || !after || resourceRevision(before) !== resourceRevision(after); + }); + if (sessionsChanged || workspaceAccessChanged) { + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + } + } + + workspaceList(principal?: string): WorkspaceListResult { + return { + cursor: this.workspaceCursor, + workspaces: [...this.#workspaces.values()] + .filter(resource => principal === undefined || workspaceOwner(resource) === principal) + .sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)) + .map(resource => this.#workspace(resource)), + }; + } + sessionRefs(principal?: string): SessionRef[] { + return [...this.#sessions.values()] + .filter(resource => principal === undefined || this.#ownsSessionResource(resource, principal)) + .sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)) + .flatMap(resource => { + const authoritative = this.#authoritativeSessions.get(resource.metadata.name); + return authoritative ? [this.#session(resource, authoritative)] : []; + }); + } + + allowedOrigins(): readonly string[] { + const values = record(this.#host?.spec).allowedOrigins; + if (!Array.isArray(values) || values.length > 64) return []; + const origins: string[] = []; + for (const value of values) { + if (typeof value !== "string") continue; + try { + const url = new URL(value); + if ((url.protocol === "https:" || url.protocol === "http:") && !url.username && !url.password && url.pathname === "/" && !url.search && !url.hash) + origins.push(url.origin); + } catch {} + } + return origins; + } + + ownsWorkspace(workspace: string, principal: string): boolean { + const resource = this.#workspaces.get(workspace); + return resource !== undefined && workspaceOwner(resource) === principal; + } + ownsSession(session: string, principal: string): boolean { + const resource = this.#sessions.get(session); + return resource !== undefined && this.#ownsSessionResource(resource, principal); + } + sessionEndpoints(): PodHostEndpoint[] { + return [...this.#sessions.values()].flatMap(resource => { + const service = serviceName(record(resource.status).serviceRef); + return service ? [{ clusterSessionId: resource.metadata.name, url: `ws://${service}.${this.namespace}.svc:8787/v1/ws` }] : []; + }); + } + sessionRoute(clusterSessionId: string, principal?: string): PodHostRoute | undefined { + const resource = this.#sessions.get(clusterSessionId); + const authoritative = this.#authoritativeSessions.get(clusterSessionId); + if (!resource || !authoritative || principal !== undefined && !this.#ownsSessionResource(resource, principal)) return undefined; + const service = serviceName(record(resource.status).serviceRef); + if (!service) return undefined; + return { clusterSessionId, upstreamSessionId: authoritative.sessionId, url: `ws://${service}.${this.namespace}.svc:8787/v1/ws` }; + } + sessionRevision(session: string, principal?: string): string | undefined { + const resource = this.#sessions.get(session); + const authoritative = this.#authoritativeSessions.get(session); + if (!resource || !authoritative || principal !== undefined && !this.#ownsSessionResource(resource, principal)) return undefined; + return authoritative.revision; + } + sessionRef(session: string, principal?: string): SessionRef | undefined { + const resource = this.#sessions.get(session); + const authoritative = this.#authoritativeSessions.get(session); + if (!resource || !authoritative || principal !== undefined && !this.#ownsSessionResource(resource, principal)) return undefined; + return this.#session(resource, authoritative); + } + setSessionAuthority(clusterSessionId: string, value: SessionRef): void { + const resource = this.#sessions.get(clusterSessionId); + if (!resource) return; + const authoritative = decodeSessionRef(value, `authority.${clusterSessionId}`); + const previous = this.#authoritativeSessions.get(clusterSessionId); + if (previous && JSON.stringify(previous) === JSON.stringify(authoritative)) return; + this.#authoritativeSessions.set(clusterSessionId, authoritative); + this.#sessionSequence++; + const projected = this.#session(resource, authoritative); + for (const waiter of this.#authorityWaiters.get(clusterSessionId) ?? []) { + clearTimeout(waiter.timer); + waiter.resolve(projected); + } + this.#authorityWaiters.delete(clusterSessionId); + for (const listener of this.#sessionListeners) listener(); + } + clearSessionAuthority(clusterSessionId: string): void { + if (!this.#authoritativeSessions.delete(clusterSessionId)) return; + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + } + waitForSessionAuthority(clusterSessionId: string, timeoutMs = 30_000): Promise { + const current = this.sessionRef(clusterSessionId); + if (current) return Promise.resolve(current); + const deferred = Promise.withResolvers(); + let waiter: SessionAuthorityWaiter | undefined; + const timer = setTimeout(() => { + const waiters = this.#authorityWaiters.get(clusterSessionId); + if (waiter) waiters?.delete(waiter); + if (waiters?.size === 0) this.#authorityWaiters.delete(clusterSessionId); + deferred.reject(new Error("authoritative OMP session did not become available")); + }, timeoutMs); + waiter = { resolve: deferred.resolve, reject: deferred.reject, timer }; + const waiters = this.#authorityWaiters.get(clusterSessionId) ?? new Set(); + waiters.add(waiter); + this.#authorityWaiters.set(clusterSessionId, waiters); + return deferred.promise; + } + sessionCiSelection(session: string, principal?: string): { repositoryId: string; ref: string; commit: string } | undefined { + const resource = this.#sessions.get(session); + if (!resource || principal !== undefined && !this.#ownsSessionResource(resource, principal)) return undefined; + const ci = record(record(resource.spec).ci); + if (typeof ci.repositoryId !== "string" || typeof ci.ref !== "string" || typeof ci.commit !== "string") return undefined; + return { repositoryId: ci.repositoryId, ref: ci.ref, commit: ci.commit }; + } + + applyWatch(event: KubernetesWatchEvent): void { + const resource = event.object; + const name = resource.metadata.name; + if (resource.kind === "T4ClusterHost") { + if (!this.#host || name !== this.#host.metadata.name) return; + if (event.type === "DELETED") throw new Error("selected T4ClusterHost was deleted"); + if (!resource.metadata.uid || clusterHostIdFromUid(resource.metadata.uid) !== this.hostId) throw new Error("selected T4ClusterHost identity changed"); + } else if (resource.kind === "T4Workspace" || resource.kind === "T4Session") { + if (!this.#host || record(resource.spec).hostRef !== this.#host.metadata.name) return; + } else return; + const key = `${resource.kind}/${name}`; + const version = text(resource.metadata.resourceVersion); + if (version && this.#versions.get(key) === version) return; + if (version) { + this.#versions.set(key, version); + this.#resourceVersion = version; + const collection = resource.kind === "T4ClusterHost" ? "t4clusterhosts" : resource.kind === "T4Workspace" ? "t4workspaces" : "t4sessions"; + this.#resourceVersions = { ...this.#resourceVersions, [collection]: version }; + } + if (resource.kind === "T4ClusterHost") { this.#host = resource; return; } + if (resource.kind === "T4Session") { + if (event.type === "DELETED") { + if (!this.#sessions.delete(name)) return; + this.#authoritativeSessions.delete(name); + } else { + if (!this.#sessions.has(name) && this.#sessions.size >= this.maxSessions) throw new Error("session projection limit exceeded"); + this.#sessions.set(name, resource); + } + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + return; + } + const existing = this.#workspaces.get(name); + let workspaceAccessChanged = false; + if (event.type === "DELETED") { + if (!existing) return; + this.#workspaces.delete(name); + this.#publishWorkspace(resource, true, workspaceOwner(existing) ?? workspaceOwner(resource)); + workspaceAccessChanged = true; + } else { + if (!existing && this.#workspaces.size >= this.maxWorkspaces) throw new Error("workspace projection limit exceeded"); + this.#workspaces.set(name, resource); + const previousOwner = existing ? workspaceOwner(existing) : undefined; + const nextOwner = workspaceOwner(resource); + if (existing && previousOwner !== nextOwner) this.#publishWorkspace(existing, true, previousOwner); + this.#publishWorkspace(resource, false, nextOwner); + workspaceAccessChanged = !existing || previousOwner !== nextOwner; + } + if (workspaceAccessChanged) { + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + } + } + + subscribe(listener: (frame: WorkspaceStateFrame) => void, cursor?: Cursor, principal?: string): () => void { + if (cursor && cursor.epoch === this.epoch && cursor.seq < this.#workspaceSequence) + for (const item of this.#replay) if (item.frame.cursor.seq > cursor.seq && (principal === undefined || item.owner === principal)) listener(item.frame); + const subscription: WorkspaceSubscription = { listener, ...(principal === undefined ? {} : { principal }) }; + this.#workspaceListeners.add(subscription); + return () => { this.#workspaceListeners.delete(subscription); }; + } + subscribeSessions(listener: SessionListener): () => void { + this.#sessionListeners.add(listener); + return () => { this.#sessionListeners.delete(listener); }; + } + + #ownsSessionResource(resource: KubernetesResource, principal: string): boolean { + const workspace = text(record(resource.spec).workspaceRef); + return workspace.length > 0 && this.ownsWorkspace(workspace, principal); + } + #publishWorkspace(resource: KubernetesResource, deleted: boolean, owner?: string): void { + this.#workspaceSequence++; + const name = resource.metadata.name; + const currentRevision = revision(resourceRevision(resource)); + const frame: WorkspaceStateFrame = deleted + ? { v: "omp-app/1", type: "workspace.state", hostId: this.hostId, workspaceId: name, cursor: this.workspaceCursor, revision: currentRevision, remove: name } + : { v: "omp-app/1", type: "workspace.state", hostId: this.hostId, workspaceId: name, cursor: this.workspaceCursor, revision: currentRevision, upsert: this.#workspace(resource) }; + this.#replay.push({ frame, ...(owner === undefined ? {} : { owner }) }); + if (this.#replay.length > CLUSTER_WORKSPACE_REPLAY_FRAMES) this.#replay.shift(); + for (const subscription of this.#workspaceListeners) + if (subscription.principal === undefined || subscription.principal === owner) subscription.listener(frame); + } + #session(resource: KubernetesResource, authoritative: SessionRef): SessionRef { + const spec = record(resource.spec); + const status = record(resource.status); + const workspaceId = text(spec.workspaceRef, "unknown-workspace"); + const ci = record(spec.ci); + const condition = firstCondition(status); + const guiEnabled = record(spec.gui).enabled === true; + const infrastructurePhase = categorical(status.phase, ["Pending", "Running", "Failed", "Terminating", "Unknown"] as const, "Unknown"); + const guiState = !guiEnabled + ? "Unavailable" + : infrastructurePhase === "Running" + ? "Ready" + : infrastructurePhase === "Failed" || infrastructurePhase === "Terminating" + ? "Failed" + : "Starting"; + const { cluster: _upstreamCluster, ci: _upstreamCi, ...ompLiveState } = authoritative.liveState; + return decodeSessionRef({ + ...authoritative, + hostId: this.hostId, + sessionId: resource.metadata.name, + liveState: { + ...ompLiveState, + cluster: { + workspaceId, + phase: infrastructurePhase, + ...(condition ? { condition } : {}), + gui: { state: guiState }, + }, + ...(ci.repositoryId && ci.ref && ci.commit ? { ci: { + provider: "woodpecker", correlation: "unknown", repositoryId: ci.repositoryId, + ref: ci.ref, commit: ci.commit, + } } : {}), + }, + }, `session.${resource.metadata.name}`); + } + #workspace(resource: KubernetesResource): WorkspaceInfrastructureProjection { + const spec = record(resource.spec); + const status = record(resource.status); + const condition = firstCondition(status); + return decodeWorkspaceInfrastructureProjection({ + id: resource.metadata.name, + displayName: text(spec.displayName, resource.metadata.name).slice(0, 256), + phase: resource.metadata.deletionTimestamp ? "Terminating" : categorical(status.phase, ["Pending", "Ready", "Failed", "Terminating", "Unknown"] as const, "Unknown"), + retentionPolicy: spec.retentionPolicy === "Delete" ? "Delete" : "Retain", + ...(typeof spec.storageClass === "string" ? { storageClass: spec.storageClass.slice(0, 128) } : {}), + ...(typeof status.capacity === "string" ? { capacity: status.capacity.slice(0, 64) } : typeof spec.size === "string" ? { capacity: spec.size.slice(0, 64) } : {}), + accessMode: "ReadWriteMany", + revision: resourceRevision(resource), + ...(condition ? { condition } : {}), + }, `workspace.${resource.metadata.name}`); + } +} diff --git a/packages/cluster-server/src/kubernetes-runner.ts b/packages/cluster-server/src/kubernetes-runner.ts new file mode 100644 index 00000000..0278ab8e --- /dev/null +++ b/packages/cluster-server/src/kubernetes-runner.ts @@ -0,0 +1,120 @@ +import { KubernetesApiClient, KubernetesApiError } from "./kubernetes-client.ts"; +import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; + +export interface KubernetesProjectionRunnerOptions { + readonly client: KubernetesApiClient; + readonly projection: ClusterInfrastructureProjection; + readonly hostName: string; + readonly onSynchronized?: () => void; + readonly onError?: (error: unknown) => void; + readonly retryMs?: number; +} + +function delay(milliseconds: number, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const finish = (): void => { signal.removeEventListener("abort", abort); resolve(); }; + const timer = setTimeout(finish, milliseconds); + const abort = (): void => { clearTimeout(timer); reject(signal.reason); }; + signal.addEventListener("abort", abort, { once: true }); + }); +} + +/** Rebuildable list/watch runner. Kubernetes remains the sole infrastructure source. */ +export class KubernetesProjectionRunner { + readonly #options: KubernetesProjectionRunnerOptions; + readonly #synchronized = new Set(); + #abort?: AbortController; + #running?: Promise; + #relist?: Promise; + constructor(options: KubernetesProjectionRunnerOptions) { this.#options = options; } + + async start(): Promise { + if (this.#running) throw new Error("Kubernetes projection runner already started"); + this.#abort = new AbortController(); + const signal = this.#abort.signal; + const initial = await this.#options.client.listInfrastructure(this.#options.hostName, signal); + this.#options.projection.replace(initial); + this.#synchronized.clear(); + for (const resource of ["t4clusterhosts", "t4workspaces", "t4sessions"]) this.#synchronized.add(resource); + this.#options.onSynchronized?.(); + this.#running = this.#run(signal); + } + + async stop(): Promise { + this.#abort?.abort(new Error("Kubernetes projection runner stopped")); + await this.#running?.catch(() => undefined); + this.#running = undefined; + this.#abort = undefined; + this.#relist = undefined; + this.#synchronized.clear(); + } + + async #run(signal: AbortSignal): Promise { + const resources = ["t4clusterhosts", "t4workspaces", "t4sessions"] as const; + while (!signal.aborted) { + const generation = new AbortController(); + const stopGeneration = (): void => generation.abort(signal.reason); + signal.addEventListener("abort", stopGeneration, { once: true }); + await Promise.all(resources.map(resource => this.#watchResource(resource, resources.length, signal, generation))); + signal.removeEventListener("abort", stopGeneration); + } + } + + async #watchResource( + resource: string, + resourceCount: number, + rootSignal: AbortSignal, + generation: AbortController, + ): Promise { + let version = this.#options.projection.resourceVersionFor(resource); + while (!generation.signal.aborted) { + try { + await this.#options.client.watch( + resource, + version, + event => { + this.#options.projection.applyWatch(event); + version = event.object.metadata.resourceVersion ?? version; + }, + generation.signal, + () => { + this.#synchronized.add(resource); + if (this.#synchronized.size === resourceCount) this.#options.onSynchronized?.(); + }, + ); + if (generation.signal.aborted) return; + this.#synchronized.delete(resource); + this.#options.onError?.(new Error(`Kubernetes ${resource} watch ended`)); + } catch (error) { + if (generation.signal.aborted || rootSignal.aborted) return; + this.#synchronized.delete(resource); + this.#options.onError?.(error); + if (error instanceof KubernetesApiError && error.status === 410) { + try { await this.#restartGeneration(generation, rootSignal); } + catch (relistError) { + if (rootSignal.aborted) return; + this.#options.onError?.(relistError); + try { await delay(this.#options.retryMs ?? 1_000, generation.signal); } catch { return; } + continue; + } + return; + } + } + try { await delay(this.#options.retryMs ?? 1_000, generation.signal); } + catch { return; } + } + } + + async #restartGeneration(generation: AbortController, signal: AbortSignal): Promise { + if (!this.#relist) { + this.#relist = (async () => { + const snapshot = await this.#options.client.listInfrastructure(this.#options.hostName, signal); + this.#options.projection.replace(snapshot); + this.#synchronized.clear(); + generation.abort(new Error("Kubernetes watch generation relisted")); + })().finally(() => { this.#relist = undefined; }); + } + await this.#relist; + } +} diff --git a/packages/cluster-server/src/main.ts b/packages/cluster-server/src/main.ts new file mode 100755 index 00000000..42e96890 --- /dev/null +++ b/packages/cluster-server/src/main.ts @@ -0,0 +1,97 @@ +#!/usr/bin/env bun +import { clusterServerConfigFromEnv, loadKubernetesCredentials } from "./config.ts"; +import { ClusterGateway } from "./gateway.ts"; +import { KubernetesApiClient, KubernetesGatewayMutationBackend } from "./kubernetes-client.ts"; +import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; +import { KubernetesProjectionRunner } from "./kubernetes-runner.ts"; +import { ClusterMetrics, ClusterServerHealth, JsonLogger } from "./observability.ts"; +import { WebSocketPodHostConnector } from "./pod-host-router.ts"; +import { startClusterHttpServers } from "./server.ts"; +import { SessionAuthorityRunner } from "./session-authority-runner.ts"; +import { WoodpeckerProvider } from "./woodpecker.ts"; + +export async function runClusterServer(env: Readonly> = process.env): Promise { + const config = clusterServerConfigFromEnv(env); + const logger = new JsonLogger(undefined, { component: "cluster-server", version: "0.1.30", namespace: config.namespace }); + const health = new ClusterServerHealth(); + const metrics = new ClusterMetrics({ component: "cluster-server", version: "0.1.30", namespace: config.namespace }); + const credentials = await loadKubernetesCredentials(config); + const kubernetes = new KubernetesApiClient({ + baseUrl: config.kubernetesBaseUrl, + namespace: config.namespace, + token: credentials.token, + ca: credentials.ca, + }); + const projection = new ClusterInfrastructureProjection({ epoch: config.epoch, namespace: config.namespace }); + const runner = new KubernetesProjectionRunner({ + client: kubernetes, + projection, + hostName: config.hostName, + onSynchronized: () => health.markKubernetesSynced(), + onError: error => { + health.markKubernetesUnavailable(); + logger.warn("Kubernetes watch reconnecting", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }); + }, + }); + await runner.start(); + const connector = new WebSocketPodHostConnector({ internalToken: config.internalToken }); + const authority = new SessionAuthorityRunner({ + projection, + connector, + onError: error => logger.warn("session authority reconnecting", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }), + }); + authority.start(); + const ciProvider = config.woodpecker ? new WoodpeckerProvider(config.woodpecker) : undefined; + const gateway = new ClusterGateway({ + projection, + connector, + mutations: new KubernetesGatewayMutationBackend({ client: kubernetes, hostRef: config.hostName }), + internalToken: config.internalToken, + ...(ciProvider ? { ciProvider } : {}), + }); + const servers = startClusterHttpServers({ + gateway, + projection, + gatewayPort: config.gatewayPort, + adminPort: config.adminPort, + trustedProxyAddresses: config.trustedProxyAddresses, + trustedProxyCidrs: config.trustedProxyCidrs, + health, + metrics, + logger, + }); + const stopped = Promise.withResolvers(); + let stopping = false; + const stop = (): void => { + if (stopping) return; + stopping = true; + void (async () => { + await servers.drain(); + await authority.stop(); + await runner.stop(); + await servers.stop(); + })().then(stopped.resolve, stopped.reject); + }; + process.once("SIGTERM", stop); + process.once("SIGINT", stop); + try { await stopped.promise; } + finally { + process.off("SIGTERM", stop); + process.off("SIGINT", stop); + if (!stopping) { + await servers.stop(); + await authority.stop(); + await runner.stop(); + } + } +} + +async function main(): Promise { + try { await runClusterServer(); } + catch (error) { + const logger = new JsonLogger(undefined, { component: "cluster-server", version: "0.1.30" }); + logger.error("cluster server failed", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }); + process.exitCode = 1; + } +} +if (import.meta.main) await main(); diff --git a/packages/cluster-server/src/observability.ts b/packages/cluster-server/src/observability.ts new file mode 100644 index 00000000..42d2afe5 --- /dev/null +++ b/packages/cluster-server/src/observability.ts @@ -0,0 +1,117 @@ +const REDACTED = "[REDACTED]"; +const SECRET_MARKERS = ["password", "passwd", "secret", "token", "credential", "apikey", "privatekey", "cookie", "auth", "pairing", "prompt", "transcript", "path"]; +const METRIC_LABELS = new Set(["component", "version", "namespace", "workspace", "session", "condition", "provider", "result"]); +const METRIC_NAME = /^[a-zA-Z_:][a-zA-Z0-9_:]*$/u; +const LABEL_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/u; + +function redactKey(key: string): boolean { + const normalized = key.normalize("NFKC").toLowerCase().replace(/[^a-z0-9]+/gu, ""); + return SECRET_MARKERS.some(marker => normalized.includes(marker)); +} +export function redactStructuredValue(value: unknown, seen = new WeakSet()): unknown { + if (Array.isArray(value)) return value.slice(0, 1_000).map(item => redactStructuredValue(item, seen)); + if (!value || typeof value !== "object") return typeof value === "string" && value.length > 2_048 ? `${value.slice(0, 2_048)}…` : value; + if (seen.has(value)) return "[CYCLE]"; + seen.add(value); + const output: Record = {}; + for (const [key, child] of Object.entries(value as Record).slice(0, 128)) + output[key] = redactKey(key) ? REDACTED : redactStructuredValue(child, seen); + return output; +} + +export class JsonLogger { + constructor( + private readonly write: (line: string) => void = line => process.stdout.write(`${line}\n`), + private readonly base: Readonly> = {}, + ) {} + info(message: string, fields: Readonly> = {}): void { this.#log("info", message, fields); } + warn(message: string, fields: Readonly> = {}): void { this.#log("warn", message, fields); } + error(message: string, fields: Readonly> = {}): void { this.#log("error", message, fields); } + #log(level: "info" | "warn" | "error", message: string, fields: Readonly>): void { + const value = redactStructuredValue({ timestamp: new Date().toISOString(), level, message: message.slice(0, 512), ...this.base, ...fields }); + this.write(JSON.stringify(value)); + } +} + +interface MetricValue { value: number; labels: Readonly>; } +function labelsKey(labels: Readonly>): string { + return Object.entries(labels).sort(([a], [b]) => a.localeCompare(b)).map(([key, value]) => `${key}=${value}`).join("\u0000"); +} +function escaped(value: string): string { return value.replace(/\\/gu, "\\\\").replace(/\n/gu, "\\n").replace(/"/gu, '\\"'); } + +export class ClusterMetrics { + readonly #base: Readonly>; + readonly #metrics = new Map(); + constructor(base: Readonly>) { + this.#base = this.#labels(base); + } + increment(name: string, labels: Readonly>, amount = 1): void { + this.#name(name); + if (!Number.isFinite(amount) || amount < 0) throw new Error("metric increment is invalid"); + const merged = this.#labels({ ...this.#base, ...labels }); + const key = `${name}\u0000${labelsKey(merged)}`; + const current = this.#metrics.get(key); + this.#metrics.set(key, { labels: merged, value: (current?.value ?? 0) + amount }); + } + set(name: string, value: number, labels: Readonly> = {}): void { + this.#name(name); + if (!Number.isFinite(value)) throw new Error("metric value is invalid"); + const merged = this.#labels({ ...this.#base, ...labels }); + this.#metrics.set(`${name}\u0000${labelsKey(merged)}`, { labels: merged, value }); + } + render(): string { + return [...this.#metrics.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([key, metric]) => { + const name = key.slice(0, key.indexOf("\u0000")); + const labels = Object.entries(metric.labels).sort(([a], [b]) => a.localeCompare(b)).map(([label, value]) => `${label}="${escaped(value)}"`).join(","); + return `${name}${labels ? `{${labels}}` : ""} ${metric.value}`; + }).join("\n") + "\n"; + } + #name(name: string): void { if (!METRIC_NAME.test(name)) throw new Error("metric name is invalid"); } + #labels(labels: Readonly>): Readonly> { + if (Object.keys(labels).length > 8) throw new Error("metric label limit exceeded"); + const output: Record = {}; + for (const [key, value] of Object.entries(labels)) { + if (!LABEL_NAME.test(key) || !METRIC_LABELS.has(key) || redactKey(key) || typeof value !== "string" || value.length === 0 || new TextEncoder().encode(value).byteLength > 128) + throw new Error("metric label is invalid"); + output[key] = value; + } + return Object.freeze(output); + } +} + +export class ClusterServerHealth { + #kubernetesSynced = false; + #gatewayListening = false; + #draining = false; + markKubernetesSynced(): void { this.#kubernetesSynced = true; } + markKubernetesUnavailable(): void { this.#kubernetesSynced = false; } + markGatewayListening(): void { this.#gatewayListening = true; } + markGatewayStopped(): void { this.#gatewayListening = false; } + beginDrain(): void { this.#draining = true; } + get healthy(): boolean { return true; } + get ready(): boolean { return this.#kubernetesSynced && this.#gatewayListening && !this.#draining; } + get draining(): boolean { return this.#draining; } +} + +export interface AdminHandlerOptions { + readonly health: ClusterServerHealth; + readonly metrics: ClusterMetrics; + readonly drain: () => Promise | void; +} +function json(value: unknown, status = 200): Response { + return Response.json(value, { status, headers: { "cache-control": "no-store" } }); +} +export function createAdminHandler(options: AdminHandlerOptions): (request: Request) => Promise { + return async request => { + const path = new URL(request.url).pathname; + if (path === "/healthz") return request.method === "GET" ? json({ healthy: options.health.healthy }) : new Response(null, { status: 405 }); + if (path === "/readyz") return request.method === "GET" ? json({ ready: options.health.ready }, options.health.ready ? 200 : 503) : new Response(null, { status: 405 }); + if (path === "/metrics") return request.method === "GET" ? new Response(options.metrics.render(), { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8", "cache-control": "no-store" } }) : new Response(null, { status: 405 }); + if (path === "/drainz") { + if (request.method !== "POST") return new Response(null, { status: 405 }); + await options.drain(); + return json({ draining: true }, 202); + } + return new Response("not found", { status: 404 }); + }; +} diff --git a/packages/cluster-server/src/pod-host-router.ts b/packages/cluster-server/src/pod-host-router.ts new file mode 100644 index 00000000..1debb8bc --- /dev/null +++ b/packages/cluster-server/src/pod-host-router.ts @@ -0,0 +1,131 @@ +import { + DEVICE_CAPABILITIES, + PROTOCOL_FEATURES, + decodeServerFrame, + parseBounded, + type ClientFrame, + type HostId, + type ServerFrame, +} from "@t4-code/host-wire"; +import { canonicalInternalDeviceToken } from "./session-host-policy.ts"; + +export interface PodHostEndpoint { + readonly clusterSessionId: string; + readonly url: string; +} +export interface PodHostRoute extends PodHostEndpoint { + readonly upstreamSessionId: string; +} +export interface PodHostConnection { + readonly hostId?: string; + send(frame: ClientFrame): void; + close(code?: number, reason?: string): void; +} +export interface PodHostConnector { + connect(endpoint: PodHostEndpoint, onFrame: (frame: ServerFrame) => void, onClose?: () => void): Promise; +} +export interface WebSocketPodHostConnectorOptions { + readonly internalToken: string; + readonly openTimeoutMs?: number; + readonly webSocketFactory?: (url: string) => WebSocket; +} + +export class WebSocketPodHostConnector implements PodHostConnector { + readonly #token: string; + readonly #timeoutMs: number; + readonly #factory: (url: string) => WebSocket; + constructor(options: WebSocketPodHostConnectorOptions) { + this.#token = canonicalInternalDeviceToken(options.internalToken); + this.#timeoutMs = options.openTimeoutMs ?? 10_000; + this.#factory = options.webSocketFactory ?? (url => new WebSocket(url)); + } + connect(endpoint: PodHostEndpoint, onFrame: (frame: ServerFrame) => void, onClose?: () => void): Promise { + if (!endpoint.url.startsWith("ws://") && !endpoint.url.startsWith("wss://")) return Promise.reject(new Error("pod host URL is invalid")); + const socket = this.#factory(endpoint.url); + return new Promise((resolve, reject) => { + let settled = false; + let upstreamHostId: string | undefined; + const timer = setTimeout(() => { + if (!settled) { settled = true; socket.close(1013, "upstream timeout"); reject(new Error("pod host handshake timed out")); } + }, this.#timeoutMs); + socket.addEventListener("open", () => { + socket.send(JSON.stringify({ + v: "omp-app/1", type: "hello", + protocol: { min: "omp-app/1", max: "omp-app/1" }, + client: { name: "cluster-server", version: "0.1.30", build: "cluster", platform: "linux" }, + requestedFeatures: PROTOCOL_FEATURES.filter(feature => feature !== "cluster.operator"), + savedCursors: [], + capabilities: { client: DEVICE_CAPABILITIES.filter(capability => capability !== "ci.trigger") }, + authentication: { deviceId: "cluster-server", deviceToken: this.#token }, + })); + }); + socket.addEventListener("message", event => { + try { + const frame = decodeServerFrame(parseBounded(typeof event.data === "string" ? event.data : new Uint8Array(event.data as ArrayBuffer))); + if (frame.type === "welcome") { + if (settled) throw new Error("duplicate pod host welcome"); + if (frame.authentication !== "paired") throw new Error("pod host authentication failed"); + upstreamHostId = frame.hostId; + settled = true; + clearTimeout(timer); + resolve({ + get hostId() { return upstreamHostId; }, + send: value => { + if (socket.readyState !== WebSocket.OPEN) throw new Error("pod host connection is closed"); + socket.send(JSON.stringify(value)); + }, + close: (code, reason) => socket.close(code, reason), + }); + return; + } + if (!settled) throw new Error("pod host frame arrived before welcome"); + onFrame(frame); + } catch (error) { + if (!settled) { settled = true; clearTimeout(timer); reject(error); } + else socket.close(1002, "invalid upstream frame"); + } + }); + socket.addEventListener("error", () => { + if (!settled) { settled = true; clearTimeout(timer); reject(new Error("pod host connection failed")); } + }); + socket.addEventListener("close", () => { + if (!settled) { settled = true; clearTimeout(timer); reject(new Error("pod host connection closed")); } + else onClose?.(); + }); + }); + } +} + +/** Only app-wire address fields are translated; opaque ids and payload values remain byte-semantically unchanged. */ +export function rewriteClientAddress( + frame: ClientFrame, + route: PodHostRoute, + upstreamHostId: string, +): ClientFrame { + if (!("hostId" in frame)) return frame; + return { + ...frame, + hostId: upstreamHostId as HostId, + ...(frame.sessionId === undefined ? {} : { sessionId: route.upstreamSessionId }), + } as ClientFrame; +} + +function rewriteEntry(entry: Record, clusterHostId: string, clusterSessionId: string): Record { + return { ...entry, hostId: clusterHostId, sessionId: clusterSessionId }; +} +export function rewriteServerAddress( + frame: ServerFrame, + route: PodHostRoute, + clusterHostId: string, +): ServerFrame { + let output: Record = { ...frame }; + if ("hostId" in output) output.hostId = clusterHostId; + if ("sessionId" in output) output.sessionId = route.clusterSessionId; + if (frame.type === "entry") output.entry = rewriteEntry(frame.entry as unknown as Record, clusterHostId, route.clusterSessionId); + if (frame.type === "snapshot") output.entries = frame.entries.map(entry => rewriteEntry(entry as unknown as Record, clusterHostId, route.clusterSessionId)); + if (frame.type === "agent.transcript") output.entries = frame.entries.map(entry => rewriteEntry(entry as unknown as Record, clusterHostId, route.clusterSessionId)); + if (frame.type === "session.delta" && frame.upsert) output.upsert = { ...frame.upsert, hostId: clusterHostId, sessionId: route.clusterSessionId }; + if (frame.type === "session.delta" && frame.remove) output.remove = route.clusterSessionId; + if (frame.type === "sessions") output.sessions = frame.sessions.map(ref => ({ ...ref, hostId: clusterHostId, sessionId: route.clusterSessionId })); + return output as unknown as ServerFrame; +} diff --git a/packages/cluster-server/src/server.ts b/packages/cluster-server/src/server.ts new file mode 100644 index 00000000..df91df4d --- /dev/null +++ b/packages/cluster-server/src/server.ts @@ -0,0 +1,117 @@ +import { BlockList, isIP } from "node:net"; +import { normalizeIpAddress } from "@t4-code/host-service"; +import type { GatewayConnection, ClusterGateway } from "./gateway.ts"; +import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; +import { ClusterMetrics, ClusterServerHealth, JsonLogger, createAdminHandler } from "./observability.ts"; + +interface SocketData { connection?: GatewayConnection; principal: string; } +export interface ClusterHttpServersOptions { + readonly gateway: ClusterGateway; + readonly projection: ClusterInfrastructureProjection; + readonly gatewayPort: number; + readonly adminPort: number; + readonly trustedProxyAddresses?: readonly string[]; + readonly trustedProxyCidrs?: readonly string[]; + readonly health: ClusterServerHealth; + readonly metrics: ClusterMetrics; + readonly logger: JsonLogger; +} +export interface ClusterHttpServers { + drain(): Promise; + stop(): Promise; +} +function trustedProxyMatcher(addresses: readonly string[], cidrs: readonly string[]): (address: string) => boolean { + const exact = new Set(addresses.map(normalizeIpAddress)); + const subnets = new BlockList(); + for (const cidr of cidrs) { + const [address, prefixText] = cidr.split("/"); + const family = isIP(address!); + if ((family !== 4 && family !== 6) || !/^(?:0|[1-9][0-9]*)$/u.test(prefixText ?? "")) + throw new Error("trusted proxy CIDR is invalid"); + subnets.addSubnet(address!, Number(prefixText), family === 4 ? "ipv4" : "ipv6"); + } + return address => { + const normalized = normalizeIpAddress(address); + const family = isIP(normalized); + return exact.has(normalized) || family !== 0 && subnets.check(normalized, family === 4 ? "ipv4" : "ipv6"); + }; +} + +function gatewayPrincipal(request: Request, remoteAddress: string, trustedSource: (address: string) => boolean): string | undefined { + if (!trustedSource(remoteAddress)) return undefined; + if (request.headers.get("x-forwarded-proto") !== "https") return undefined; + const principal = request.headers.get("tailscale-user-login") ?? request.headers.get("tailscale-user-name"); + if (!principal || principal !== principal.trim() || new TextEncoder().encode(principal).byteLength > 256 || /[\u0000-\u001f\u007f]/u.test(principal)) return undefined; + return principal; +} + +export function startClusterHttpServers(options: ClusterHttpServersOptions): ClusterHttpServers { + let draining = false; + const trustedSource = trustedProxyMatcher(options.trustedProxyAddresses ?? [], options.trustedProxyCidrs ?? []); + const gatewayServer = Bun.serve({ + hostname: "0.0.0.0", + port: options.gatewayPort, + fetch(request, server) { + const url = new URL(request.url); + if (url.pathname !== "/omp-app/v1") return new Response("not found", { status: 404 }); + if (request.method !== "GET") return new Response("method not allowed", { status: 405 }); + if (draining) return new Response("draining", { status: 503 }); + const remoteAddress = server.requestIP(request)?.address; + if (!remoteAddress) return new Response("authenticated proxy required", { status: 401 }); + const principal = gatewayPrincipal(request, remoteAddress, trustedSource); + if (!principal) return new Response("authenticated Tailscale proxy identity required", { status: 401 }); + const origin = request.headers.get("origin"); + if (origin && !options.projection.allowedOrigins().includes(origin)) return new Response("forbidden", { status: 403 }); + if (!server.upgrade(request, { data: { principal } })) return new Response("upgrade required", { status: 426 }); + return undefined; + }, + websocket: { + maxPayloadLength: 1_048_576, + idleTimeout: 120, + backpressureLimit: 1_048_576, + closeOnBackpressureLimit: true, + perMessageDeflate: false, + open(socket) { + socket.data.connection = options.gateway.connect({ + send(frame) { + const accepted = socket.send(JSON.stringify(frame)); + if (accepted <= 0) socket.close(1013, "gateway backpressure"); + }, + close(code, reason) { socket.close(code, reason); }, + }, socket.data.principal); + options.metrics.set("t4_cluster_gateway_connections", options.gateway.connectionCount); + }, + message(socket, message) { + const input = typeof message === "string" ? message : new Uint8Array(message); + void socket.data.connection?.receive(input).catch(() => socket.close(1011, "gateway error")); + }, + close(socket) { + socket.data.connection?.close(); + socket.data.connection = undefined; + options.metrics.set("t4_cluster_gateway_connections", options.gateway.connectionCount); + }, + }, + }); + let stopping: Promise | undefined; + const drain = async (): Promise => { + if (stopping) return await stopping; + draining = true; + options.health.beginDrain(); + options.gateway.beginDrain(); + stopping = gatewayServer.stop(false).then(() => undefined); + options.logger.info("gateway drain started", { result: "success" }); + await stopping; + options.health.markGatewayStopped(); + }; + const adminHandler = createAdminHandler({ health: options.health, metrics: options.metrics, drain }); + const adminServer = Bun.serve({ hostname: "0.0.0.0", port: options.adminPort, fetch: adminHandler }); + options.health.markGatewayListening(); + options.logger.info("cluster server listening", { result: "success" }); + return { + drain, + async stop() { + await drain(); + await adminServer.stop(true); + }, + }; +} diff --git a/packages/cluster-server/src/session-authority-runner.ts b/packages/cluster-server/src/session-authority-runner.ts new file mode 100644 index 00000000..307cd21d --- /dev/null +++ b/packages/cluster-server/src/session-authority-runner.ts @@ -0,0 +1,100 @@ +import type { ServerFrame, SessionRef } from "@t4-code/host-wire"; +import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; +import type { PodHostConnection, PodHostConnector, PodHostEndpoint } from "./pod-host-router.ts"; + +export interface SessionAuthorityRunnerOptions { + readonly projection: ClusterInfrastructureProjection; + readonly connector: PodHostConnector; + readonly retryMs?: number; + readonly onError?: (error: unknown) => void; +} + +interface AuthorityConnection { + readonly endpoint: PodHostEndpoint; + readonly pending: Promise; +} + +/** Reconstructs session truth from each pod's authenticated omp-app/1 inventory. */ +export class SessionAuthorityRunner { + readonly #options: SessionAuthorityRunnerOptions; + readonly #connections = new Map(); + #unsubscribe?: () => void; + #retry?: ReturnType; + #started = false; + + constructor(options: SessionAuthorityRunnerOptions) { this.#options = options; } + + start(): void { + if (this.#started) throw new Error("session authority runner already started"); + this.#started = true; + this.#unsubscribe = this.#options.projection.subscribeSessions(() => this.#reconcile()); + this.#reconcile(); + } + + async stop(): Promise { + if (!this.#started) return; + this.#started = false; + this.#unsubscribe?.(); + this.#unsubscribe = undefined; + if (this.#retry) clearTimeout(this.#retry); + this.#retry = undefined; + const pending = [...this.#connections.values()].map(entry => entry.pending); + this.#connections.clear(); + await Promise.allSettled(pending.map(async connection => (await connection).close(1001, "cluster server stopping"))); + } + + #reconcile(): void { + if (!this.#started) return; + const endpoints = new Map(this.#options.projection.sessionEndpoints().map(endpoint => [endpoint.clusterSessionId, endpoint])); + for (const [session, existing] of this.#connections) { + const current = endpoints.get(session); + if (current?.url === existing.endpoint.url) continue; + this.#connections.delete(session); + this.#options.projection.clearSessionAuthority(session); + void existing.pending.then(connection => connection.close(1001, "session endpoint changed"), () => undefined); + } + for (const [session, endpoint] of endpoints) { + if (this.#connections.has(session)) continue; + let pending: Promise; + pending = this.#options.connector.connect( + endpoint, + frame => this.#projectFrame(endpoint, frame), + () => this.#disconnected(session, pending), + ); + this.#connections.set(session, { endpoint, pending }); + pending.catch(error => { + if (this.#connections.get(session)?.pending !== pending) return; + this.#connections.delete(session); + this.#options.projection.clearSessionAuthority(session); + this.#options.onError?.(error); + this.#scheduleRetry(); + }); + } + } + + #projectFrame(endpoint: PodHostEndpoint, frame: ServerFrame): void { + if (frame.type === "sessions") { + if (frame.sessions.length !== 1) throw new Error("session pod must expose exactly one authoritative OMP session"); + this.#options.projection.setSessionAuthority(endpoint.clusterSessionId, frame.sessions[0] as SessionRef); + return; + } + if (frame.type !== "session.delta") return; + if (frame.upsert) this.#options.projection.setSessionAuthority(endpoint.clusterSessionId, frame.upsert); + else if (frame.remove) this.#options.projection.clearSessionAuthority(endpoint.clusterSessionId); + } + + #disconnected(session: string, pending: Promise): void { + if (this.#connections.get(session)?.pending !== pending) return; + this.#connections.delete(session); + this.#options.projection.clearSessionAuthority(session); + this.#scheduleRetry(); + } + + #scheduleRetry(): void { + if (!this.#started || this.#retry) return; + this.#retry = setTimeout(() => { + this.#retry = undefined; + this.#reconcile(); + }, this.#options.retryMs ?? 1_000); + } +} diff --git a/packages/cluster-server/src/session-host-main.ts b/packages/cluster-server/src/session-host-main.ts new file mode 100755 index 00000000..57bca589 --- /dev/null +++ b/packages/cluster-server/src/session-host-main.ts @@ -0,0 +1,114 @@ +#!/usr/bin/env bun +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { + OmpAuthorityBridgeClient, + TranscriptSearchIndex, + appserverSupportedCapabilities, + appserverSupportedFeatures, + createAppserver, + type AppserverHandle, +} from "@t4-code/host-service"; +import { hostId } from "@t4-code/host-wire"; +import { ClusterInternalRemotePolicy, sessionHostConfigFromEnv, type SessionHostConfig } from "./session-host-policy.ts"; + +const OMP_VERSION = "17.0.5"; +const OMP_COMMIT = "8476f4451ed95c5d5401785d279a93d3c659fac4"; + +export async function runSessionHost( + config: SessionHostConfig, + registerSignal: (signal: "SIGINT" | "SIGTERM", listener: () => void) => void = (signal, listener) => process.on(signal, listener), +): Promise { + const home = join(config.stateRoot, "home"); + const runtime = join(config.stateRoot, "run"); + await Promise.all([mkdir(home, { recursive: true, mode: 0o700 }), mkdir(runtime, { recursive: true, mode: 0o700 })]); + const bridge = new OmpAuthorityBridgeClient({ + executable: config.ompExecutable, + cwd: "/workspace", + environment: { + OMP_PROFILE: config.sessionName, + HOME: home, + XDG_CONFIG_HOME: join(home, ".config"), + XDG_STATE_HOME: join(home, ".local", "state"), + XDG_CACHE_HOME: join(home, ".cache"), + XDG_RUNTIME_DIR: runtime, + }, + }); + const ready = await bridge.start(); + if (ready.ompVersion !== OMP_VERSION || ready.ompBuild !== OMP_COMMIT) { + await bridge.stop(); + throw new Error("session pod OMP authority identity does not match the pinned release"); + } + let appserver: AppserverHandle | undefined; + const search = new TranscriptSearchIndex(join(config.stateRoot, "transcript-search.sqlite")); + try { + const authorities = bridge.createAuthorities(); + const existing = await authorities.sessionAuthority.list(); + if (existing.length > 1) throw new Error("session pod contains more than one authoritative OMP session"); + if (existing.length === 0) await authorities.sessionAuthority.create("/workspace", config.sessionName); + const hostInfo = await authorities.hostInfo(); + const base = { + hostId: hostId(`pod:${config.sessionName}`), + ...(process.env.POD_UID ? { epoch: `pod:${process.env.POD_UID}` } : {}), + socketPath: join(runtime, "appserver.sock"), + attentionOutcomePath: join(config.stateRoot, "attention-outcomes.json"), + ompVersion: ready.ompVersion, + ompBuild: ready.ompBuild, + appserverVersion: "0.1.30", + appserverBuild: "cluster-session", + sessionAuthority: authorities.sessionAuthority, + discovery: authorities.discovery, + operationsAuthority: authorities.operationsAuthority, + ...(authorities.usageAuthority ? { usageAuthority: authorities.usageAuthority } : {}), + transcriptSearchAuthority: search, + projectRootForProject: authorities.projectRootForProject, + lockCheck: authorities.lockCheck, + lockStatus: authorities.lockStatus, + transcriptImageRoot: hostInfo.transcriptImageRoot, + rpcChildInvocation: { executable: config.ompExecutable, prefixArgv: [] }, + }; + const policy = new ClusterInternalRemotePolicy({ + token: config.internalToken, + supportedCapabilities: appserverSupportedCapabilities(base), + supportedFeatures: appserverSupportedFeatures(base, true), + }); + appserver = createAppserver({ + ...base, + remoteEndpoint: { + address: "0.0.0.0", + port: config.port, + internalPeerNodeId: "cluster-server", + originAllowlist: [], + maxConnections: 8, + maxFrameBytes: 1_048_576, + idleTimeoutSeconds: 120, + backpressureLimit: 1_048_576, + }, + remotePolicy: policy, + }); + await appserver.start(); + const stopped = Promise.withResolvers(); + let stopping = false; + const stop = (): void => { + if (stopping) return; + stopping = true; + void appserver!.stop().then(stopped.resolve, stopped.reject); + }; + registerSignal("SIGINT", stop); + registerSignal("SIGTERM", stop); + await stopped.promise; + } finally { + await appserver?.stop().catch(() => undefined); + await Promise.resolve(search.close()).catch(() => undefined); + await bridge.stop(); + } +} + +async function main(): Promise { + try { await runSessionHost(sessionHostConfigFromEnv(process.env)); } + catch (error) { + process.stderr.write(`${JSON.stringify({ component: "session-host", level: "error", message: error instanceof Error ? error.message : "session host failed" })}\n`); + process.exitCode = 1; + } +} +if (import.meta.main) await main(); diff --git a/packages/cluster-server/src/session-host-policy.ts b/packages/cluster-server/src/session-host-policy.ts new file mode 100644 index 00000000..7d2f2ad8 --- /dev/null +++ b/packages/cluster-server/src/session-host-policy.ts @@ -0,0 +1,95 @@ +import { createHash, timingSafeEqual } from "node:crypto"; +import { isAbsolute } from "node:path"; +import { + requiredCapability, + type ClientFrame, + type HelloFrame, +} from "@t4-code/host-wire"; +import type { + RemoteAuthorizationContext, + RemoteConnectionPolicy, + RemoteHelloDecision, +} from "@t4-code/host-service"; +import type { RemoteConnection } from "@t4-code/host-service"; + +const SESSION_NAME = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; +interface ConnectionGrant { capabilities: Set; features: Set; } +export interface ClusterInternalRemotePolicyOptions { + readonly token: string; + readonly supportedCapabilities: readonly string[]; + readonly supportedFeatures: readonly string[]; +} + +/** Converts the mounted secret to the canonical 32-byte base64url token used by omp-app/1. */ +export function canonicalInternalDeviceToken(secret: string): string { + if (new TextEncoder().encode(secret).byteLength < 32 || secret.length > 16_384) throw new Error("cluster internal token is invalid"); + return createHash("sha256").update(secret, "utf8").digest("base64url"); +} +function sameToken(expectedSecret: string, supplied: string): boolean { + try { + const expected = Buffer.from(canonicalInternalDeviceToken(expectedSecret)); + const canonical = /^[A-Za-z0-9_-]{43}$/u.test(supplied) ? supplied : canonicalInternalDeviceToken(supplied); + const actual = Buffer.from(canonical); + return expected.length === actual.length && timingSafeEqual(expected, actual); + } catch { return false; } +} + +export class ClusterInternalRemotePolicy implements RemoteConnectionPolicy { + readonly #token: string; + readonly #capabilities: readonly string[]; + readonly #features: readonly string[]; + readonly #connections = new Map(); + constructor(options: ClusterInternalRemotePolicyOptions) { + canonicalInternalDeviceToken(options.token); + this.#token = options.token; + this.#capabilities = [...new Set(options.supportedCapabilities)].filter(value => value !== "ci.trigger"); + this.#features = [...new Set(options.supportedFeatures)].filter(value => value !== "cluster.operator"); + } + async authenticate(connection: RemoteConnection, hello: HelloFrame): Promise { + const authentication = hello.authentication; + if (connection.peer.identity.nodeId !== "cluster-server" || authentication?.deviceId !== "cluster-server" || !sameToken(this.#token, authentication.deviceToken)) { + this.#connections.delete(connection.connectionId); + return { authenticated: false, authentication: "denied", grantedCapabilities: [], grantedFeatures: [] }; + } + const requestedCapabilities = new Set(hello.capabilities?.client ?? this.#capabilities); + const requestedFeatures = new Set(hello.requestedFeatures); + const grantedCapabilities = this.#capabilities.filter(value => requestedCapabilities.has(value)); + const grantedFeatures = this.#features.filter(value => requestedFeatures.has(value)); + this.#connections.set(connection.connectionId, { capabilities: new Set(grantedCapabilities), features: new Set(grantedFeatures) }); + return { authenticated: true, authentication: "paired", deviceId: "cluster-server", grantedCapabilities, grantedFeatures }; + } + authorize(connection: RemoteConnection, frame: ClientFrame, _context: RemoteAuthorizationContext): boolean { + const grant = this.#connections.get(connection.connectionId); + if (!grant) return false; + if (frame.type === "confirm") return true; + if (frame.type === "ping") return true; + if (frame.type === "terminal.input") return grant.features.has("terminal.io") && grant.capabilities.has("term.input"); + if (frame.type === "terminal.resize") return grant.features.has("terminal.io") && grant.capabilities.has("term.resize"); + if (frame.type === "terminal.close") return grant.features.has("terminal.io") && grant.capabilities.has("term.open"); + if (frame.type !== "command") return false; + const capability = requiredCapability(frame.command); + return capability !== undefined && grant.capabilities.has(capability); + } + disconnected(connection: RemoteConnection): void { this.#connections.delete(connection.connectionId); } +} + +export interface SessionHostConfig { + readonly internalToken: string; + readonly sessionName: string; + readonly ompExecutable: string; + readonly stateRoot: string; + readonly port: number; +} +export function sessionHostConfigFromEnv(env: Readonly>): SessionHostConfig { + const internalToken = env.T4_CLUSTER_INTERNAL_TOKEN ?? ""; + canonicalInternalDeviceToken(internalToken); + const sessionName = env.T4_SESSION_NAME ?? ""; + if (!SESSION_NAME.test(sessionName)) throw new Error("T4_SESSION_NAME is invalid"); + const ompExecutable = env.T4_OMP_EXECUTABLE ?? "/opt/t4/bin/omp"; + if (!isAbsolute(ompExecutable)) throw new Error("T4_OMP_EXECUTABLE must be absolute"); + const stateRoot = env.T4_SESSION_STATE_ROOT ?? `/workspace/.t4/sessions/${sessionName}`; + if (stateRoot !== `/workspace/.t4/sessions/${sessionName}`) throw new Error("T4_SESSION_STATE_ROOT must select the isolated session directory"); + const port = Number(env.T4_SESSION_HOST_PORT ?? "8787"); + if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) throw new Error("T4_SESSION_HOST_PORT is invalid"); + return { internalToken, sessionName, ompExecutable, stateRoot, port }; +} diff --git a/packages/cluster-server/src/woodpecker.ts b/packages/cluster-server/src/woodpecker.ts new file mode 100644 index 00000000..002e7552 --- /dev/null +++ b/packages/cluster-server/src/woodpecker.ts @@ -0,0 +1,210 @@ +import type { CiRunResult, SessionCiState } from "@t4-code/host-wire"; + +export const WOODPECKER_MAX_PIPELINES = 100; +export const WOODPECKER_MAX_RESPONSE_BYTES = 1024 * 1024; +export const WOODPECKER_MAX_INFLIGHT_TRIGGERS = 128; +export const WOODPECKER_MAX_STAGES = 128; + +export interface CiCorrelation { + readonly sessionId: string; + readonly repositoryId: string; + readonly ref: string; + readonly commit: string; +} +export interface CiProvider { + readonly name: "woodpecker"; + query(correlation: CiCorrelation, signal?: AbortSignal): Promise; + run(correlation: CiCorrelation, signal?: AbortSignal): Promise; +} +export interface WoodpeckerRepositoryConfig { readonly slug: string; } +export interface WoodpeckerProviderOptions { + readonly baseUrl: string; + readonly token: string; + readonly repositories: Readonly>; + readonly fetch?: typeof globalThis.fetch; +} +interface WoodpeckerPipeline { + readonly number?: unknown; + readonly status?: unknown; + readonly ref?: unknown; + readonly branch?: unknown; + readonly commit?: unknown; + readonly created?: unknown; + readonly started?: unknown; + readonly finished?: unknown; + readonly variables?: unknown; + readonly stages?: unknown; +} + +function httpsBase(value: string): string { + const url = new URL(value); + if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash) + throw new Error("Woodpecker base URL must use HTTPS without credentials"); + return url.href.replace(/\/$/u, ""); +} +function bounded(value: unknown, name: string, max: number): string { + if (typeof value !== "string" || value.length === 0 || new TextEncoder().encode(value).byteLength > max || /[\u0000-\u001f\u007f]/u.test(value)) + throw new Error(`${name} is invalid`); + return value; +} +function slug(value: string): string { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value) || value.includes("..")) throw new Error("Woodpecker repository slug is invalid"); + return value; +} +function pipelineStatus(value: unknown): NonNullable { + switch (value) { + case "pending": case "queued": case "blocked": return value === "blocked" ? "unknown" : "queued"; + case "running": return "running"; + case "success": return "success"; + case "failure": case "error": return "failure"; + case "killed": case "canceled": case "cancelled": return "killed"; + default: return "unknown"; + } +} +function positiveInteger(value: unknown): number | undefined { + return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : undefined; +} +function timestamp(value: unknown): string | undefined { + const seconds = typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : undefined; + return seconds === undefined ? undefined : new Date(seconds * 1_000).toISOString(); +} +function object(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} +function exactPipeline(pipeline: WoodpeckerPipeline, correlation: CiCorrelation): boolean { + const variables = object(pipeline.variables); + return pipeline.ref === correlation.ref && pipeline.commit === correlation.commit && variables.T4_SESSION_ID === correlation.sessionId; +} + +export function mapWoodpeckerPipeline( + pipeline: WoodpeckerPipeline, + correlation: CiCorrelation, + link?: string, +): SessionCiState { + const number = positiveInteger(pipeline.number); + if (Array.isArray(pipeline.stages) && pipeline.stages.length > WOODPECKER_MAX_STAGES) throw new Error("Woodpecker stage response limit exceeded"); + const stages = Array.isArray(pipeline.stages) ? pipeline.stages.map(object) : []; + const current = stages.find(stage => stage.status === "running")?.name; + const repositoryId = bounded(correlation.repositoryId, "CI repository id", 256); + const ref = bounded(correlation.ref, "CI ref", 512); + const commit = bounded(correlation.commit, "CI commit", 512); + return { + provider: "woodpecker", + correlation: "exact", + repositoryId, + ...(typeof pipeline.branch === "string" ? { branch: bounded(pipeline.branch, "Woodpecker branch", 512) } : {}), + ref, + commit, + ...(number === undefined ? {} : { pipelineNumber: number }), + status: pipelineStatus(pipeline.status), + ...(typeof current === "string" ? { currentStage: bounded(current, "Woodpecker stage", 256) } : {}), + ...(timestamp(pipeline.created) ? { createdAt: timestamp(pipeline.created)! } : {}), + ...(timestamp(pipeline.started) ? { startedAt: timestamp(pipeline.started)! } : {}), + ...(timestamp(pipeline.finished) ? { finishedAt: timestamp(pipeline.finished)! } : {}), + ...(link ? { link: httpsBase(link) } : {}), + }; +} + +export class WoodpeckerProvider implements CiProvider { + readonly name = "woodpecker" as const; + readonly #baseUrl: string; + readonly #token: string; + readonly #repositories: Readonly>; + readonly #fetch: typeof globalThis.fetch; + readonly #inflight = new Map>(); + + constructor(options: WoodpeckerProviderOptions) { + this.#baseUrl = httpsBase(options.baseUrl); + this.#token = bounded(options.token, "Woodpecker token", 16_384); + const repositories: Record = {}; + for (const [id, config] of Object.entries(options.repositories)) { + bounded(id, "Woodpecker repository id", 256); + repositories[id] = { slug: slug(config.slug) }; + } + this.#repositories = Object.freeze(repositories); + this.#fetch = options.fetch ?? globalThis.fetch; + } + + async query(correlation: CiCorrelation, signal?: AbortSignal): Promise { + const repository = this.#repository(correlation.repositoryId); + this.#validateCorrelation(correlation); + const query = new URLSearchParams({ ref: correlation.ref, commit: correlation.commit, limit: String(WOODPECKER_MAX_PIPELINES) }); + const pipelines = await this.#json(`/api/repos/${encodeURIComponent(repository.slug)}/pipelines?${query}`, { signal }); + if (!Array.isArray(pipelines)) throw new Error("Woodpecker pipeline response is invalid"); + if (pipelines.length > WOODPECKER_MAX_PIPELINES) throw new Error("Woodpecker pipeline response limit exceeded"); + const exact = (pipelines as WoodpeckerPipeline[]).find(value => exactPipeline(value, correlation)); + if (!exact) return { + provider: "woodpecker", correlation: "unknown", repositoryId: correlation.repositoryId, + ref: correlation.ref, commit: correlation.commit, + }; + const number = positiveInteger(exact.number); + return mapWoodpeckerPipeline(exact, correlation, number ? `${this.#baseUrl}/repos/${repository.slug}/pipeline/${number}` : undefined); + } + + async run(correlation: CiCorrelation, signal?: AbortSignal): Promise { + const key = `${correlation.repositoryId}\u0000${correlation.ref}\u0000${correlation.commit}\u0000${correlation.sessionId}`; + const pending = this.#inflight.get(key); + if (pending) return await pending; + if (this.#inflight.size >= WOODPECKER_MAX_INFLIGHT_TRIGGERS) throw new Error("Woodpecker trigger concurrency limit exceeded"); + const operation = this.#runExact(correlation, signal); + this.#inflight.set(key, operation); + try { return await operation; } + finally { if (this.#inflight.get(key) === operation) this.#inflight.delete(key); } + } + + async #runExact(correlation: CiCorrelation, signal?: AbortSignal): Promise { + const existing = await this.query(correlation, signal); + if (existing.correlation === "exact" && existing.pipelineNumber !== undefined) + return { triggered: false, pipelineNumber: existing.pipelineNumber, ...(existing.status ? { status: existing.status } : {}) }; + const repository = this.#repository(correlation.repositoryId); + const created = object(await this.#json(`/api/repos/${encodeURIComponent(repository.slug)}/pipelines`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ref: correlation.ref, commit: correlation.commit, event: "manual", variables: { T4_SESSION_ID: correlation.sessionId } }), + signal, + })); + const mapped = mapWoodpeckerPipeline(created, correlation); + return { + triggered: true, + ...(mapped.pipelineNumber === undefined ? {} : { pipelineNumber: mapped.pipelineNumber }), + ...(mapped.status === undefined ? {} : { status: mapped.status }), + }; + } + + #repository(id: string): WoodpeckerRepositoryConfig { + const repository = this.#repositories[id]; + if (!repository) throw new Error("Woodpecker repository is not allowlisted"); + return repository; + } + #validateCorrelation(value: CiCorrelation): void { + bounded(value.sessionId, "CI session id", 256); + bounded(value.repositoryId, "CI repository id", 256); + bounded(value.ref, "CI ref", 512); + bounded(value.commit, "CI commit", 512); + } + async #json(path: string, init: RequestInit): Promise { + const headers = { Authorization: `Bearer ${this.#token}`, ...Object.fromEntries(new Headers(init.headers).entries()) }; + const response = await this.#fetch(`${this.#baseUrl}${path}`, { ...init, headers }); + const declaredLength = Number(response.headers.get("content-length")); + if (Number.isFinite(declaredLength) && declaredLength > WOODPECKER_MAX_RESPONSE_BYTES) throw new Error("Woodpecker response exceeds byte limit"); + if (!response.body) throw new Error("Woodpecker response body is unavailable"); + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let byteLength = 0; + for (;;) { + const chunk = await reader.read(); + if (chunk.done) break; + byteLength += chunk.value.byteLength; + if (byteLength > WOODPECKER_MAX_RESPONSE_BYTES) { await reader.cancel(); throw new Error("Woodpecker response exceeds byte limit"); } + chunks.push(chunk.value); + } + const bytes = new Uint8Array(byteLength); + let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } + let value: unknown; + try { value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); } + catch { throw new Error("Woodpecker response is not JSON"); } + if (!response.ok) throw new Error(`Woodpecker request failed with ${response.status}`); + return value; + } +} diff --git a/packages/cluster-server/test/config.test.ts b/packages/cluster-server/test/config.test.ts new file mode 100644 index 00000000..fa2ec157 --- /dev/null +++ b/packages/cluster-server/test/config.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, test } from "bun:test"; +import { clusterServerConfigFromEnv } from "../src/config.ts"; + +const BASE_ENV = { + POD_NAMESPACE: "cluster-system", + POD_NAME: "cluster-server-0", + POD_UID: "12345678-abcd", + KUBERNETES_SERVICE_HOST: "10.96.0.1", + KUBERNETES_SERVICE_PORT_HTTPS: "443", + T4_CLUSTER_HOST_NAME: "default", + T4_CLUSTER_INTERNAL_TOKEN: "x".repeat(32), +} as const; + +describe("trusted cluster gateway proxy sources", () => { + test("accepts bounded canonical IPv4 and IPv6 networks", () => { + const config = clusterServerConfigFromEnv({ + ...BASE_ENV, + T4_CLUSTER_TRUSTED_PROXY_ADDRESSES: "10.42.1.7,fd7a:115c:a1e0::1", + T4_CLUSTER_TRUSTED_PROXY_CIDRS: "10.42.0.0/16,fd7a:115c:a1e0::/48", + }); + expect(config.trustedProxyAddresses).toEqual(["10.42.1.7", "fd7a:115c:a1e0::1"]); + expect(config.trustedProxyCidrs).toEqual(["10.42.0.0/16", "fd7a:115c:a1e0::/48"]); + }); + + test("rejects CIDRs with host bits or non-canonical notation", () => { + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: "10.42.1.7/16" })).toThrow(); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: "fd7a:115c:a1e0:0::/48" })).toThrow(); + }); + + test("bounds the trusted CIDR list", () => { + const cidrs = Array.from({ length: 65 }, (_, index) => `10.${index}.0.0/16`).join(","); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: cidrs })).toThrow(); + }); +}); diff --git a/packages/cluster-server/tsconfig.json b/packages/cluster-server/tsconfig.json new file mode 100644 index 00000000..38e67895 --- /dev/null +++ b/packages/cluster-server/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ESNext", "DOM", "DOM.Iterable"], + "types": ["bun", "node"] + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/packages/host-service/src/remote/listener.ts b/packages/host-service/src/remote/listener.ts index fe8e805b..138f1fee 100644 --- a/packages/host-service/src/remote/listener.ts +++ b/packages/host-service/src/remote/listener.ts @@ -47,6 +47,15 @@ export function createListenerPlan(config: RemoteListenerConfig): ListenerPlan { throw new Error("listener port is invalid"); return { mode: "direct", address: config.address, port: config.port, path: "/v1/ws", trustedServeProxy: false }; } +export function createInternalListenerPlan(config: RemoteListenerConfig): ListenerPlan { + if (config.address !== "0.0.0.0" && config.address !== "::") + throw new Error("internal listener must bind an unspecified pod address"); + if (!config.internalPeerNodeId || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u.test(config.internalPeerNodeId)) + throw new Error("internal listener peer id is invalid"); + if (!Number.isInteger(config.port) || config.port < 1 || config.port > 65535) + throw new Error("listener port is invalid"); + return { mode: "direct", address: config.address, port: config.port, path: "/v1/ws", trustedServeProxy: false }; +} export function originAllowed(origin: string | null, allowlist: readonly string[] = []): boolean { return origin === null || allowlist.includes(origin); } @@ -135,9 +144,14 @@ export class BunRemoteListener { const address = normalizeIpAddress(requested); let peer: ListenerPeerContext | undefined; if (this.plan.mode === "direct") { - if (!isTailnetAddress(address) || !this.resolver) - return new Response("Unauthorized", { status: 401 }); - peer = { address, source: "direct", identity: await this.resolver.resolve(address) }; + if (this.config.internalPeerNodeId) { + const normalized = normalizeIpAddress(address); + peer = { address: normalized, source: "direct", identity: { nodeId: this.config.internalPeerNodeId, addresses: [normalized], source: "direct" } }; + } + else { + if (!isTailnetAddress(address) || !this.resolver) return new Response("Unauthorized", { status: 401 }); + peer = { address, source: "direct", identity: await this.resolver.resolve(address) }; + } } else { peer = resolveServePeer(address, request.headers, this.plan.trustedServeProxy); if (!peer) return new Response("Forbidden", { status: 403 }); diff --git a/packages/host-service/src/remote/types.ts b/packages/host-service/src/remote/types.ts index 11fc4b7d..09069c8e 100644 --- a/packages/host-service/src/remote/types.ts +++ b/packages/host-service/src/remote/types.ts @@ -38,6 +38,8 @@ export interface RemoteListenerConfig { port: number; trustedServeProxy?: boolean; serveProxy?: boolean; + /** Fixed peer identity for a pod-network listener whose omp-app hello is authenticated by a dedicated policy. */ + internalPeerNodeId?: string; originAllowlist?: readonly string[]; maxConnections?: number; maxFrameBytes?: number; diff --git a/packages/host-service/src/server.ts b/packages/host-service/src/server.ts index 3d2171fe..55449819 100644 --- a/packages/host-service/src/server.ts +++ b/packages/host-service/src/server.ts @@ -91,7 +91,7 @@ import { unlinkIfExists, } from "./ownership.ts"; import { SessionProjection } from "./projection.ts"; -import { BunRemoteListener, createListenerPlan, createServeProxyPlan } from "./remote/listener.ts"; +import { BunRemoteListener, createInternalListenerPlan, createListenerPlan, createServeProxyPlan } from "./remote/listener.ts"; import type { RemoteConnection, RemoteListenerConfig } from "./remote/types.ts"; import { BunRpcChildFactory, RpcChildSupervisor } from "./rpc-child.ts"; import type { @@ -1070,9 +1070,11 @@ export class LocalAppserver implements AppserverHandle { const listener = this.#remoteListener ?? new BunRemoteListener( - this.#remoteEndpoint.serveProxy === true - ? createServeProxyPlan(this.#remoteEndpoint) - : createListenerPlan(this.#remoteEndpoint), + this.#remoteEndpoint.internalPeerNodeId + ? createInternalListenerPlan(this.#remoteEndpoint) + : this.#remoteEndpoint.serveProxy === true + ? createServeProxyPlan(this.#remoteEndpoint) + : createListenerPlan(this.#remoteEndpoint), { connected: connection => this.#remoteConnected(connection), message: (connection, message) => this.#remoteMessage(connection, message), diff --git a/packages/host-wire/src/additive.ts b/packages/host-wire/src/additive.ts index 479539d5..adbeaa84 100644 --- a/packages/host-wire/src/additive.ts +++ b/packages/host-wire/src/additive.ts @@ -1,3 +1,4 @@ +import { decodeWorkspaceState, type WorkspaceStateFrame } from "./cluster.js"; import { type Cursor, decodeCursor } from "./cursor.js"; import { type DurableEntry, decodeEntry } from "./entry.js"; import { fail } from "./errors.js"; @@ -74,6 +75,7 @@ export const ADDITIVE_FEATURES = [ "catalog.metadata", "settings.metadata", "preview.control", + "cluster.operator", ] as const; export type AdditiveFeature = (typeof ADDITIVE_FEATURES)[number]; export type WireFeature = AdditiveFeature | "resume"; @@ -1039,10 +1041,12 @@ export type AdditiveServerFrame = | AuditEventFrame | CatalogFrame | SettingsFrame + | WorkspaceStateFrame | PreviewFrame; export function decodeAdditiveServerFrame(input: unknown): AdditiveServerFrame { const type = inputObject(input).type; if (typeof type !== "string") fail("INVALID_FRAME", "frame type must be string", "type"); + if (type === "workspace.state") return decodeWorkspaceState(input); if (["host.watch", "session.watch", "session.state", "session.delta"].includes(type)) return decodeWatch(input); if (type === "lease" || type === "prompt.lease") return decodeLease(input); if (type.startsWith("agent.")) return decodeAgentAdditive(input); diff --git a/packages/host-wire/src/capabilities.ts b/packages/host-wire/src/capabilities.ts index 7403dcb5..3ffc8a0c 100644 --- a/packages/host-wire/src/capabilities.ts +++ b/packages/host-wire/src/capabilities.ts @@ -23,6 +23,7 @@ export const DEVICE_CAPABILITIES = [ "preview.read", "preview.control", "preview.input", + "ci.trigger", ] as const; export type DeviceCapability = (typeof DEVICE_CAPABILITIES)[number]; export const PROTOCOL_FEATURES = [ @@ -52,6 +53,7 @@ export const PROTOCOL_FEATURES = [ "preview.control", "runtime.adapters", "workspace.lifecycle", + "cluster.operator", ] as const; export type ProtocolFeature = (typeof PROTOCOL_FEATURES)[number]; export const REMOTE_DEFAULT_CAPABILITIES = [ diff --git a/packages/host-wire/src/cluster.ts b/packages/host-wire/src/cluster.ts new file mode 100644 index 00000000..75de16ef --- /dev/null +++ b/packages/host-wire/src/cluster.ts @@ -0,0 +1,361 @@ +import { type Cursor, decodeCursor } from "./cursor.js"; +import { fail } from "./errors.js"; +import { boundedMap, controlFree, inputObject, safeSeq } from "./guards.js"; +import { type HostId, hostId, type Revision, revision } from "./ids.js"; +import { PROTOCOL_VERSION } from "./limits.js"; + +export const CLUSTER_OPERATOR_FEATURE = "cluster.operator" as const; +export const CI_TRIGGER_CAPABILITY = "ci.trigger" as const; +export const CLUSTER_MAX_WORKSPACES = 256; +export const CLUSTER_MAX_CONDITION_MESSAGE_BYTES = 2_048; +export const CLUSTER_MAX_REFERENCE_BYTES = 512; +export const CLUSTER_MAX_REPOSITORY_ID_BYTES = 256; + +export const CLUSTER_CONDITION_STATUSES = ["True", "False", "Unknown"] as const; +export type ClusterConditionStatus = (typeof CLUSTER_CONDITION_STATUSES)[number]; +export interface ClusterCondition { + readonly type: string; + readonly status: ClusterConditionStatus; + readonly reason: string; + readonly message: string; + readonly observedGeneration: number; +} + +export const WORKSPACE_PHASES = ["Pending", "Ready", "Failed", "Terminating", "Unknown"] as const; +export type WorkspacePhase = (typeof WORKSPACE_PHASES)[number]; +export const WORKSPACE_RETENTION_POLICIES = ["Retain", "Delete"] as const; +export type WorkspaceRetentionPolicy = (typeof WORKSPACE_RETENTION_POLICIES)[number]; +export interface WorkspaceInfrastructureProjection { + readonly id: string; + readonly displayName: string; + readonly phase: WorkspacePhase; + readonly retentionPolicy: WorkspaceRetentionPolicy; + readonly storageClass?: string; + readonly capacity?: string; + readonly accessMode: "ReadWriteMany"; + readonly revision: Revision; + readonly condition?: ClusterCondition; +} + +interface WorkspaceStateBase { + readonly v: typeof PROTOCOL_VERSION; + readonly type: "workspace.state"; + readonly hostId: HostId; + readonly workspaceId: string; + readonly cursor: Cursor; + readonly revision: Revision; +} +export interface WorkspaceStateUpsertFrame extends WorkspaceStateBase { + readonly upsert: WorkspaceInfrastructureProjection; + readonly remove?: never; +} +export interface WorkspaceStateRemoveFrame extends WorkspaceStateBase { + readonly upsert?: never; + readonly remove: string; +} +export type WorkspaceStateFrame = WorkspaceStateUpsertFrame | WorkspaceStateRemoveFrame; + +export const SESSION_INFRASTRUCTURE_PHASES = ["Pending", "Running", "Failed", "Terminating", "Unknown"] as const; +export type SessionInfrastructurePhase = (typeof SESSION_INFRASTRUCTURE_PHASES)[number]; +export const SESSION_GUI_STATES = ["Unavailable", "Starting", "Ready", "Failed"] as const; +export type SessionGuiState = (typeof SESSION_GUI_STATES)[number]; +export interface SessionClusterState { + readonly workspaceId: string; + readonly phase: SessionInfrastructurePhase; + readonly condition?: ClusterCondition; + readonly gui: { + readonly state: SessionGuiState; + readonly previewId?: string; + readonly reason?: string; + }; +} + +export const CI_PIPELINE_STATUSES = ["queued", "running", "success", "failure", "killed", "unknown"] as const; +export type CiPipelineStatus = (typeof CI_PIPELINE_STATUSES)[number]; +export interface SessionCiState { + readonly provider: "woodpecker"; + readonly correlation: "exact" | "unknown"; + readonly repositoryId: string; + readonly branch?: string; + readonly ref: string; + readonly commit: string; + readonly pipelineNumber?: number; + readonly status?: CiPipelineStatus; + readonly currentStage?: string; + readonly createdAt?: string; + readonly startedAt?: string; + readonly finishedAt?: string; + readonly link?: string; +} + +export interface LegacyWorkspaceResultItem { + readonly repositoryId: string; + readonly instanceId: string; + readonly ownership: "managed" | "imported-user" | "detected-external" | "repository-root"; + readonly branch: string; + readonly sourceCommit: string; + readonly expectedHead: string; + readonly lifecycle: "creating" | "active" | "archiving" | "archived" | "recovery-required"; + readonly createdAt: number; + readonly updatedAt: number; + readonly archivedAt?: number; +} +export type WorkspaceListItem = LegacyWorkspaceResultItem | WorkspaceInfrastructureProjection; +export interface WorkspaceListResult { + readonly workspaces: readonly WorkspaceListItem[]; + readonly cursor?: Cursor; +} + +export interface ClusterRepositorySelection { + readonly repositoryId: string; + readonly ref?: string; + readonly commit?: string; +} +export interface ClusterWorkspaceCreateArguments { + readonly displayName: string; + readonly retentionPolicy: WorkspaceRetentionPolicy; + readonly capacity: string; + readonly storageClass?: string; + readonly repository?: ClusterRepositorySelection; +} +export interface ClusterSessionCiSelection { + readonly provider: "woodpecker"; + readonly repositoryId: string; + readonly ref: string; + readonly commit: string; +} +export interface ClusterSessionCreateArguments { + readonly workspaceId: string; + readonly title?: string; + readonly runtimeProfile: string; + readonly guiEnabled: boolean; + readonly ci?: ClusterSessionCiSelection; +} +export interface CiRunArguments { + readonly provider: "woodpecker"; + readonly action: "run"; + readonly repositoryId: string; + readonly ref: string; + readonly commit: string; +} +export interface CiRunResult { + readonly triggered: boolean; + readonly pipelineNumber?: number; + readonly status?: CiPipelineStatus; +} + +function exact(value: unknown, path: string, keys: readonly string[]): Record { + const result = boundedMap(value, path); + const allowed = new Set(keys); + for (const key of Object.keys(result)) + if (!allowed.has(key)) fail("INVALID_FRAME", "unknown cluster field", `${path}.${key}`); + return result; +} +function oneOf(value: unknown, path: string, values: T): T[number] { + if (typeof value !== "string" || !(values as readonly string[]).includes(value)) fail("INVALID_FRAME", "unknown categorical state", path); + return value as T[number]; +} +function identifier(value: unknown, path: string, max = 256): string { + const result = controlFree(value, path, max); + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/u.test(result)) fail("INVALID_FRAME", "invalid cluster identifier", path); + return result; +} +function repositoryId(value: unknown, path: string): string { + const result = controlFree(value, path, CLUSTER_MAX_REPOSITORY_ID_BYTES); + if (!/^[A-Za-z0-9][A-Za-z0-9._/-]{0,255}$/u.test(result) || result.includes("..") || result.includes("://")) + fail("INVALID_FRAME", "invalid repository identifier", path); + return result; +} +function reference(value: unknown, path: string): string { + return controlFree(value, path, CLUSTER_MAX_REFERENCE_BYTES); +} +function quantity(value: unknown, path: string): string { + const result = controlFree(value, path, 64); + if (!/^(?:0|[1-9][0-9]*)(?:[EPTGMK]i?|m)?$/u.test(result)) fail("INVALID_FRAME", "invalid Kubernetes quantity", path); + return result; +} +function canonicalTimestamp(value: unknown, path: string): string { + const result = controlFree(value, path, 128); + const milliseconds = Date.parse(result); + if (!Number.isFinite(milliseconds) || new Date(milliseconds).toISOString() !== result) + fail("INVALID_FRAME", "timestamp must be canonical ISO", path); + return result; +} +function httpsUrl(value: unknown, path: string): string { + const text = controlFree(value, path, 2_048); + let url: URL; + try { + url = new URL(text); + } catch { + fail("INVALID_FRAME", "invalid HTTPS URL", path); + } + if (url!.protocol !== "https:" || url!.username || url!.password) + fail("INVALID_FRAME", "URL must use HTTPS without credentials", path); + return url!.href; +} + +export function decodeClusterCondition(value: unknown, path = "condition"): ClusterCondition { + const input = exact(value, path, ["type", "status", "reason", "message", "observedGeneration"]); + return { + type: identifier(input.type, `${path}.type`, 128), + status: oneOf(input.status, `${path}.status`, CLUSTER_CONDITION_STATUSES), + reason: identifier(input.reason, `${path}.reason`, 128), + message: controlFree(input.message, `${path}.message`, CLUSTER_MAX_CONDITION_MESSAGE_BYTES), + observedGeneration: safeSeq(input.observedGeneration, `${path}.observedGeneration`), + }; +} + +export function decodeWorkspaceInfrastructureProjection( + value: unknown, + path = "workspace", +): WorkspaceInfrastructureProjection { + const input = exact(value, path, [ + "id", "displayName", "phase", "retentionPolicy", "storageClass", "capacity", "accessMode", "revision", "condition", + ]); + const currentRevision = revision(input.revision, `${path}.revision`); + if (input.accessMode !== "ReadWriteMany") fail("INVALID_FRAME", "workspace access mode must be ReadWriteMany", `${path}.accessMode`); + return { + id: identifier(input.id, `${path}.id`), + displayName: controlFree(input.displayName, `${path}.displayName`, 256), + phase: oneOf(input.phase, `${path}.phase`, WORKSPACE_PHASES), + retentionPolicy: oneOf(input.retentionPolicy, `${path}.retentionPolicy`, WORKSPACE_RETENTION_POLICIES), + ...(input.storageClass === undefined ? {} : { storageClass: identifier(input.storageClass, `${path}.storageClass`, 128) }), + ...(input.capacity === undefined ? {} : { capacity: quantity(input.capacity, `${path}.capacity`) }), + accessMode: "ReadWriteMany", + revision: currentRevision, + ...(input.condition === undefined ? {} : { condition: decodeClusterCondition(input.condition, `${path}.condition`) }), + }; +} + +export function decodeWorkspaceState(value: unknown): WorkspaceStateFrame { + const input = inputObject(value); + if (input.v !== PROTOCOL_VERSION) fail("MISSING_VERSION", `expected ${PROTOCOL_VERSION}`, "v"); + if (input.type !== "workspace.state") fail("INVALID_FRAME", "expected workspace.state frame", "type"); + const keys = ["v", "type", "hostId", "workspaceId", "cursor", "revision", "upsert", "remove"]; + for (const key of Object.keys(input)) if (!keys.includes(key)) fail("INVALID_FRAME", "unknown workspace state field", key); + const host = hostId(input.hostId); + const workspace = identifier(input.workspaceId, "workspaceId"); + const cursor = decodeCursor(input.cursor); + const currentRevision = revision(input.revision); + if ((input.upsert === undefined) === (input.remove === undefined)) + fail("INVALID_FRAME", "workspace state requires exactly one upsert or remove", "frame"); + if (input.upsert !== undefined) { + const upsert = decodeWorkspaceInfrastructureProjection(input.upsert, "upsert"); + if (upsert.id !== workspace || upsert.revision !== currentRevision) + fail("INVALID_FRAME", "workspace state address mismatch", "upsert"); + return { v: PROTOCOL_VERSION, type: "workspace.state", hostId: host, workspaceId: workspace, cursor, revision: currentRevision, upsert }; + } + if (input.remove !== workspace) fail("INVALID_FRAME", "workspace removal must match workspaceId", "remove"); + return { v: PROTOCOL_VERSION, type: "workspace.state", hostId: host, workspaceId: workspace, cursor, revision: currentRevision, remove: workspace }; +} + +export function decodeSessionClusterState(value: unknown, path = "cluster"): SessionClusterState { + const input = exact(value, path, ["workspaceId", "phase", "condition", "gui"]); + const gui = exact(input.gui, `${path}.gui`, ["state", "previewId", "reason"]); + return { + workspaceId: identifier(input.workspaceId, `${path}.workspaceId`), + phase: oneOf(input.phase, `${path}.phase`, SESSION_INFRASTRUCTURE_PHASES), + ...(input.condition === undefined ? {} : { condition: decodeClusterCondition(input.condition, `${path}.condition`) }), + gui: { + state: oneOf(gui.state, `${path}.gui.state`, SESSION_GUI_STATES), + ...(gui.previewId === undefined ? {} : { previewId: identifier(gui.previewId, `${path}.gui.previewId`) }), + ...(gui.reason === undefined ? {} : { reason: controlFree(gui.reason, `${path}.gui.reason`, 512) }), + }, + }; +} + +export function decodeSessionCiState(value: unknown, path = "ci"): SessionCiState { + const input = exact(value, path, [ + "provider", "correlation", "repositoryId", "branch", "ref", "commit", "pipelineNumber", "status", "currentStage", + "createdAt", "startedAt", "finishedAt", "link", + ]); + if (input.provider !== "woodpecker") fail("INVALID_FRAME", "unknown CI provider", `${path}.provider`); + if (input.correlation !== "exact" && input.correlation !== "unknown") + fail("INVALID_FRAME", "unknown CI correlation", `${path}.correlation`); + const output: SessionCiState = { + provider: "woodpecker", + correlation: input.correlation, + repositoryId: repositoryId(input.repositoryId, `${path}.repositoryId`), + ...(input.branch === undefined ? {} : { branch: reference(input.branch, `${path}.branch`) }), + ref: reference(input.ref, `${path}.ref`), + commit: reference(input.commit, `${path}.commit`), + ...(input.pipelineNumber === undefined ? {} : { pipelineNumber: safeSeq(input.pipelineNumber, `${path}.pipelineNumber`) }), + ...(input.status === undefined ? {} : { status: oneOf(input.status, `${path}.status`, CI_PIPELINE_STATUSES) }), + ...(input.currentStage === undefined ? {} : { currentStage: controlFree(input.currentStage, `${path}.currentStage`, 256) }), + ...(input.createdAt === undefined ? {} : { createdAt: canonicalTimestamp(input.createdAt, `${path}.createdAt`) }), + ...(input.startedAt === undefined ? {} : { startedAt: canonicalTimestamp(input.startedAt, `${path}.startedAt`) }), + ...(input.finishedAt === undefined ? {} : { finishedAt: canonicalTimestamp(input.finishedAt, `${path}.finishedAt`) }), + ...(input.link === undefined ? {} : { link: httpsUrl(input.link, `${path}.link`) }), + }; + if ( + output.correlation === "unknown" && + (output.pipelineNumber !== undefined || output.status !== undefined || output.currentStage !== undefined || + output.createdAt !== undefined || output.startedAt !== undefined || output.finishedAt !== undefined || output.link !== undefined) + ) + fail("INVALID_FRAME", "unknown CI correlation cannot carry pipeline state", path); + return output; +} + +function decodeRepository(value: unknown, path: string, requireRefCommit: boolean): ClusterRepositorySelection { + const input = exact(value, path, ["repositoryId", "ref", "commit"]); + if (requireRefCommit && (input.ref === undefined || input.commit === undefined)) + fail("INVALID_FRAME", "CI repository requires ref and commit", path); + return { + repositoryId: repositoryId(input.repositoryId, `${path}.repositoryId`), + ...(input.ref === undefined ? {} : { ref: reference(input.ref, `${path}.ref`) }), + ...(input.commit === undefined ? {} : { commit: reference(input.commit, `${path}.commit`) }), + }; +} + +export function decodeClusterWorkspaceCreateArguments(value: unknown): ClusterWorkspaceCreateArguments { + const input = exact(value, "args", ["displayName", "retentionPolicy", "capacity", "storageClass", "repository"]); + return { + displayName: controlFree(input.displayName, "args.displayName", 256), + retentionPolicy: oneOf(input.retentionPolicy, "args.retentionPolicy", WORKSPACE_RETENTION_POLICIES), + capacity: quantity(input.capacity, "args.capacity"), + ...(input.storageClass === undefined ? {} : { storageClass: identifier(input.storageClass, "args.storageClass", 128) }), + ...(input.repository === undefined ? {} : { repository: decodeRepository(input.repository, "args.repository", false) }), + }; +} + +export function decodeClusterSessionCreateArguments(value: unknown): ClusterSessionCreateArguments { + const input = exact(value, "args", ["workspaceId", "title", "runtimeProfile", "guiEnabled", "ci"]); + let ci: ClusterSessionCiSelection | undefined; + if (input.ci !== undefined) { + const raw = exact(input.ci, "args.ci", ["provider", "repositoryId", "ref", "commit"]); + if (raw.provider !== "woodpecker") fail("INVALID_FRAME", "unknown CI provider", "args.ci.provider"); + const repository = decodeRepository({ repositoryId: raw.repositoryId, ref: raw.ref, commit: raw.commit }, "args.ci", true); + ci = { provider: "woodpecker", repositoryId: repository.repositoryId, ref: repository.ref!, commit: repository.commit! }; + } + if (typeof input.guiEnabled !== "boolean") fail("INVALID_FRAME", "guiEnabled must be boolean", "args.guiEnabled"); + return { + workspaceId: identifier(input.workspaceId, "args.workspaceId"), + ...(input.title === undefined ? {} : { title: controlFree(input.title, "args.title", 512) }), + runtimeProfile: identifier(input.runtimeProfile, "args.runtimeProfile", 64), + guiEnabled: input.guiEnabled, + ...(ci === undefined ? {} : { ci }), + }; +} + +export function decodeCiRunArguments(value: unknown): CiRunArguments { + const input = exact(value, "args", ["provider", "action", "repositoryId", "ref", "commit"]); + if (input.provider !== "woodpecker") fail("INVALID_FRAME", "unknown CI provider", "args.provider"); + if (input.action !== "run") fail("INVALID_FRAME", "unknown CI action", "args.action"); + return { + provider: "woodpecker", + action: "run", + repositoryId: repositoryId(input.repositoryId, "args.repositoryId"), + ref: reference(input.ref, "args.ref"), + commit: reference(input.commit, "args.commit"), + }; +} + +export function decodeCiRunResult(value: unknown): CiRunResult { + const input = exact(value, "result", ["triggered", "pipelineNumber", "status"]); + if (typeof input.triggered !== "boolean") fail("INVALID_FRAME", "triggered must be boolean", "result.triggered"); + return { + triggered: input.triggered, + ...(input.pipelineNumber === undefined ? {} : { pipelineNumber: safeSeq(input.pipelineNumber, "result.pipelineNumber") }), + ...(input.status === undefined ? {} : { status: oneOf(input.status, "result.status", CI_PIPELINE_STATUSES) }), + }; +} diff --git a/packages/host-wire/src/command.ts b/packages/host-wire/src/command.ts index 668e4ffa..b0918626 100644 --- a/packages/host-wire/src/command.ts +++ b/packages/host-wire/src/command.ts @@ -7,6 +7,13 @@ import { type PreviewAction, type PreviewSnapshot, } from "./additive.js"; +import { + decodeCiRunArguments, + decodeCiRunResult, + decodeClusterSessionCreateArguments, + decodeClusterWorkspaceCreateArguments, + decodeWorkspaceInfrastructureProjection, +} from "./cluster.js"; import { decodeBrokerStatusResult } from "./broker.js"; import type { DeviceCapability } from "./capabilities.js"; import { decodeCursor } from "./cursor.js"; @@ -708,6 +715,13 @@ export const COMMAND_DESCRIPTORS: Readonly> = revisionOwner: "session", confirmation: "none", }, + "ci.run": { + capability: "ci.trigger", + scope: "session", + revision: "required", + revisionOwner: "session", + confirmation: "none", + }, }; export const DESKTOP_CATALOG_COMMANDS: readonly string[] = Object.freeze( Object.entries(COMMAND_DESCRIPTORS) @@ -1008,7 +1022,10 @@ function decodeRuntimeResultItem(value: unknown, path: string): Record { - const item = strictMap(value, path, [ + const candidate = boundedMap(value, path); + if (Object.hasOwn(candidate, "id")) + return decodeWorkspaceInfrastructureProjection(candidate, path) as unknown as Record; + const item = strictMap(candidate, path, [ "repositoryId", "instanceId", "ownership", @@ -1575,7 +1592,10 @@ export const COMMAND_ARGUMENT_DECODERS: Readonly { - const x = strictArgs(value, ["projectId", "name", "branch", "sourceCommit"]); + const candidate = boundedMap(value, "args"); + if (Object.hasOwn(candidate, "displayName")) + return decodeClusterWorkspaceCreateArguments(candidate) as unknown as CommandArguments; + const x = strictArgs(candidate, ["projectId", "name", "branch", "sourceCommit"]); projectId(x.projectId, "args.projectId"); controlFree(x.name, "args.name", 128); controlFree(x.branch, "args.branch", 256); @@ -1605,7 +1625,10 @@ export const COMMAND_ARGUMENT_DECODERS: Readonly { - const x = strictArgs(value, ["projectId", "title", "runtimeId", "workspaceInstanceId"]); + const candidate = boundedMap(value, "args"); + if (Object.hasOwn(candidate, "workspaceId")) + return decodeClusterSessionCreateArguments(candidate) as unknown as CommandArguments; + const x = strictArgs(candidate, ["projectId", "title", "runtimeId", "workspaceInstanceId"]); projectId(x.projectId, "args.projectId"); if (x.title !== undefined) boundedText(x.title, "args.title", 512); const runtimeId = x.runtimeId === undefined ? undefined : controlFree(x.runtimeId, "args.runtimeId", 64); @@ -1829,6 +1852,7 @@ export const COMMAND_ARGUMENT_DECODERS: Readonly decodeCiRunArguments(value) as unknown as CommandArguments, }; export const COMMAND_RESULT_DECODERS: Readonly CommandResult>> = { "runtime.list": value => { @@ -1840,11 +1864,12 @@ export const COMMAND_RESULT_DECODERS: Readonly { - const x = strictResult(value, ["workspaces"]); + const x = strictResult(value, ["workspaces", "cursor"]); return { workspaces: boundedArray(x.workspaces, "result.workspaces", 256).map((workspace, index) => decodeWorkspaceResultItem(workspace, `result.workspaces[${index}]`), ), + ...(x.cursor === undefined ? {} : { cursor: decodeCursor(x.cursor, "result.cursor") }), }; }, "workspace.create": value => { @@ -1985,6 +2010,7 @@ export const COMMAND_RESULT_DECODERS: Readonly decodeCiRunResult(value) as unknown as CommandResult, }; export function decodeCommandArguments(command: string, value: unknown): CommandArguments { const decoder = COMMAND_ARGUMENT_DECODERS[command]; diff --git a/packages/host-wire/src/index.ts b/packages/host-wire/src/index.ts index 997fb506..370789fb 100644 --- a/packages/host-wire/src/index.ts +++ b/packages/host-wire/src/index.ts @@ -3,6 +3,7 @@ export * from "./agents.js"; export * from "./audit.js"; export * from "./broker.js"; export * from "./capabilities.js"; +export * from "./cluster.js"; export * from "./command.js"; export * from "./cursor.js"; export * from "./entry.js"; diff --git a/packages/host-wire/src/session-index.ts b/packages/host-wire/src/session-index.ts index 955013b2..da7fa369 100644 --- a/packages/host-wire/src/session-index.ts +++ b/packages/host-wire/src/session-index.ts @@ -1,3 +1,9 @@ +import { + decodeSessionCiState, + decodeSessionClusterState, + type SessionCiState, + type SessionClusterState, +} from "./cluster.js"; import { type Cursor, decodeCursor } from "./cursor.js"; import { fail } from "./errors.js"; import { @@ -117,6 +123,8 @@ export type SessionControlState = export interface SessionLiveState { sessionControl?: SessionControlState; providerTransport?: ProviderTransportState; + cluster?: SessionClusterState; + ci?: SessionCiState; [key: string]: unknown; } export interface SessionRef { @@ -363,6 +371,9 @@ export function decodeSessionRef(value: unknown, path: string): SessionRef { decodeSessionControl(liveState.sessionControl, `${path}.liveState.sessionControl`); if (liveState.providerTransport !== undefined) decodeProviderTransportState(liveState.providerTransport, `${path}.liveState.providerTransport`); + if (liveState.cluster !== undefined) + decodeSessionClusterState(liveState.cluster, `${path}.liveState.cluster`); + if (liveState.ci !== undefined) decodeSessionCiState(liveState.ci, `${path}.liveState.ci`); } if (session.model !== undefined) controlFree(session.model, `${path}.model`, 256); if (session.thinking !== undefined) controlFree(session.thinking, `${path}.thinking`, 256); diff --git a/packages/protocol/src/server-event.ts b/packages/protocol/src/server-event.ts index 914083d9..7f6d0941 100644 --- a/packages/protocol/src/server-event.ts +++ b/packages/protocol/src/server-event.ts @@ -26,6 +26,7 @@ const OMP_SERVER_EVENT_KIND_MEMBERS = { "session.watch": true, "session.state": true, "session.delta": true, + "workspace.state": true, lease: true, "prompt.lease": true, "agent.state": true, From 03da5ec15d0930cdff9ec4a5217c1cc056ae9a7f Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 16:14:13 +0000 Subject: [PATCH 13/98] fix(cluster): close server integration gaps --- .woodpecker.yml | 20 +++- packages/cluster-server/package.json | 6 +- .../src/kubernetes-projection.ts | 4 +- packages/cluster-server/src/main.ts | 103 +++++++++--------- packages/cluster-server/src/observability.ts | 6 - packages/cluster-server/src/server.ts | 23 +++- packages/cluster-server/test/config.test.ts | 2 +- packages/cluster-server/test/gateway.test.ts | 25 ++++- .../test/kubernetes-client.test.ts | 71 ++++++++---- .../test/kubernetes-projection.test.ts | 20 +++- .../cluster-server/test/observability.test.ts | 23 ++-- .../test/session-host-policy.test.ts | 2 +- .../cluster-server/test/woodpecker.test.ts | 2 +- pnpm-lock.yaml | 22 ++++ .../cluster-ci/cluster-ci-contract.test.mjs | 1 + 15 files changed, 227 insertions(+), 103 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 0e9bac05..4a2c52c5 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -182,6 +182,24 @@ steps: - corepack enable - pnpm --filter @t4-code/cluster-server test - pnpm --filter @t4-code/cluster-server typecheck + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 2Gi + + cluster-wire-tests: + depends_on: [cluster-server-tests] + image: mirror.gcr.io/library/node:24.13.1-bookworm + commands: + - corepack enable - pnpm --filter @t4-code/host-wire test - pnpm --filter @t4-code/host-service test backend_options: @@ -198,7 +216,7 @@ steps: ephemeral-storage: 2Gi cluster-operator-tests: - depends_on: [cluster-server-tests] + depends_on: [cluster-wire-tests] image: mirror.gcr.io/library/golang:1.25.1-bookworm directory: packages/cluster-operator commands: diff --git a/packages/cluster-server/package.json b/packages/cluster-server/package.json index 537956da..3326df93 100644 --- a/packages/cluster-server/package.json +++ b/packages/cluster-server/package.json @@ -8,9 +8,9 @@ "types": "./src/index.ts", "exports": { ".": { "types": "./src/index.ts", "import": "./src/index.ts" }, "./*": { "types": "./src/*.ts", "import": "./src/*.ts" } }, "scripts": { - "build": "bun run typecheck", - "check": "bun run typecheck", - "test": "bun test test", + "build": "tsgo -p tsconfig.json --noEmit", + "check": "tsgo -p tsconfig.json --noEmit", + "test": "vp test run", "typecheck": "tsgo -p tsconfig.json --noEmit" }, "dependencies": { diff --git a/packages/cluster-server/src/kubernetes-projection.ts b/packages/cluster-server/src/kubernetes-projection.ts index 751a3ff1..e7b2304a 100644 --- a/packages/cluster-server/src/kubernetes-projection.ts +++ b/packages/cluster-server/src/kubernetes-projection.ts @@ -279,9 +279,9 @@ export class ClusterInfrastructureProjection { const authoritative = decodeSessionRef(value, `authority.${clusterSessionId}`); const previous = this.#authoritativeSessions.get(clusterSessionId); if (previous && JSON.stringify(previous) === JSON.stringify(authoritative)) return; + const projected = this.#session(resource, authoritative); this.#authoritativeSessions.set(clusterSessionId, authoritative); this.#sessionSequence++; - const projected = this.#session(resource, authoritative); for (const waiter of this.#authorityWaiters.get(clusterSessionId) ?? []) { clearTimeout(waiter.timer); waiter.resolve(projected); @@ -416,7 +416,7 @@ export class ClusterInfrastructureProjection { : infrastructurePhase === "Failed" || infrastructurePhase === "Terminating" ? "Failed" : "Starting"; - const { cluster: _upstreamCluster, ci: _upstreamCi, ...ompLiveState } = authoritative.liveState; + const { cluster: _upstreamCluster, ci: _upstreamCi, ...ompLiveState } = authoritative.liveState ?? {}; return decodeSessionRef({ ...authoritative, hostId: this.hostId, diff --git a/packages/cluster-server/src/main.ts b/packages/cluster-server/src/main.ts index 42e96890..bf08288b 100755 --- a/packages/cluster-server/src/main.ts +++ b/packages/cluster-server/src/main.ts @@ -6,7 +6,7 @@ import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; import { KubernetesProjectionRunner } from "./kubernetes-runner.ts"; import { ClusterMetrics, ClusterServerHealth, JsonLogger } from "./observability.ts"; import { WebSocketPodHostConnector } from "./pod-host-router.ts"; -import { startClusterHttpServers } from "./server.ts"; +import { startClusterHttpServers, type ClusterHttpServers } from "./server.ts"; import { SessionAuthorityRunner } from "./session-authority-runner.ts"; import { WoodpeckerProvider } from "./woodpecker.ts"; @@ -33,55 +33,60 @@ export async function runClusterServer(env: Readonly logger.warn("session authority reconnecting", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }), - }); - authority.start(); - const ciProvider = config.woodpecker ? new WoodpeckerProvider(config.woodpecker) : undefined; - const gateway = new ClusterGateway({ - projection, - connector, - mutations: new KubernetesGatewayMutationBackend({ client: kubernetes, hostRef: config.hostName }), - internalToken: config.internalToken, - ...(ciProvider ? { ciProvider } : {}), - }); - const servers = startClusterHttpServers({ - gateway, - projection, - gatewayPort: config.gatewayPort, - adminPort: config.adminPort, - trustedProxyAddresses: config.trustedProxyAddresses, - trustedProxyCidrs: config.trustedProxyCidrs, - health, - metrics, - logger, - }); const stopped = Promise.withResolvers(); - let stopping = false; - const stop = (): void => { - if (stopping) return; - stopping = true; - void (async () => { - await servers.drain(); - await authority.stop(); - await runner.stop(); - await servers.stop(); - })().then(stopped.resolve, stopped.reject); - }; - process.once("SIGTERM", stop); - process.once("SIGINT", stop); - try { await stopped.promise; } - finally { - process.off("SIGTERM", stop); - process.off("SIGINT", stop); - if (!stopping) { - await servers.stop(); - await authority.stop(); - await runner.stop(); + const stop = (): void => stopped.resolve(); + let authority: SessionAuthorityRunner | undefined; + let servers: ClusterHttpServers | undefined; + let signalsInstalled = false; + try { + await runner.start(); + const connector = new WebSocketPodHostConnector({ internalToken: config.internalToken }); + authority = new SessionAuthorityRunner({ + projection, + connector, + onError: error => logger.warn("session authority reconnecting", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }), + }); + authority.start(); + const ciProvider = config.woodpecker ? new WoodpeckerProvider(config.woodpecker) : undefined; + const gateway = new ClusterGateway({ + projection, + connector, + mutations: new KubernetesGatewayMutationBackend({ client: kubernetes, hostRef: config.hostName }), + internalToken: config.internalToken, + ...(ciProvider ? { ciProvider } : {}), + }); + servers = startClusterHttpServers({ + gateway, + projection, + gatewayPort: config.gatewayPort, + adminPort: config.adminPort, + trustedProxyAddresses: config.trustedProxyAddresses, + trustedProxyCidrs: config.trustedProxyCidrs, + health, + metrics, + logger, + }); + process.once("SIGTERM", stop); + process.once("SIGINT", stop); + signalsInstalled = true; + await stopped.promise; + } finally { + if (signalsInstalled) { + process.off("SIGTERM", stop); + process.off("SIGINT", stop); + } + try { + await servers?.drain(); + } finally { + try { + await authority?.stop(); + } finally { + try { + await runner.stop(); + } finally { + await servers?.stop(); + } + } } } } diff --git a/packages/cluster-server/src/observability.ts b/packages/cluster-server/src/observability.ts index 42d2afe5..9a8b47be 100644 --- a/packages/cluster-server/src/observability.ts +++ b/packages/cluster-server/src/observability.ts @@ -96,7 +96,6 @@ export class ClusterServerHealth { export interface AdminHandlerOptions { readonly health: ClusterServerHealth; readonly metrics: ClusterMetrics; - readonly drain: () => Promise | void; } function json(value: unknown, status = 200): Response { return Response.json(value, { status, headers: { "cache-control": "no-store" } }); @@ -107,11 +106,6 @@ export function createAdminHandler(options: AdminHandlerOptions): (request: Requ if (path === "/healthz") return request.method === "GET" ? json({ healthy: options.health.healthy }) : new Response(null, { status: 405 }); if (path === "/readyz") return request.method === "GET" ? json({ ready: options.health.ready }, options.health.ready ? 200 : 503) : new Response(null, { status: 405 }); if (path === "/metrics") return request.method === "GET" ? new Response(options.metrics.render(), { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8", "cache-control": "no-store" } }) : new Response(null, { status: 405 }); - if (path === "/drainz") { - if (request.method !== "POST") return new Response(null, { status: 405 }); - await options.drain(); - return json({ draining: true }, 202); - } return new Response("not found", { status: 404 }); }; } diff --git a/packages/cluster-server/src/server.ts b/packages/cluster-server/src/server.ts index df91df4d..35565c69 100644 --- a/packages/cluster-server/src/server.ts +++ b/packages/cluster-server/src/server.ts @@ -37,6 +37,14 @@ function trustedProxyMatcher(addresses: readonly string[], cidrs: readonly strin }; } +export function isLoopbackAddress(address: string): boolean { + const normalized = normalizeIpAddress(address); + if (normalized === "::1") return true; + if (isIP(normalized) !== 4) return false; + const firstOctet = Number(normalized.split(".", 1)[0]); + return firstOctet === 127; +} + function gatewayPrincipal(request: Request, remoteAddress: string, trustedSource: (address: string) => boolean): string | undefined { if (!trustedSource(remoteAddress)) return undefined; if (request.headers.get("x-forwarded-proto") !== "https") return undefined; @@ -103,8 +111,19 @@ export function startClusterHttpServers(options: ClusterHttpServersOptions): Clu await stopping; options.health.markGatewayStopped(); }; - const adminHandler = createAdminHandler({ health: options.health, metrics: options.metrics, drain }); - const adminServer = Bun.serve({ hostname: "0.0.0.0", port: options.adminPort, fetch: adminHandler }); + const adminHandler = createAdminHandler({ health: options.health, metrics: options.metrics }); + const adminServer = Bun.serve({ + hostname: "0.0.0.0", + port: options.adminPort, + async fetch(request, server) { + if (new URL(request.url).pathname !== "/drainz") return adminHandler(request); + if (request.method !== "POST") return new Response(null, { status: 405 }); + const source = server.requestIP(request)?.address; + if (!source || !isLoopbackAddress(source)) return new Response("forbidden", { status: 403 }); + await drain(); + return Response.json({ draining: true }, { status: 202, headers: { "cache-control": "no-store" } }); + }, + }); options.health.markGatewayListening(); options.logger.info("cluster server listening", { result: "success" }); return { diff --git a/packages/cluster-server/test/config.test.ts b/packages/cluster-server/test/config.test.ts index fa2ec157..f164dbae 100644 --- a/packages/cluster-server/test/config.test.ts +++ b/packages/cluster-server/test/config.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, test } from "vite-plus/test" import { clusterServerConfigFromEnv } from "../src/config.ts"; const BASE_ENV = { diff --git a/packages/cluster-server/test/gateway.test.ts b/packages/cluster-server/test/gateway.test.ts index d6b1e410..ebb95e74 100644 --- a/packages/cluster-server/test/gateway.test.ts +++ b/packages/cluster-server/test/gateway.test.ts @@ -1,9 +1,22 @@ -import { describe, expect, test } from "bun:test"; -import type { ClientFrame, ServerFrame } from "@t4-code/host-wire"; +import { describe, expect, test } from "vite-plus/test" +import { hostId, projectId, revision, sessionId, type ClientFrame, type ServerFrame, type SessionRef } from "@t4-code/host-wire"; import { ClusterGateway, type GatewayClient, type GatewayMutationBackend } from "../src/gateway.ts"; import { ClusterInfrastructureProjection } from "../src/kubernetes-projection.ts"; import type { PodHostConnection, PodHostConnector, PodHostRoute } from "../src/pod-host-router.ts"; +const PRINCIPAL = "owner@example.com"; +function authority(upstreamSessionId: string): SessionRef { + return { + hostId: hostId("session-pod"), + sessionId: sessionId(upstreamSessionId), + project: { projectId: projectId("t4-code"), name: "T4 code" }, + revision: revision("authority-r1"), + title: "Authoritative OMP session", + status: "idle", + updatedAt: "2026-07-20T00:00:00.000Z", + }; +} + const host = { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4ClusterHost", @@ -14,14 +27,14 @@ const workspace = { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", metadata: { name: "workspace-one", uid: "workspace-uid", resourceVersion: "11", generation: 1 }, - spec: { displayName: "Workspace one", retentionPolicy: "Retain", size: "20Gi" }, + spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "Workspace one", retentionPolicy: "Retain", size: "20Gi" }, status: { observedGeneration: 1, phase: "Ready", conditions: [] }, }; const session = (name: string, upstream: string, previewId?: string) => ({ apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", metadata: { name, uid: `${name}-uid`, resourceVersion: name === "session-one" ? "12" : "13", generation: 1 }, - spec: { workspaceRef: "workspace-one", title: name, runtimeProfile: "omp-17.0.5", gui: { enabled: true } }, + spec: { hostRef: "primary", workspaceRef: "workspace-one", title: name, runtimeProfile: "omp-17.0.5", gui: { enabled: true } }, status: { observedGeneration: 1, phase: "Running", @@ -77,11 +90,13 @@ function setup(epoch = "replica-uid-1") { sessions: [session("session-one", "omp-private-one", "preview-one"), session("session-two", "omp-private-two")], resourceVersion: "13", }); + projection.setSessionAuthority("session-one", authority("omp-private-one")); + projection.setSessionAuthority("session-two", authority("omp-private-two")); const connector = new MemoryConnector(); const mutations = new MemoryMutations(); const gateway = new ClusterGateway({ projection, connector, mutations, internalToken: "x".repeat(32) }); const client = new MemoryClient(); - const connection = gateway.connect(client); + const connection = gateway.connect(client, PRINCIPAL); return { projection, connector, mutations, gateway, client, connection }; } diff --git a/packages/cluster-server/test/kubernetes-client.test.ts b/packages/cluster-server/test/kubernetes-client.test.ts index 49455d0f..29d27993 100644 --- a/packages/cluster-server/test/kubernetes-client.test.ts +++ b/packages/cluster-server/test/kubernetes-client.test.ts @@ -1,10 +1,12 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, test } from "vite-plus/test" import { KubernetesApiClient, KubernetesGatewayMutationBackend, semanticResourceHash, } from "../src/kubernetes-client.ts"; +const PRINCIPAL = "owner@example.com"; + function recordingFetch(responses: unknown[]) { const requests: Array<{ url: string; init?: RequestInit }> = []; const fetch = (async (input: string | URL | Request, init?: RequestInit) => { @@ -14,10 +16,24 @@ function recordingFetch(responses: unknown[]) { return { requests, fetch }; } +function conflictFetch(existing: unknown) { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + return requests.length === 1 + ? Response.json({ reason: "AlreadyExists" }, { status: 409 }) + : Response.json(existing); + }) as typeof globalThis.fetch; + return { requests, fetch }; +} + describe("namespaced Kubernetes client", () => { test("lists and watches only the three cluster.t4.dev resources with bounded resource versions", async () => { const values = recordingFetch([ - { metadata: { resourceVersion: "20" }, items: [] }, + { + metadata: { resourceVersion: "20" }, + items: [{ apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4ClusterHost", metadata: { name: "primary", uid: "host-uid", resourceVersion: "20" }, spec: {} }], + }, { metadata: { resourceVersion: "21" }, items: [] }, { metadata: { resourceVersion: "22" }, items: [] }, ]); @@ -34,12 +50,17 @@ describe("namespaced Kubernetes client", () => { "https://kubernetes.default.svc/apis/cluster.t4.dev/v1alpha1/namespaces/development/t4workspaces?limit=256", "https://kubernetes.default.svc/apis/cluster.t4.dev/v1alpha1/namespaces/development/t4sessions?limit=1000", ]); - for (const request of values.requests) expect(request.init?.headers).toMatchObject({ Authorization: "Bearer service-account-token" }); + for (const request of values.requests) { + expect(new Headers(request.init?.headers).get("authorization")).toBe("Bearer service-account-token"); + } expect(JSON.stringify(listed)).not.toContain("service-account-token"); }); test("persists idempotent CR identity as command id plus semantic hash without credentials or arbitrary URLs", async () => { - const values = recordingFetch([]); + const values = recordingFetch([ + {}, + { kind: "T4Workspace", metadata: { name: "workspace-one" }, spec: { hostRef: "primary", owner: PRINCIPAL } }, + ]); const client = new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", @@ -54,7 +75,7 @@ describe("namespaced Kubernetes client", () => { storageClass: "t4-workspaces-rwx", repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, }; - await backend.createWorkspace("command-create-workspace", workspaceArgs); + await backend.createWorkspace("command-create-workspace", workspaceArgs, PRINCIPAL); const workspaceBody = JSON.parse(String(values.requests[0]?.init?.body)); expect(values.requests[0]).toMatchObject({ url: "https://kubernetes.default.svc/apis/cluster.t4.dev/v1alpha1/namespaces/development/t4workspaces", @@ -67,10 +88,11 @@ describe("namespaced Kubernetes client", () => { name: expect.stringMatching(/^workspace-[a-f0-9]{16}$/), annotations: { "cluster.t4.dev/command-id": "command-create-workspace", - "cluster.t4.dev/semantic-hash": semanticResourceHash(workspaceArgs), + "cluster.t4.dev/principal-hash": semanticResourceHash(PRINCIPAL), + "cluster.t4.dev/semantic-hash": semanticResourceHash({ args: workspaceArgs, principal: PRINCIPAL }), }, }, - spec: { hostRef: "primary", displayName: "Created workspace", retentionPolicy: "Retain", size: "20Gi" }, + spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "Created workspace", retentionPolicy: "Retain", size: "20Gi" }, }); expect(JSON.stringify(workspaceBody)).not.toContain("token"); expect(JSON.stringify(workspaceBody)).not.toContain("url"); @@ -81,8 +103,8 @@ describe("namespaced Kubernetes client", () => { runtimeProfile: "omp-17.0.5", guiEnabled: true, ci: { provider: "woodpecker", repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, - }); - const sessionBody = JSON.parse(String(values.requests[1]?.init?.body)); + }, PRINCIPAL); + const sessionBody = JSON.parse(String(values.requests[2]?.init?.body)); expect(sessionBody).toMatchObject({ apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", @@ -91,25 +113,30 @@ describe("namespaced Kubernetes client", () => { }); }); - test("reuses an existing exact command annotation and rejects semantic conflicts", async () => { + test("reuses exact principal-scoped annotations and rejects semantic conflicts", async () => { const args = { displayName: "Created", retentionPolicy: "Delete" as const, capacity: "10Gi" }; + const annotations = { + "cluster.t4.dev/command-id": "command-one", + "cluster.t4.dev/principal-hash": semanticResourceHash(PRINCIPAL), + "cluster.t4.dev/semantic-hash": semanticResourceHash({ args, principal: PRINCIPAL }), + }; const existing = { - metadata: { - name: "workspace-existing", - resourceVersion: "9", - annotations: { - "cluster.t4.dev/command-id": "command-one", - "cluster.t4.dev/semantic-hash": semanticResourceHash(args), - }, - }, + metadata: { name: "workspace-existing", resourceVersion: "9", annotations }, status: { revision: "workspace-r1" }, }; - const values = recordingFetch([{ items: [existing], metadata: { resourceVersion: "9" } }]); + const exact = conflictFetch(existing); const backend = new KubernetesGatewayMutationBackend({ - client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: exact.fetch }), + hostRef: "primary", + }); + expect(await backend.createWorkspace("command-one", args, PRINCIPAL)).toEqual({ id: "workspace-existing", revision: "9" }); + expect(exact.requests.map(request => request.init?.method ?? "GET")).toEqual(["POST", "GET"]); + + const conflicting = conflictFetch({ ...existing, metadata: { ...existing.metadata, annotations: { ...annotations, "cluster.t4.dev/semantic-hash": "wrong" } } }); + const conflictingBackend = new KubernetesGatewayMutationBackend({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: conflicting.fetch }), hostRef: "primary", }); - expect(await backend.createWorkspace("command-one", args)).toEqual({ id: "workspace-existing", revision: "9" }); - expect(values.requests.every(request => request.init?.method !== "POST")).toBe(true); + await expect(conflictingBackend.createWorkspace("command-one", args, PRINCIPAL)).rejects.toThrow("idempotency conflict"); }); }); diff --git a/packages/cluster-server/test/kubernetes-projection.test.ts b/packages/cluster-server/test/kubernetes-projection.test.ts index 78a98682..0e715585 100644 --- a/packages/cluster-server/test/kubernetes-projection.test.ts +++ b/packages/cluster-server/test/kubernetes-projection.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, test } from "vite-plus/test" +import { hostId, projectId, revision, sessionId, type SessionRef } from "@t4-code/host-wire"; import { CLUSTER_MAX_SESSIONS, CLUSTER_MAX_WORKSPACES, @@ -6,6 +7,19 @@ import { clusterHostIdFromUid, } from "../src/kubernetes-projection.ts"; +const PRINCIPAL = "owner@example.com"; +function authority(upstreamSessionId: string): SessionRef { + return { + hostId: hostId("session-pod"), + sessionId: sessionId(upstreamSessionId), + project: { projectId: projectId("t4-code"), name: "T4 code" }, + revision: revision("authority-r1"), + title: "Authoritative OMP session", + status: "idle", + updatedAt: "2026-07-20T00:00:00.000Z", + }; +} + const host = { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4ClusterHost", @@ -18,6 +32,8 @@ const workspace = { kind: "T4Workspace", metadata: { name: "workspace-one", uid: "workspace-uid", resourceVersion: "101", generation: 3 }, spec: { + hostRef: "primary", + owner: PRINCIPAL, displayName: "T4 code", retentionPolicy: "Retain", storageClass: "t4-workspaces-rwx", @@ -37,6 +53,7 @@ const session = { kind: "T4Session", metadata: { name: "session-one", uid: "session-uid", resourceVersion: "102", generation: 5 }, spec: { + hostRef: "primary", workspaceRef: "workspace-one", title: "Cluster task", runtimeProfile: "omp-17.0.5", @@ -103,6 +120,7 @@ describe("Kubernetes infrastructure projection", () => { test("projects routable pod authority and removes deleted sessions without local truth", () => { const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development" }); projection.replace({ host, workspaces: [workspace], sessions: [session], resourceVersion: "102" }); + projection.setSessionAuthority("session-one", authority("omp-session-private")); expect(projection.sessionRoute("session-one")).toEqual({ clusterSessionId: "session-one", upstreamSessionId: "omp-session-private", diff --git a/packages/cluster-server/test/observability.test.ts b/packages/cluster-server/test/observability.test.ts index 760dfd98..97282d30 100644 --- a/packages/cluster-server/test/observability.test.ts +++ b/packages/cluster-server/test/observability.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, test } from "vite-plus/test" import { ClusterMetrics, ClusterServerHealth, @@ -6,6 +6,7 @@ import { createAdminHandler, redactStructuredValue, } from "../src/observability.ts"; +import { isLoopbackAddress } from "../src/server.ts"; describe("cluster-server observability", () => { test("redacts credentials, prompt content, pairing data, and private paths recursively", () => { @@ -34,12 +35,10 @@ describe("cluster-server observability", () => { expect(JSON.parse(lines[0] ?? "{}")).toMatchObject({ level: "info", message: "ci lookup", provider: "woodpecker", token: "[REDACTED]" }); }); - test("serves separate health/readiness, bounded metrics, and graceful drain", async () => { + test("serves separate health/readiness and bounded metrics without a remote drain route", async () => { const health = new ClusterServerHealth(); const metrics = new ClusterMetrics({ component: "cluster-server", version: "test" }); - let drained = 0; - const handler = createAdminHandler({ health, metrics, drain: async () => { drained++; health.beginDrain(); } }); - + const handler = createAdminHandler({ health, metrics }); expect((await handler(new Request("http://admin/healthz"))).status).toBe(200); expect((await handler(new Request("http://admin/readyz"))).status).toBe(503); health.markKubernetesSynced(); @@ -54,10 +53,16 @@ describe("cluster-server observability", () => { expect(body).toContain('t4_cluster_gateway_commands_total{component="cluster-server",provider="woodpecker",result="success",version="test"} 1'); expect(body).not.toContain("token"); - expect((await handler(new Request("http://admin/drainz", { method: "POST" }))).status).toBe(202); - expect(drained).toBe(1); - expect((await handler(new Request("http://admin/readyz"))).status).toBe(503); - expect((await handler(new Request("http://admin/drainz"))).status).toBe(405); + expect((await handler(new Request("http://admin/drainz", { method: "POST" }))).status).toBe(404); + expect((await handler(new Request("http://admin/readyz"))).status).toBe(200); + }); + + test("recognizes only kernel loopback sources for the preStop drain route", () => { + expect(isLoopbackAddress("127.0.0.1")).toBe(true); + expect(isLoopbackAddress("127.42.0.7")).toBe(true); + expect(isLoopbackAddress("::1")).toBe(true); + expect(isLoopbackAddress("10.42.0.7")).toBe(false); + expect(isLoopbackAddress("fd7a:115c:a1e0::1")).toBe(false); }); test("rejects unbounded or secret-like metric labels", () => { diff --git a/packages/cluster-server/test/session-host-policy.test.ts b/packages/cluster-server/test/session-host-policy.test.ts index 918aeaf6..c759d14b 100644 --- a/packages/cluster-server/test/session-host-policy.test.ts +++ b/packages/cluster-server/test/session-host-policy.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, test } from "vite-plus/test" import type { RemoteConnection } from "@t4-code/host-service"; import { ClusterInternalRemotePolicy, sessionHostConfigFromEnv } from "../src/session-host-policy.ts"; diff --git a/packages/cluster-server/test/woodpecker.test.ts b/packages/cluster-server/test/woodpecker.test.ts index c889ea41..cf503f24 100644 --- a/packages/cluster-server/test/woodpecker.test.ts +++ b/packages/cluster-server/test/woodpecker.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, test } from "vite-plus/test" import { WoodpeckerProvider, mapWoodpeckerPipeline } from "../src/woodpecker.ts"; const correlation = { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4fe2d092..e8b726fc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -259,6 +259,28 @@ importers: specifier: 'catalog:' version: 0.2.2(@types/node@26.1.1)(jiti@2.7.0)(typescript@6.0.3) + packages/cluster-server: + dependencies: + '@t4-code/host-service': + specifier: workspace:* + version: link:../host-service + '@t4-code/host-wire': + specifier: workspace:* + version: link:../host-wire + devDependencies: + '@types/bun': + specifier: 1.3.5 + version: 1.3.5 + '@types/node': + specifier: 24.12.4 + version: 24.12.4 + typescript: + specifier: ~6.0.3 + version: 6.0.3 + vite-plus: + specifier: 0.2.2 + version: 0.2.2(@types/node@24.12.4)(jiti@2.7.0)(typescript@6.0.3) + packages/client: dependencies: '@t4-code/protocol': diff --git a/scripts/cluster-ci/cluster-ci-contract.test.mjs b/scripts/cluster-ci/cluster-ci-contract.test.mjs index 6bcf9d07..630f3758 100644 --- a/scripts/cluster-ci/cluster-ci-contract.test.mjs +++ b/scripts/cluster-ci/cluster-ci-contract.test.mjs @@ -227,6 +227,7 @@ test("Woodpecker keeps upstream gates and serializes bounded cluster publication assert.ok( steps["cluster-server-tests"].commands.includes("pnpm --filter @t4-code/cluster-server test"), ); + assert.ok(steps["cluster-wire-tests"].commands.includes("pnpm --filter @t4-code/host-wire test")); assert.equal(JSON.stringify(pipeline).includes("from_secret"), false); assert.deepEqual(steps["harbor-auth"].depends_on, ["cluster-chart-tests", "android-debug"]); assert.equal( From 0bbe04ca542b00d071ad080a766a666784fda1d2 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 16:17:19 +0000 Subject: [PATCH 14/98] fix(cluster): align gateway resources with CRDs --- .../cluster-server/src/kubernetes-client.ts | 8 +++---- .../src/kubernetes-projection.ts | 8 +++---- packages/cluster-server/test/gateway.test.ts | 24 ++++++++++++------- .../test/kubernetes-client.test.ts | 12 ++++++++-- .../test/kubernetes-projection.test.ts | 15 +++++------- 5 files changed, 39 insertions(+), 28 deletions(-) diff --git a/packages/cluster-server/src/kubernetes-client.ts b/packages/cluster-server/src/kubernetes-client.ts index faf9479f..b60cc502 100644 --- a/packages/cluster-server/src/kubernetes-client.ts +++ b/packages/cluster-server/src/kubernetes-client.ts @@ -206,8 +206,8 @@ export class KubernetesGatewayMutationBackend { displayName: args.displayName, retentionPolicy: args.retentionPolicy, size: args.capacity, - ...(args.storageClass ? { storageClass: args.storageClass } : {}), - ...(args.repository ? { repository: { id: args.repository.repositoryId, ...(args.repository.ref ? { ref: args.repository.ref } : {}), ...(args.repository.commit ? { commit: args.repository.commit } : {}) } } : {}), + ...(args.storageClass ? { storageClassName: args.storageClass } : {}), + ...(args.repository ? { repository: { repositoryId: args.repository.repositoryId, ...(args.repository.ref ? { ref: args.repository.ref } : {}), ...(args.repository.commit ? { commit: args.repository.commit } : {}) } } : {}), }, }; const resource = await this.#createOrRead("t4workspaces", name, body, commandId, semanticResourceHash({ args, principal }), principal); @@ -227,9 +227,9 @@ export class KubernetesGatewayMutationBackend { spec: { hostRef: this.#hostRef, workspaceRef: args.workspaceId, - ...(args.title ? { title: args.title } : {}), + title: args.title ?? "Cluster session", runtimeProfile: args.runtimeProfile, - gui: { enabled: args.guiEnabled }, + guiEnabled: args.guiEnabled, ...(args.ci ? { ci: { repositoryId: args.ci.repositoryId, ref: args.ci.ref, commit: args.ci.commit } } : {}), }, }; diff --git a/packages/cluster-server/src/kubernetes-projection.ts b/packages/cluster-server/src/kubernetes-projection.ts index e7b2304a..b0a136bd 100644 --- a/packages/cluster-server/src/kubernetes-projection.ts +++ b/packages/cluster-server/src/kubernetes-projection.ts @@ -249,7 +249,7 @@ export class ClusterInfrastructureProjection { } sessionEndpoints(): PodHostEndpoint[] { return [...this.#sessions.values()].flatMap(resource => { - const service = serviceName(record(resource.status).serviceRef); + const service = serviceName(record(resource.status).serviceName); return service ? [{ clusterSessionId: resource.metadata.name, url: `ws://${service}.${this.namespace}.svc:8787/v1/ws` }] : []; }); } @@ -257,7 +257,7 @@ export class ClusterInfrastructureProjection { const resource = this.#sessions.get(clusterSessionId); const authoritative = this.#authoritativeSessions.get(clusterSessionId); if (!resource || !authoritative || principal !== undefined && !this.#ownsSessionResource(resource, principal)) return undefined; - const service = serviceName(record(resource.status).serviceRef); + const service = serviceName(record(resource.status).serviceName); if (!service) return undefined; return { clusterSessionId, upstreamSessionId: authoritative.sessionId, url: `ws://${service}.${this.namespace}.svc:8787/v1/ws` }; } @@ -407,7 +407,7 @@ export class ClusterInfrastructureProjection { const workspaceId = text(spec.workspaceRef, "unknown-workspace"); const ci = record(spec.ci); const condition = firstCondition(status); - const guiEnabled = record(spec.gui).enabled === true; + const guiEnabled = spec.guiEnabled === true; const infrastructurePhase = categorical(status.phase, ["Pending", "Running", "Failed", "Terminating", "Unknown"] as const, "Unknown"); const guiState = !guiEnabled ? "Unavailable" @@ -445,7 +445,7 @@ export class ClusterInfrastructureProjection { displayName: text(spec.displayName, resource.metadata.name).slice(0, 256), phase: resource.metadata.deletionTimestamp ? "Terminating" : categorical(status.phase, ["Pending", "Ready", "Failed", "Terminating", "Unknown"] as const, "Unknown"), retentionPolicy: spec.retentionPolicy === "Delete" ? "Delete" : "Retain", - ...(typeof spec.storageClass === "string" ? { storageClass: spec.storageClass.slice(0, 128) } : {}), + ...(typeof spec.storageClassName === "string" ? { storageClass: spec.storageClassName.slice(0, 128) } : {}), ...(typeof status.capacity === "string" ? { capacity: status.capacity.slice(0, 64) } : typeof spec.size === "string" ? { capacity: spec.size.slice(0, 64) } : {}), accessMode: "ReadWriteMany", revision: resourceRevision(resource), diff --git a/packages/cluster-server/test/gateway.test.ts b/packages/cluster-server/test/gateway.test.ts index ebb95e74..cbc17575 100644 --- a/packages/cluster-server/test/gateway.test.ts +++ b/packages/cluster-server/test/gateway.test.ts @@ -30,18 +30,15 @@ const workspace = { spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "Workspace one", retentionPolicy: "Retain", size: "20Gi" }, status: { observedGeneration: 1, phase: "Ready", conditions: [] }, }; -const session = (name: string, upstream: string, previewId?: string) => ({ +const session = (name: string, upstream: string) => ({ apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", metadata: { name, uid: `${name}-uid`, resourceVersion: name === "session-one" ? "12" : "13", generation: 1 }, - spec: { hostRef: "primary", workspaceRef: "workspace-one", title: name, runtimeProfile: "omp-17.0.5", gui: { enabled: true } }, + spec: { hostRef: "primary", workspaceRef: "workspace-one", title: name, runtimeProfile: "omp-17.0.5", guiEnabled: true }, status: { observedGeneration: 1, phase: "Running", - serviceRef: name, - upstreamSessionId: upstream, - guiState: "Ready", - ...(previewId ? { previewId } : {}), + serviceName: name, conditions: [], }, }); @@ -87,7 +84,7 @@ function setup(epoch = "replica-uid-1") { projection.replace({ host, workspaces: [workspace], - sessions: [session("session-one", "omp-private-one", "preview-one"), session("session-two", "omp-private-two")], + sessions: [session("session-one", "omp-private-one"), session("session-two", "omp-private-two")], resourceVersion: "13", }); projection.setSessionAuthority("session-one", authority("omp-private-one")); @@ -181,14 +178,23 @@ describe("stateless omp-app cluster gateway", () => { }); }); - test("denies cross-session preview ids before opening an upstream socket", async () => { + test("denies a preview id learned from another session without opening a second upstream socket", async () => { const value = setup(); await value.connection.receive(hello); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "request-owner", commandId: "command-owner", + hostId: "cluster:host-uid", sessionId: "session-one", command: "preview.state", args: { previewId: "preview-one" }, + }); + value.connector.onFrame?.({ + v: "omp-app/1", type: "preview.state", hostId: "upstream" as never, sessionId: "omp-private-one" as never, + previewId: "preview-one" as never, state: "ready", url: "https://example.test", revision: "preview-r1" as never, + cursor: { epoch: "preview-e1", seq: 1 }, + }); await value.connection.receive({ v: "omp-app/1", type: "command", requestId: "request-preview", commandId: "command-preview", hostId: "cluster:host-uid", sessionId: "session-two", command: "preview.state", args: { previewId: "preview-one" }, }); - expect(value.connector.routes).toHaveLength(0); + expect(value.connector.routes).toHaveLength(1); expect(value.client.frames.at(-1)).toMatchObject({ type: "response", commandId: "command-preview", ok: false, error: { code: "NOT_AUTHORIZED" } }); }); diff --git a/packages/cluster-server/test/kubernetes-client.test.ts b/packages/cluster-server/test/kubernetes-client.test.ts index 29d27993..f568e536 100644 --- a/packages/cluster-server/test/kubernetes-client.test.ts +++ b/packages/cluster-server/test/kubernetes-client.test.ts @@ -92,7 +92,15 @@ describe("namespaced Kubernetes client", () => { "cluster.t4.dev/semantic-hash": semanticResourceHash({ args: workspaceArgs, principal: PRINCIPAL }), }, }, - spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "Created workspace", retentionPolicy: "Retain", size: "20Gi" }, + spec: { + hostRef: "primary", + owner: PRINCIPAL, + displayName: "Created workspace", + retentionPolicy: "Retain", + size: "20Gi", + storageClassName: "t4-workspaces-rwx", + repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, + }, }); expect(JSON.stringify(workspaceBody)).not.toContain("token"); expect(JSON.stringify(workspaceBody)).not.toContain("url"); @@ -109,7 +117,7 @@ describe("namespaced Kubernetes client", () => { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", metadata: { name: expect.stringMatching(/^session-[a-f0-9]{16}$/) }, - spec: { hostRef: "primary", workspaceRef: "workspace-one", runtimeProfile: "omp-17.0.5", gui: { enabled: true } }, + spec: { hostRef: "primary", workspaceRef: "workspace-one", title: "Task", runtimeProfile: "omp-17.0.5", guiEnabled: true }, }); }); diff --git a/packages/cluster-server/test/kubernetes-projection.test.ts b/packages/cluster-server/test/kubernetes-projection.test.ts index 0e715585..f0ca764a 100644 --- a/packages/cluster-server/test/kubernetes-projection.test.ts +++ b/packages/cluster-server/test/kubernetes-projection.test.ts @@ -36,9 +36,9 @@ const workspace = { owner: PRINCIPAL, displayName: "T4 code", retentionPolicy: "Retain", - storageClass: "t4-workspaces-rwx", + storageClassName: "t4-workspaces-rwx", size: "20Gi", - repository: { id: "t4-code", ref: "refs/heads/main", commit: "abc", credentialPath: "/secret" }, + repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, }, status: { observedGeneration: 3, @@ -57,17 +57,14 @@ const session = { workspaceRef: "workspace-one", title: "Cluster task", runtimeProfile: "omp-17.0.5", - gui: { enabled: true }, + guiEnabled: true, ci: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, }, status: { observedGeneration: 5, phase: "Running", - podRef: "session-one-pod", - serviceRef: "session-one", - upstreamSessionId: "omp-session-private", - previewId: "preview-one", - guiState: "Ready", + podName: "session-one-pod", + serviceName: "session-one", conditions: [{ type: "Available", status: "True", reason: "PodReady", message: "ready", observedGeneration: 5 }], }, }; @@ -93,8 +90,8 @@ describe("Kubernetes infrastructure projection", () => { }); expect(JSON.stringify(projection.workspaceList())).not.toContain("credentialPath"); expect(JSON.stringify(projection.workspaceList())).not.toContain("pvcRef"); + expect(JSON.stringify(projection.workspaceList())).not.toContain("repositoryId"); }); - test("keeps workspace cursors separate and reconnect replacement idempotent", () => { const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development" }); projection.replace({ host, workspaces: [workspace], sessions: [session], resourceVersion: "102" }); From 6b4553576b90acb0a6f13ce03980c25329e2671e Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 16:21:37 +0000 Subject: [PATCH 15/98] fix(operator): wait for sessions before workspace deletion --- .../controllers/reconciler_test.go | 27 +++++++++++++++++++ .../controllers/workspace_controller.go | 15 +++++++++++ 2 files changed, 42 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index fdd5ec3e..a056e642 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -135,6 +135,33 @@ func TestRetainDeletionOrphansPVCBeforeRemovingFinalizer(t *testing.T) { } } +func TestWorkspaceDeletionWaitsForSessionResources(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) + workspace.UID = "workspace-uid" + workspace.Finalizers = []string{clusterv1alpha1.WorkspaceFinalizer} + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllers.WorkspacePVCName(workspace), Namespace: workspace.Namespace, + OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, + }, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + } + session := testSession() + session.Spec.WorkspaceRef = workspace.Name + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc, session).Build() + if err := c.Delete(context.Background(), workspace); err != nil { t.Fatal(err) } + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { t.Fatal(err) } + var waiting clusterv1alpha1.T4Workspace + if err := c.Get(context.Background(), client.ObjectKeyFromObject(workspace), &waiting); err != nil { t.Fatalf("workspace deletion did not wait: %v", err) } + condition := findCondition(waiting.Status.Conditions, "Ready") + if condition == nil || condition.Reason != "SessionsRemain" { t.Fatalf("Ready = %#v, want SessionsRemain", condition) } + var retained corev1.PersistentVolumeClaim + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pvc), &retained); err != nil { t.Fatalf("workspace PVC changed during wait: %v", err) } + if len(retained.OwnerReferences) != 1 { t.Fatalf("workspace PVC was orphaned before sessions exited: %#v", retained.OwnerReferences) } +} + func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index 780d17fa..4c1dea76 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -127,6 +127,21 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl if !reflect.DeepEqual(originalStatus, workspace.Status) { if err := r.Status().Update(ctx, workspace); err != nil { return ctrl.Result{}, err } } + var sessions clusterv1alpha1.T4SessionList + if err := r.List(ctx, &sessions, client.InNamespace(workspace.Namespace)); err != nil { return ctrl.Result{}, err } + remainingSessions := 0 + for i := range sessions.Items { + if sessions.Items[i].Spec.WorkspaceRef == workspace.Name { remainingSessions++ } + } + if remainingSessions > 0 { + before := workspace.Status + if workspace.Status.Conditions != nil { before.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) } + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "SessionsRemain", fmt.Sprintf("workspace deletion is waiting for %d session resources", remainingSessions), workspace.Generation)) + if !reflect.DeepEqual(before, workspace.Status) { + if err := r.Status().Update(ctx, workspace); err != nil { return ctrl.Result{}, err } + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } pvcKey := types.NamespacedName{Namespace: workspace.Namespace, Name: WorkspacePVCName(workspace)} var pvc corev1.PersistentVolumeClaim err := r.Get(ctx, pvcKey, &pvc) From b8bd3c03f93eb4f26640997c2307774b5c872757 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 16:25:14 +0000 Subject: [PATCH 16/98] Replace cluster static auth with TokenReview identity --- .../session-runtime/session-entrypoint.sh | 8 +- .../t4-cluster/templates/deployments.yaml | 30 ++- .../templates/internal-auth-secret.yaml | 15 -- .../t4-cluster/templates/networkpolicies.yaml | 7 + deploy/charts/t4-cluster/templates/rbac.yaml | 34 ++++ deploy/charts/t4-cluster/values.schema.json | 7 +- deploy/charts/t4-cluster/values.yaml | 4 - docs/CLUSTER_OPERATOR.md | 7 +- .../charttests/chart_contract_test.go | 51 +++++ packages/cluster-operator/cmd/manager/main.go | 74 +++++--- .../cluster-operator/cmd/manager/main_test.go | 17 ++ .../controllers/reconciler_test.go | 52 ++++- .../controllers/session_controller.go | 179 +++++++++++++----- packages/cluster-server/src/config.ts | 50 +++-- packages/cluster-server/src/gateway.ts | 2 - .../cluster-server/src/kubernetes-client.ts | 84 +++++++- packages/cluster-server/src/main.ts | 4 +- .../cluster-server/src/pod-host-router.ts | 35 ++-- .../cluster-server/src/session-host-main.ts | 9 +- .../cluster-server/src/session-host-policy.ts | 92 ++++++--- packages/cluster-server/test/config.test.ts | 44 ++++- packages/cluster-server/test/gateway.test.ts | 24 ++- .../test/kubernetes-client.test.ts | 100 +++++++++- .../test/kubernetes-projection.test.ts | 11 +- .../cluster-server/test/observability.test.ts | 10 +- .../test/pod-host-router.test.ts | 62 ++++++ .../test/session-host-policy.test.ts | 87 ++++++--- .../cluster-server/test/woodpecker.test.ts | 10 +- packages/host-service/src/server.ts | 5 +- packages/host-service/src/types.ts | 2 + 30 files changed, 880 insertions(+), 236 deletions(-) delete mode 100644 deploy/charts/t4-cluster/templates/internal-auth-secret.yaml create mode 100644 packages/cluster-server/test/pod-host-router.test.ts diff --git a/cluster/images/session-runtime/session-entrypoint.sh b/cluster/images/session-runtime/session-entrypoint.sh index f001f10c..60f7389c 100755 --- a/cluster/images/session-runtime/session-entrypoint.sh +++ b/cluster/images/session-runtime/session-entrypoint.sh @@ -4,7 +4,13 @@ umask 077 : "${T4_AUTHORITY_STATE_DIR:?T4_AUTHORITY_STATE_DIR is required}" : "${T4_BROWSER_STATE_DIR:?T4_BROWSER_STATE_DIR is required}" -: "${T4_CLUSTER_INTERNAL_TOKEN:?T4_CLUSTER_INTERNAL_TOKEN is required}" +: "${T4_CLUSTER_SERVER_SERVICE_ACCOUNT:?T4_CLUSTER_SERVER_SERVICE_ACCOUNT is required}" +export T4_KUBERNETES_TOKEN_PATH="${T4_KUBERNETES_TOKEN_PATH:-/var/run/secrets/kubernetes.io/serviceaccount/token}" +export T4_KUBERNETES_CA_PATH="${T4_KUBERNETES_CA_PATH:-/var/run/secrets/kubernetes.io/serviceaccount/ca.crt}" +export T4_KUBERNETES_NAMESPACE_PATH="${T4_KUBERNETES_NAMESPACE_PATH:-/var/run/secrets/kubernetes.io/serviceaccount/namespace}" +for projected_file in "${T4_KUBERNETES_TOKEN_PATH}" "${T4_KUBERNETES_CA_PATH}" "${T4_KUBERNETES_NAMESPACE_PATH}"; do + [[ -f "${projected_file}" && -r "${projected_file}" ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"kubernetes_api_projection"}' >&2; exit 64; } +done case "${T4_AUTHORITY_STATE_DIR}" in /workspace/.t4/sessions/*/authority) ;; diff --git a/deploy/charts/t4-cluster/templates/deployments.yaml b/deploy/charts/t4-cluster/templates/deployments.yaml index af1d670d..5e06f8ae 100644 --- a/deploy/charts/t4-cluster/templates/deployments.yaml +++ b/deploy/charts/t4-cluster/templates/deployments.yaml @@ -73,8 +73,10 @@ spec: valueFrom: {fieldRef: {fieldPath: metadata.namespace}} - name: T4_SESSION_RUNTIME_IMAGE value: {{ include "t4-cluster.image" .Values.images.sessionRuntime }} - - name: T4_INTERNAL_AUTH_SECRET - value: {{ .Values.internalAuth.secretName }} + - name: T4_SESSION_SERVICE_ACCOUNT + value: {{ include "t4-cluster.fullname" . }}-session + - name: T4_CLUSTER_SERVER_SERVICE_ACCOUNT + value: {{ include "t4-cluster.fullname" . }}-server - name: T4_SESSION_EXCLUDED_NODES value: {{ join "," .Values.session.nodeExclude | quote }} - name: T4_SESSION_REQUEST_CPU @@ -202,17 +204,20 @@ spec: value: "9090" - name: T4_CLUSTER_HOST_NAME value: {{ .Values.clusterHost.name }} + - name: T4_CLUSTER_IDENTITY_TOKEN_FILE + value: /var/run/secrets/t4-cluster-identity/token + - name: T4_CLUSTER_SERVER_SERVICE_ACCOUNT + value: {{ include "t4-cluster.fullname" . }}-server + - name: T4_KUBERNETES_TOKEN_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: T4_KUBERNETES_CA_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - name: POD_NAMESPACE valueFrom: {fieldRef: {fieldPath: metadata.namespace}} - name: POD_NAME valueFrom: {fieldRef: {fieldPath: metadata.name}} - name: POD_UID valueFrom: {fieldRef: {fieldPath: metadata.uid}} - - name: T4_CLUSTER_INTERNAL_TOKEN - valueFrom: - secretKeyRef: - name: {{ .Values.internalAuth.secretName }} - key: token {{- if .Values.woodpecker.existingSecret }} - name: T4_WOODPECKER_TOKEN valueFrom: @@ -265,7 +270,18 @@ spec: volumeMounts: - name: temporary mountPath: /tmp + - name: cluster-identity + mountPath: /var/run/secrets/t4-cluster-identity + readOnly: true volumes: + - name: cluster-identity + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: + audience: t4-cluster-internal + expirationSeconds: 600 + path: token - name: temporary emptyDir: sizeLimit: 256Mi diff --git a/deploy/charts/t4-cluster/templates/internal-auth-secret.yaml b/deploy/charts/t4-cluster/templates/internal-auth-secret.yaml deleted file mode 100644 index 767b5498..00000000 --- a/deploy/charts/t4-cluster/templates/internal-auth-secret.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.enabled }} -{{- $name := .Values.internalAuth.secretName }} -{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ $name }} - labels: - {{- include "t4-cluster.labels" . | nindent 4 }} - annotations: - helm.sh/resource-policy: keep -type: Opaque -data: - token: {{ if and $existing $existing.data $existing.data.token }}{{ index $existing.data "token" }}{{ else }}{{ randAlphaNum 64 | b64enc }}{{ end }} -{{- end }} diff --git a/deploy/charts/t4-cluster/templates/networkpolicies.yaml b/deploy/charts/t4-cluster/templates/networkpolicies.yaml index 22a64bf8..37d94e6c 100644 --- a/deploy/charts/t4-cluster/templates/networkpolicies.yaml +++ b/deploy/charts/t4-cluster/templates/networkpolicies.yaml @@ -110,6 +110,13 @@ spec: ports: - {protocol: TCP, port: 8787} egress: + {{- range .Values.networkPolicy.kubernetesApiCIDRs }} + - to: + - ipBlock: {cidr: {{ . | quote }}} + ports: + - {protocol: TCP, port: 443} + - {protocol: TCP, port: 6443} + {{- end }} {{- range .Values.networkPolicy.modelRouteCIDRs }} - to: - ipBlock: {cidr: {{ . | quote }}} diff --git a/deploy/charts/t4-cluster/templates/rbac.yaml b/deploy/charts/t4-cluster/templates/rbac.yaml index 3f3209b5..3c330c49 100644 --- a/deploy/charts/t4-cluster/templates/rbac.yaml +++ b/deploy/charts/t4-cluster/templates/rbac.yaml @@ -47,6 +47,17 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: {{ include "t4-cluster.fullname" . }}-session-token-reviewer + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: [authentication.k8s.io] + resources: [tokenreviews] + verbs: [create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: {{ include "t4-cluster.fullname" . }}-storage-reader labels: @@ -72,6 +83,14 @@ metadata: {{- include "t4-cluster.labels" . | nindent 4 }} automountServiceAccountToken: true --- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "t4-cluster.fullname" . }}-session + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +automountServiceAccountToken: false +--- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -116,4 +135,19 @@ roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: {{ include "t4-cluster.fullname" . }}-server +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "t4-cluster.fullname" . }}-session-token-reviewer + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "t4-cluster.fullname" . }}-session + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "t4-cluster.fullname" . }}-session-token-reviewer {{- end }} diff --git a/deploy/charts/t4-cluster/values.schema.json b/deploy/charts/t4-cluster/values.schema.json index 8a3fe1e9..e693624f 100644 --- a/deploy/charts/t4-cluster/values.schema.json +++ b/deploy/charts/t4-cluster/values.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", - "required": ["enabled", "clusterHost", "images", "controller", "server", "session", "storage", "internalAuth", "networkPolicy"], + "required": ["enabled", "clusterHost", "images", "controller", "server", "session", "storage", "networkPolicy"], "properties": { "enabled": { "type": "boolean", "default": false }, "clusterHost": { @@ -63,11 +63,6 @@ "required": ["adminRWXStorageClass"], "properties": { "adminRWXStorageClass": { "type": "string", "maxLength": 63 } } }, - "internalAuth": { - "type": "object", - "required": ["secretName"], - "properties": { "secretName": { "type": "string", "const": "t4-cluster-internal-auth" } } - }, "woodpecker": { "type": "object" }, "ingress": { "type": "object" }, "networkPolicy": { "type": "object" }, diff --git a/deploy/charts/t4-cluster/values.yaml b/deploy/charts/t4-cluster/values.yaml index cfef846c..5b722052 100644 --- a/deploy/charts/t4-cluster/values.yaml +++ b/deploy/charts/t4-cluster/values.yaml @@ -63,10 +63,6 @@ storage: # Required when enabled. The administrator creates this backend-neutral RWX StorageClass. adminRWXStorageClass: "" -internalAuth: - # Fixed default shared by the gateway and controller-created session pods. - secretName: t4-cluster-internal-auth - woodpecker: existingSecret: "" tokenKey: token diff --git a/docs/CLUSTER_OPERATOR.md b/docs/CLUSTER_OPERATOR.md index 346068ea..a0f680db 100644 --- a/docs/CLUSTER_OPERATOR.md +++ b/docs/CLUSTER_OPERATOR.md @@ -79,7 +79,7 @@ helm upgrade --install t4-cluster deploy/charts/t4-cluster --namespace t4-system The controller always has two replicas and uses a Kubernetes Lease named from `t4-cluster-operator.cluster.t4.dev`; one replica reconciles at a time. The server defaults to three stateless replicas and supports a minimum of two. Its Deployment uses `maxUnavailable: 0`, a `minAvailable: 2` PDB, topology spread, anti-affinity, readiness draining, and an explicit `k3s-worker-02` exclusion. Session pods also exclude that node by default. Additional cluster-specific exclusions belong in deployment values, not this portable chart. -The chart creates `t4-cluster-internal-auth` on first enabled install and retains its token across upgrades. The token is mounted only into the gateway and one-session pods. It is used by the existing `omp-app/1` upstream hello and must never appear in a URL, header, CR, log, or client frame. +The chart creates dedicated server and session ServiceAccounts instead of a shared credential Secret. Each server pod receives a 10-minute projected token with the fixed `t4-cluster-internal` audience and presents it only in the existing `omp-app/1` upstream authentication field when dialing a session host. Session pods use `automountServiceAccountToken: false`; their only Kubernetes identity mount is an explicit short-lived API-audience token plus the cluster CA and namespace. The session ServiceAccount may only create `authentication.k8s.io/tokenreviews`, and the host accepts a connection only when TokenReview confirms the expected server ServiceAccount username and audience. ## API configuration @@ -101,7 +101,7 @@ The optional initial prompt is referenced by Secret name and key `prompt`; the S The session runtime verifies and builds the exact OMP tag `t4code-17.0.5-appserver-10` at commit `8476f4451ed95c5d5401785d279a93d3c659fac4`. It preserves `t4-omp-authority/1`, starts the existing T4 session-host entrypoint, and provides Xvfb, a minimal window manager, and Chromium without privileged mode or host display access. The shared claim is mounted at `/workspace`; authority and browser state live in controller-selected per-session subdirectories. `/dev/shm` is an explicit memory-backed volume. Browser Preview remains the existing GUI stream and control surface. -Session pods do not receive ServiceAccount tokens. All containers drop capabilities, disallow privilege escalation, use RuntimeDefault seccomp, and use read-only root filesystems. No per-session NodePort, LoadBalancer, host network, host PID, host display, or hostPath is created. +Session pods do not receive an automatically mounted ServiceAccount token. The explicit projected reviewer token can only create TokenReviews and is not resource-authorized. All containers drop capabilities, disallow privilege escalation, use RuntimeDefault seccomp, and use read-only root filesystems. No per-session NodePort, LoadBalancer, host network, host PID, host display, or hostPath is created. ## Upgrade and rollback @@ -125,7 +125,6 @@ Do not roll OMP independently of the T4 session runtime. Roll back the known-com 2. Delete `T4Session` resources and wait for their pods and Services to be removed. 3. Review every `T4Workspace` retention policy. `Delete` removes its PVC; `Retain` deliberately leaves an orphaned PVC for administrator recovery. 4. Run `helm uninstall t4-cluster --namespace t4-system`. -5. After no session pod remains, delete the retained `t4-cluster-internal-auth` Secret if the installation will not be restored. -6. Remove retained PVCs only after their contents have been recovered or confirmed disposable. +5. Remove retained PVCs only after their contents have been recovered or confirmed disposable. CRDs are not removed by `helm uninstall`. This preserves custom resources and retained storage across rollback/reinstall. Remove the three CRDs only as a separate, explicit administrative operation after confirming no instances remain; CRD deletion removes all instances regardless of their retention intent. diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go index 910a84b9..dd4179c6 100644 --- a/packages/cluster-operator/charttests/chart_contract_test.go +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -68,6 +68,37 @@ func TestRBACSeparatesControllerMutationFromServerProjection(t *testing.T) { } } +func TestChartUsesOnlyProjectedServiceAccountIdentityForInternalPeers(t *testing.T) { + output := helmTemplate(t, enabledValues()...) + assertCount(t, output, "kind: ServiceAccount", 3) + assertCount(t, output, "kind: Secret", 0) + server := documentContainingKind(t, output, "Deployment", "name: release-name-t4-cluster-server") + assertContains(t, server, + "serviceAccountName: release-name-t4-cluster-server", + "name: T4_CLUSTER_IDENTITY_TOKEN_FILE", + "/var/run/secrets/t4-cluster-identity/token", + "serviceAccountToken:", + "audience: t4-cluster-internal", + "expirationSeconds: 600", + ) + controller := documentContainingKind(t, output, "Deployment", "name: release-name-t4-cluster-controller") + assertContains(t, controller, + "name: T4_SESSION_SERVICE_ACCOUNT", + "value: release-name-t4-cluster-session", + "name: T4_CLUSTER_SERVER_SERVICE_ACCOUNT", + "value: release-name-t4-cluster-server", + ) + sessionRole := documentContainingKind(t, output, "ClusterRole", "name: release-name-t4-cluster-session-token-reviewer") + assertContains(t, sessionRole, + "apiGroups: [authentication.k8s.io]", + "resources: [tokenreviews]", + "verbs: [create]", + ) + if strings.Count(sessionRole, "- apiGroups:") != 1 || strings.Contains(sessionRole, "get") || strings.Contains(sessionRole, "list") || strings.Contains(sessionRole, "watch") { + t.Fatalf("session ServiceAccount received permissions beyond TokenReview create:\n%s", sessionRole) + } +} + func TestNetworkPoliciesDefaultDenyAndAllowOnlyDeclaredFlows(t *testing.T) { output := helmTemplate(t, append(enabledValues(), "--set", "networkPolicy.kubernetesApiCIDRs[0]=192.0.2.10/32", @@ -82,6 +113,8 @@ func TestNetworkPoliciesDefaultDenyAndAllowOnlyDeclaredFlows(t *testing.T) { "port: 53", "port: 8787", ) + sessionPolicy := documentContainingKind(t, output, "NetworkPolicy", "name: release-name-t4-cluster-session-host") + assertContains(t, sessionPolicy, "192.0.2.10/32", "port: 443", "port: 6443") if strings.Contains(output, "0.0.0.0/0") { t.Fatal("network policy contains broad Internet egress") } @@ -126,6 +159,13 @@ func TestImageContractsArePinnedAndAuthorityCompatible(t *testing.T) { "Xvfb", ) assertContains(t, server, "packages/cluster-server/src/main.ts") + entrypoint := mustRead(t, filepath.Join(root, "cluster", "images", "session-runtime", "session-entrypoint.sh")) + assertContains(t, entrypoint, + "T4_CLUSTER_SERVER_SERVICE_ACCOUNT", + "/var/run/secrets/kubernetes.io/serviceaccount/token", + "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + "/var/run/secrets/kubernetes.io/serviceaccount/namespace", + ) } func helmTemplate(t *testing.T, extra ...string) string { @@ -194,3 +234,14 @@ func documentContaining(t *testing.T, rendered, needle string) string { t.Fatalf("no rendered document contains %q", needle) return "" } + +func documentContainingKind(t *testing.T, rendered, kind, needle string) string { + t.Helper() + for _, document := range strings.Split(rendered, "\n---") { + if strings.Contains(document, "kind: "+kind+"\n") && strings.Contains(document, needle) { + return document + } + } + t.Fatalf("no rendered %s contains %q", kind, needle) + return "" +} diff --git a/packages/cluster-operator/cmd/manager/main.go b/packages/cluster-operator/cmd/manager/main.go index fc456bba..23a5bede 100644 --- a/packages/cluster-operator/cmd/manager/main.go +++ b/packages/cluster-operator/cmd/manager/main.go @@ -36,16 +36,16 @@ func managerOptions() ctrl.Options { renewDeadline := 20 * time.Second retryPeriod := 5 * time.Second options := ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{BindAddress: ":8080"}, - HealthProbeBindAddress: ":8081", - LeaderElection: true, - LeaderElectionResourceLock: "leases", - LeaderElectionID: "t4-cluster-operator.cluster.t4.dev", + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: ":8080"}, + HealthProbeBindAddress: ":8081", + LeaderElection: true, + LeaderElectionResourceLock: "leases", + LeaderElectionID: "t4-cluster-operator.cluster.t4.dev", LeaderElectionReleaseOnCancel: true, - LeaseDuration: &leaseDuration, - RenewDeadline: &renewDeadline, - RetryPeriod: &retryPeriod, + LeaseDuration: &leaseDuration, + RenewDeadline: &renewDeadline, + RetryPeriod: &retryPeriod, } if namespace := strings.TrimSpace(os.Getenv("POD_NAMESPACE")); namespace != "" { options.Cache = cache.Options{DefaultNamespaces: map[string]cache.Config{namespace: {}}} @@ -62,48 +62,74 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) manager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), managerOptions()) - if err != nil { ctrl.Log.Error(err, "unable to create controller manager"); os.Exit(1) } + if err != nil { + ctrl.Log.Error(err, "unable to create controller manager") + os.Exit(1) + } if err := (&controllers.ClusterHostReconciler{Client: manager.GetClient(), Scheme: manager.GetScheme()}).SetupWithManager(manager); err != nil { - ctrl.Log.Error(err, "unable to register T4ClusterHost controller"); os.Exit(1) + ctrl.Log.Error(err, "unable to register T4ClusterHost controller") + os.Exit(1) } if err := (&controllers.WorkspaceReconciler{Client: manager.GetClient(), Scheme: manager.GetScheme()}).SetupWithManager(manager); err != nil { - ctrl.Log.Error(err, "unable to register T4Workspace controller"); os.Exit(1) + ctrl.Log.Error(err, "unable to register T4Workspace controller") + os.Exit(1) } excludedNodes := splitNonempty(os.Getenv("T4_SESSION_EXCLUDED_NODES")) + sessionServiceAccount, serverServiceAccount := sessionServiceAccountNames() if err := (&controllers.SessionReconciler{ Client: manager.GetClient(), Scheme: manager.GetScheme(), - RuntimeImage: os.Getenv("T4_SESSION_RUNTIME_IMAGE"), - InternalAuthSecret: envOr("T4_INTERNAL_AUTH_SECRET", controllers.DefaultInternalAuthSecret), - ExcludedNodeNames: excludedNodes, + RuntimeImage: os.Getenv("T4_SESSION_RUNTIME_IMAGE"), + SessionServiceAccountName: sessionServiceAccount, + ServerServiceAccountName: serverServiceAccount, + ExcludedNodeNames: excludedNodes, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceCPU: apiresource.MustParse(envOr("T4_SESSION_REQUEST_CPU", "500m")), + corev1.ResourceCPU: apiresource.MustParse(envOr("T4_SESSION_REQUEST_CPU", "500m")), corev1.ResourceMemory: apiresource.MustParse(envOr("T4_SESSION_REQUEST_MEMORY", "1Gi")), }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: apiresource.MustParse(envOr("T4_SESSION_LIMIT_CPU", "4")), + corev1.ResourceCPU: apiresource.MustParse(envOr("T4_SESSION_LIMIT_CPU", "4")), corev1.ResourceMemory: apiresource.MustParse(envOr("T4_SESSION_LIMIT_MEMORY", "8Gi")), }, }, SharedMemorySize: apiresource.MustParse(envOr("T4_SESSION_SHM_SIZE", "1Gi")), - TemporarySize: apiresource.MustParse(envOr("T4_SESSION_TEMPORARY_SIZE", "2Gi")), + TemporarySize: apiresource.MustParse(envOr("T4_SESSION_TEMPORARY_SIZE", "2Gi")), }).SetupWithManager(manager); err != nil { - ctrl.Log.Error(err, "unable to register T4Session controller"); os.Exit(1) + ctrl.Log.Error(err, "unable to register T4Session controller") + os.Exit(1) + } + if err := manager.AddHealthzCheck("healthz", healthz.Ping); err != nil { + ctrl.Log.Error(err, "unable to install health check") + os.Exit(1) + } + if err := manager.AddReadyzCheck("readyz", healthz.Ping); err != nil { + ctrl.Log.Error(err, "unable to install readiness check") + os.Exit(1) + } + if err := manager.Start(ctrl.SetupSignalHandler()); err != nil { + ctrl.Log.Error(err, "controller manager stopped") + os.Exit(1) } - if err := manager.AddHealthzCheck("healthz", healthz.Ping); err != nil { ctrl.Log.Error(err, "unable to install health check"); os.Exit(1) } - if err := manager.AddReadyzCheck("readyz", healthz.Ping); err != nil { ctrl.Log.Error(err, "unable to install readiness check"); os.Exit(1) } - if err := manager.Start(ctrl.SetupSignalHandler()); err != nil { ctrl.Log.Error(err, "controller manager stopped"); os.Exit(1) } } func splitNonempty(value string) []string { var result []string for _, item := range strings.Split(value, ",") { - if item = strings.TrimSpace(item); item != "" { result = append(result, item) } + if item = strings.TrimSpace(item); item != "" { + result = append(result, item) + } } return result } +func sessionServiceAccountNames() (string, string) { + return envOr("T4_SESSION_SERVICE_ACCOUNT", controllers.DefaultSessionServiceAccount), + envOr("T4_CLUSTER_SERVER_SERVICE_ACCOUNT", controllers.DefaultServerServiceAccount) +} + func envOr(name, fallback string) string { - if value := strings.TrimSpace(os.Getenv(name)); value != "" { return value } + if value := strings.TrimSpace(os.Getenv(name)); value != "" { + return value + } return fallback } diff --git a/packages/cluster-operator/cmd/manager/main_test.go b/packages/cluster-operator/cmd/manager/main_test.go index cc7416a3..8753c5c1 100644 --- a/packages/cluster-operator/cmd/manager/main_test.go +++ b/packages/cluster-operator/cmd/manager/main_test.go @@ -3,6 +3,8 @@ package main import ( "testing" "time" + + "github.com/LycaonLLC/t4-code/packages/cluster-operator/controllers" ) func TestManagerUsesLeaseLeaderElection(t *testing.T) { @@ -23,3 +25,18 @@ func TestManagerUsesLeaseLeaderElection(t *testing.T) { t.Fatalf("unsafe leader-election timing: lease=%v renew=%v retry=%v", *options.LeaseDuration, *options.RenewDeadline, *options.RetryPeriod) } } + +func TestManagerConfiguresDedicatedSessionAndServerServiceAccounts(t *testing.T) { + t.Setenv("T4_SESSION_SERVICE_ACCOUNT", "release-session") + t.Setenv("T4_CLUSTER_SERVER_SERVICE_ACCOUNT", "release-server") + session, server := sessionServiceAccountNames() + if session != "release-session" || server != "release-server" { + t.Fatalf("ServiceAccounts = %q/%q", session, server) + } + t.Setenv("T4_SESSION_SERVICE_ACCOUNT", "") + t.Setenv("T4_CLUSTER_SERVER_SERVICE_ACCOUNT", "") + session, server = sessionServiceAccountNames() + if session != controllers.DefaultSessionServiceAccount || server != controllers.DefaultServerServiceAccount { + t.Fatalf("default ServiceAccounts = %q/%q", session, server) + } +} diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index a056e642..15080a33 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -169,8 +169,8 @@ func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) workspace.Status.Phase = clusterv1alpha1.InfrastructurePending pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, - Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending}, } session := testSession() c := fake.NewClientBuilder().WithScheme(scheme). @@ -203,7 +203,7 @@ func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) } pod := pods.Items[0] if pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { - t.Fatal("session pod must not mount a service account token") + t.Fatal("session pod must disable automatic ServiceAccount token mounting") } if pod.Spec.Containers[0].Image != r.RuntimeImage { t.Fatalf("controller did not use administrator-owned runtime image: %q", pod.Spec.Containers[0].Image) @@ -214,6 +214,45 @@ func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) if !hasMount(pod.Spec.Containers[0].VolumeMounts, "workspace", "/workspace") || !hasMount(pod.Spec.Containers[0].VolumeMounts, "shared-memory", "/dev/shm") { t.Fatalf("session mounts = %#v", pod.Spec.Containers[0].VolumeMounts) } + if pod.Spec.ServiceAccountName != controllers.DefaultSessionServiceAccount { + t.Fatalf("session ServiceAccount = %q", pod.Spec.ServiceAccountName) + } + serverIdentity := "" + for _, env := range pod.Spec.Containers[0].Env { + if env.ValueFrom != nil && env.ValueFrom.SecretKeyRef != nil { + t.Fatalf("session environment references a credential Secret: %#v", env) + } + if env.Name == "T4_CLUSTER_SERVER_SERVICE_ACCOUNT" { + serverIdentity = env.Value + } + } + if serverIdentity != controllers.DefaultServerServiceAccount { + t.Fatalf("expected server ServiceAccount = %q", serverIdentity) + } + if !hasMount(pod.Spec.Containers[0].VolumeMounts, "kubernetes-api-access", "/var/run/secrets/kubernetes.io/serviceaccount") { + t.Fatal("explicit Kubernetes reviewer projection is not mounted") + } + var projection *corev1.ProjectedVolumeSource + for _, volume := range pod.Spec.Volumes { + if volume.Name == "kubernetes-api-access" { + projection = volume.Projected + } + } + if projection == nil || len(projection.Sources) != 3 { + t.Fatalf("Kubernetes reviewer projection = %#v", projection) + } + serviceToken := projection.Sources[0].ServiceAccountToken + if serviceToken == nil || serviceToken.Audience != controllers.KubernetesAPIAudience || serviceToken.ExpirationSeconds == nil || *serviceToken.ExpirationSeconds != controllers.SessionReviewerTokenExpirationSeconds || serviceToken.Path != "token" { + t.Fatalf("reviewer token projection = %#v", serviceToken) + } + clusterCA := projection.Sources[1].ConfigMap + if clusterCA == nil || clusterCA.Name != "kube-root-ca.crt" || len(clusterCA.Items) != 1 || clusterCA.Items[0].Key != "ca.crt" || clusterCA.Items[0].Path != "ca.crt" { + t.Fatalf("cluster CA projection = %#v", clusterCA) + } + namespace := projection.Sources[2].DownwardAPI + if namespace == nil || len(namespace.Items) != 1 || namespace.Items[0].Path != "namespace" || namespace.Items[0].FieldRef == nil || namespace.Items[0].FieldRef.FieldPath != "metadata.namespace" { + t.Fatalf("namespace projection = %#v", namespace) + } } func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { @@ -253,13 +292,13 @@ func testScheme(t *testing.T) *runtime.Scheme { func testHost() *clusterv1alpha1.T4ClusterHost { return &clusterv1alpha1.T4ClusterHost{ ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team", UID: "host-uid"}, - Spec: clusterv1alpha1.T4ClusterHostSpec{StorageClassName: "portable-rwx", RuntimeProfiles: []string{"default"}}, + Spec: clusterv1alpha1.T4ClusterHostSpec{StorageClassName: "portable-rwx", RuntimeProfiles: []string{"default"}}, } } func rwxStorageClass() *storagev1.StorageClass { return &storagev1.StorageClass{ - ObjectMeta: metav1.ObjectMeta{Name: "portable-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, + ObjectMeta: metav1.ObjectMeta{Name: "portable-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi", } } @@ -276,7 +315,7 @@ func testWorkspace(policy clusterv1alpha1.RetentionPolicy) *clusterv1alpha1.T4Wo func testSession() *clusterv1alpha1.T4Session { return &clusterv1alpha1.T4Session{ ObjectMeta: metav1.ObjectMeta{Name: "session-a", Namespace: "team", Generation: 2}, - Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a", Title: "Session A", RuntimeProfile: "default", GUIEnabled: true}, + Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a", Title: "Session A", RuntimeProfile: "default", GUIEnabled: true}, } } @@ -332,4 +371,3 @@ func hasMount(mounts []corev1.VolumeMount, name, path string) bool { } func ptr[T any](value T) *T { return &value } - diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 2d88948b..ac0738e7 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -9,8 +9,8 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - apiresource "k8s.io/apimachinery/pkg/api/resource" meta "k8s.io/apimachinery/pkg/api/meta" + apiresource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -22,17 +22,23 @@ import ( clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" ) -const DefaultInternalAuthSecret = "t4-cluster-internal-auth" +const ( + DefaultSessionServiceAccount = "t4-cluster-session" + DefaultServerServiceAccount = "t4-cluster-server" + KubernetesAPIAudience = "https://kubernetes.default.svc" + SessionReviewerTokenExpirationSeconds int64 = 3600 +) type SessionReconciler struct { client.Client - Scheme *runtime.Scheme - RuntimeImage string - InternalAuthSecret string - ExcludedNodeNames []string - Resources corev1.ResourceRequirements - SharedMemorySize apiresource.Quantity - TemporarySize apiresource.Quantity + Scheme *runtime.Scheme + RuntimeImage string + SessionServiceAccountName string + ServerServiceAccountName string + ExcludedNodeNames []string + Resources corev1.ResourceRequirements + SharedMemorySize apiresource.Quantity + TemporarySize apiresource.Quantity } func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { @@ -40,9 +46,13 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.Get(ctx, request.NamespacedName, &session); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } - if !session.DeletionTimestamp.IsZero() { return r.reconcileDelete(ctx, &session) } + if !session.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, &session) + } if controllerutil.AddFinalizer(&session, clusterv1alpha1.SessionFinalizer) { - if err := r.Update(ctx, &session); err != nil { return ctrl.Result{}, err } + if err := r.Update(ctx, &session); err != nil { + return ctrl.Result{}, err + } } if r.RuntimeImage == "" { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", "RuntimeImageMissing", "administrator-owned session runtime image is not configured") @@ -50,7 +60,9 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) var host clusterv1alpha1.T4ClusterHost if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.HostRef}, &host); err != nil { - if apierrors.IsNotFound(err) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") } + if apierrors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") + } return ctrl.Result{}, err } if !hasString(host.Spec.RuntimeProfiles, session.Spec.RuntimeProfile) { @@ -58,7 +70,9 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } var workspace clusterv1alpha1.T4Workspace if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.WorkspaceRef}, &workspace); err != nil { - if apierrors.IsNotFound(err) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") } + if apierrors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") + } return ctrl.Result{}, err } if workspace.Spec.HostRef != session.Spec.HostRef { @@ -69,7 +83,9 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } var pvc corev1.PersistentVolumeClaim if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: workspace.Status.PVCName}, &pvc); err != nil { - if apierrors.IsNotFound(err) { return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") } + if apierrors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") + } return ctrl.Result{}, err } if pvc.Status.Phase != corev1.ClaimBound || !pvcHasRWX(&pvc) { @@ -79,33 +95,47 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) serviceName := SessionServiceName(&session) podName := SessionPodName(&session) labels := map[string]string{ - "app.kubernetes.io/name": "t4-session-runtime", + "app.kubernetes.io/name": "t4-session-runtime", "app.kubernetes.io/part-of": "t4-cluster", - "cluster.t4.dev/session": podName, + "cluster.t4.dev/session": podName, } var service corev1.Service if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: serviceName}, &service); apierrors.IsNotFound(err) { service = corev1.Service{ ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: session.Namespace, Labels: labels}, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, + Type: corev1.ServiceTypeClusterIP, Selector: labels, - Ports: []corev1.ServicePort{{Name: "host", Port: 8787, TargetPort: intstr.FromString("host"), Protocol: corev1.ProtocolTCP}}, + Ports: []corev1.ServicePort{{Name: "host", Port: 8787, TargetPort: intstr.FromString("host"), Protocol: corev1.ProtocolTCP}}, }, } - if err := controllerutil.SetControllerReference(&session, &service, r.Scheme); err != nil { return ctrl.Result{}, err } - if err := r.Create(ctx, &service); err != nil && !apierrors.IsAlreadyExists(err) { return ctrl.Result{}, err } - } else if err != nil { return ctrl.Result{}, err } + if err := controllerutil.SetControllerReference(&session, &service, r.Scheme); err != nil { + return ctrl.Result{}, err + } + if err := r.Create(ctx, &service); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + } else if err != nil { + return ctrl.Result{}, err + } var pod corev1.Pod if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: podName}, &pod); apierrors.IsNotFound(err) { pod = r.desiredPod(&session, workspace.Status.PVCName, podName, labels) - if err := controllerutil.SetControllerReference(&session, &pod, r.Scheme); err != nil { return ctrl.Result{}, err } - if err := r.Create(ctx, &pod); err != nil && !apierrors.IsAlreadyExists(err) { return ctrl.Result{}, err } - } else if err != nil { return ctrl.Result{}, err } + if err := controllerutil.SetControllerReference(&session, &pod, r.Scheme); err != nil { + return ctrl.Result{}, err + } + if err := r.Create(ctx, &pod); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + } else if err != nil { + return ctrl.Result{}, err + } original := session.Status - if session.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) } + if session.Status.Conditions != nil { + original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } session.Status.ObservedGeneration = session.Generation session.Status.PodName = podName session.Status.ServiceName = serviceName @@ -121,9 +151,13 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "PodStarting", "session infrastructure pod is starting", session.Generation)) } if !reflect.DeepEqual(original, session.Status) { - if err := r.Status().Update(ctx, &session); err != nil { return ctrl.Result{}, err } + if err := r.Status().Update(ctx, &session); err != nil { + return ctrl.Result{}, err + } + } + if !podReady(&pod) { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } - if !podReady(&pod) { return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } return ctrl.Result{}, nil } @@ -134,9 +168,13 @@ func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcNa fsGroup := int64(10001) grace := int64(45) shmSize := r.SharedMemorySize.DeepCopy() - if shmSize.IsZero() { shmSize = apiresource.MustParse("1Gi") } + if shmSize.IsZero() { + shmSize = apiresource.MustParse("1Gi") + } temporarySize := r.TemporarySize.DeepCopy() - if temporarySize.IsZero() { temporarySize = apiresource.MustParse("2Gi") } + if temporarySize.IsZero() { + temporarySize = apiresource.MustParse("2Gi") + } resources := r.Resources.DeepCopy() if resources.Requests == nil { resources.Requests = corev1.ResourceList{corev1.ResourceCPU: apiresource.MustParse("500m"), corev1.ResourceMemory: apiresource.MustParse("1Gi")} @@ -144,16 +182,27 @@ func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcNa if resources.Limits == nil { resources.Limits = corev1.ResourceList{corev1.ResourceCPU: apiresource.MustParse("4"), corev1.ResourceMemory: apiresource.MustParse("8Gi")} } - internalSecret := r.InternalAuthSecret - if internalSecret == "" { internalSecret = DefaultInternalAuthSecret } + sessionServiceAccount := r.SessionServiceAccountName + if sessionServiceAccount == "" { + sessionServiceAccount = DefaultSessionServiceAccount + } + serverServiceAccount := r.ServerServiceAccountName + if serverServiceAccount == "" { + serverServiceAccount = DefaultServerServiceAccount + } excluded := r.ExcludedNodeNames - if len(excluded) == 0 { excluded = []string{"k3s-worker-02"} } + if len(excluded) == 0 { + excluded = []string{"k3s-worker-02"} + } stateID := strings.TrimPrefix(podName, "t4-session-") container := corev1.Container{ Name: "session-runtime", Image: r.RuntimeImage, ImagePullPolicy: corev1.PullIfNotPresent, Ports: []corev1.ContainerPort{{Name: "host", ContainerPort: 8787, Protocol: corev1.ProtocolTCP}}, Env: []corev1.EnvVar{ - {Name: "T4_CLUSTER_INTERNAL_TOKEN", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: internalSecret}, Key: "token"}}}, + {Name: "T4_CLUSTER_SERVER_SERVICE_ACCOUNT", Value: serverServiceAccount}, + {Name: "T4_KUBERNETES_TOKEN_PATH", Value: "/var/run/secrets/kubernetes.io/serviceaccount/token"}, + {Name: "T4_KUBERNETES_CA_PATH", Value: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"}, + {Name: "T4_KUBERNETES_NAMESPACE_PATH", Value: "/var/run/secrets/kubernetes.io/serviceaccount/namespace"}, {Name: "T4_SESSION_NAME", Value: session.Name}, {Name: "T4_WORKSPACE_ROOT", Value: "/workspace"}, {Name: "T4_AUTHORITY_STATE_DIR", Value: "/workspace/.t4/sessions/" + stateID + "/authority"}, @@ -166,21 +215,30 @@ func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcNa {Name: "runtime", MountPath: "/run"}, {Name: "temporary", MountPath: "/tmp"}, {Name: "shared-memory", MountPath: "/dev/shm"}, + {Name: "kubernetes-api-access", MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", ReadOnly: true}, }, SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: &falseValue, ReadOnlyRootFilesystem: &trueValue, RunAsNonRoot: &trueValue, RunAsUser: &runAsUser, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, }, - Resources: *resources, - StartupProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString("host")}}, FailureThreshold: 30, PeriodSeconds: 2, TimeoutSeconds: 1}, + Resources: *resources, + StartupProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString("host")}}, FailureThreshold: 30, PeriodSeconds: 2, TimeoutSeconds: 1}, ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString("host")}}, FailureThreshold: 3, PeriodSeconds: 5, TimeoutSeconds: 2}, - LivenessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString("host")}}, FailureThreshold: 3, PeriodSeconds: 10, TimeoutSeconds: 2}, + LivenessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString("host")}}, FailureThreshold: 3, PeriodSeconds: 10, TimeoutSeconds: 2}, } volumes := []corev1.Volume{ {Name: "workspace", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcName}}}, {Name: "runtime", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory, SizeLimit: ptrQuantity(apiresource.MustParse("128Mi"))}}}, {Name: "temporary", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &temporarySize}}}, {Name: "shared-memory", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory, SizeLimit: &shmSize}}}, + {Name: "kubernetes-api-access", VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: ptr(int32(0440)), + Sources: []corev1.VolumeProjection{ + {ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Audience: KubernetesAPIAudience, ExpirationSeconds: ptr(SessionReviewerTokenExpirationSeconds), Path: "token"}}, + {ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "kube-root-ca.crt"}, Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}}}, + {DownwardAPI: &corev1.DownwardAPIProjection{Items: []corev1.DownwardAPIVolumeFile{{Path: "namespace", FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}}, + }, + }}}, } if session.Spec.InitialPromptSecretRef != nil { volumes = append(volumes, corev1.Volume{Name: "initial-prompt", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: session.Spec.InitialPromptSecretRef.Name, Items: []corev1.KeyToPath{{Key: "prompt", Path: "prompt", Mode: ptr(int32(0440))}}}}}) @@ -190,25 +248,32 @@ func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcNa return corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: session.Namespace, Labels: labels}, Spec: corev1.PodSpec{ - AutomountServiceAccountToken: &falseValue, - EnableServiceLinks: &falseValue, + AutomountServiceAccountToken: &falseValue, + ServiceAccountName: sessionServiceAccount, + EnableServiceLinks: &falseValue, TerminationGracePeriodSeconds: &grace, - SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &trueValue, RunAsUser: &runAsUser, FSGroup: &fsGroup, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, - Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "kubernetes.io/hostname", Operator: corev1.NodeSelectorOpNotIn, Values: excluded}}}}}}}, - Containers: []corev1.Container{container}, Volumes: volumes, + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &trueValue, RunAsUser: &runAsUser, FSGroup: &fsGroup, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, + Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "kubernetes.io/hostname", Operator: corev1.NodeSelectorOpNotIn, Values: excluded}}}}}}}, + Containers: []corev1.Container{container}, Volumes: volumes, }, } } func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *clusterv1alpha1.T4Session) (ctrl.Result, error) { - if !controllerutil.ContainsFinalizer(session, clusterv1alpha1.SessionFinalizer) { return ctrl.Result{}, nil } + if !controllerutil.ContainsFinalizer(session, clusterv1alpha1.SessionFinalizer) { + return ctrl.Result{}, nil + } originalStatus := session.Status - if session.Status.Conditions != nil { originalStatus.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) } + if session.Status.Conditions != nil { + originalStatus.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } session.Status.ObservedGeneration = session.Generation session.Status.Phase = clusterv1alpha1.InfrastructureTerminating meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "Terminating", "session infrastructure is terminating", session.Generation)) if !reflect.DeepEqual(originalStatus, session.Status) { - if err := r.Status().Update(ctx, session); err != nil { return ctrl.Result{}, err } + if err := r.Status().Update(ctx, session); err != nil { + return ctrl.Result{}, err + } } remaining := false for _, object := range []client.Object{ @@ -219,22 +284,32 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste if err == nil { remaining = true if object.GetDeletionTimestamp().IsZero() { - if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } + if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } } - } else if !apierrors.IsNotFound(err) { return ctrl.Result{}, err } + } else if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + if remaining { + return ctrl.Result{RequeueAfter: time.Second}, nil } - if remaining { return ctrl.Result{RequeueAfter: time.Second}, nil } controllerutil.RemoveFinalizer(session, clusterv1alpha1.SessionFinalizer) return ctrl.Result{}, r.Update(ctx, session) } func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, conditionType, reason, message string) error { original := session.Status - if session.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) } + if session.Status.Conditions != nil { + original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } session.Status.ObservedGeneration = session.Generation session.Status.Phase = clusterv1alpha1.InfrastructureFailed meta.SetStatusCondition(&session.Status.Conditions, condition(conditionType, metav1.ConditionFalse, reason, message, session.Generation)) - if reflect.DeepEqual(original, session.Status) { return nil } + if reflect.DeepEqual(original, session.Status) { + return nil + } return r.Status().Update(ctx, session) } @@ -248,10 +323,12 @@ func (r *SessionReconciler) SetupWithManager(manager ctrl.Manager) error { func podReady(pod *corev1.Pod) bool { for _, item := range pod.Status.Conditions { - if item.Type == corev1.PodReady { return item.Status == corev1.ConditionTrue } + if item.Type == corev1.PodReady { + return item.Status == corev1.ConditionTrue + } } return false } func ptrQuantity(value apiresource.Quantity) *apiresource.Quantity { return &value } -func ptr[T any](value T) *T { return &value } +func ptr[T any](value T) *T { return &value } diff --git a/packages/cluster-server/src/config.ts b/packages/cluster-server/src/config.ts index 958680fa..9b4f3f71 100644 --- a/packages/cluster-server/src/config.ts +++ b/packages/cluster-server/src/config.ts @@ -1,4 +1,5 @@ -import { readFile } from "node:fs/promises"; +import { open } from "node:fs/promises"; +import { isAbsolute } from "node:path"; import { isIP } from "node:net"; const DNS_LABEL = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; @@ -12,7 +13,8 @@ export interface ClusterServerConfig { readonly kubernetesBaseUrl: string; readonly kubernetesTokenPath: string; readonly kubernetesCaPath: string; - readonly internalToken: string; + readonly identityTokenPath: string; + readonly serverServiceAccountName: string; readonly trustedProxyAddresses: readonly string[]; readonly trustedProxyCidrs: readonly string[]; readonly woodpecker?: { @@ -35,6 +37,10 @@ function port(value: string | undefined, fallback: number, name: string): number if (!Number.isSafeInteger(result) || result < 1 || result > 65_535) throw new Error(`${name} is invalid`); return result; } +function absolutePath(value: string, name: string): string { + if (!isAbsolute(value)) throw new Error(`${name} must be absolute`); + return value; +} function repositories(value: string): Readonly> { if (new TextEncoder().encode(value).byteLength > 65_536) throw new Error("T4_WOODPECKER_REPOSITORIES exceeds limit"); let input: unknown; @@ -101,8 +107,8 @@ export function clusterServerConfigFromEnv(env: Readonly 16_384) throw new Error("T4_CLUSTER_INTERNAL_TOKEN is invalid"); + const identityTokenPath = absolutePath(required(env, "T4_CLUSTER_IDENTITY_TOKEN_FILE"), "T4_CLUSTER_IDENTITY_TOKEN_FILE"); + const serverServiceAccountName = dns(required(env, "T4_CLUSTER_SERVER_SERVICE_ACCOUNT"), "T4_CLUSTER_SERVER_SERVICE_ACCOUNT"); const woodpeckerValues = [env.T4_WOODPECKER_BASE_URL, env.T4_WOODPECKER_TOKEN, env.T4_WOODPECKER_REPOSITORIES]; if (woodpeckerValues.some(Boolean) && !woodpeckerValues.every(Boolean)) throw new Error("Woodpecker configuration must be complete"); return { @@ -115,9 +121,10 @@ export function clusterServerConfigFromEnv(env: Readonly { + const file = await open(path, "r"); + try { + const metadata = await file.stat(); + if (!metadata.isFile() || metadata.size < 1 || metadata.size > maximumBytes) throw new Error(`${description} file is invalid`); + const value = await file.readFile("utf8"); + if (new TextEncoder().encode(value).byteLength > maximumBytes) throw new Error(`${description} file is invalid`); + return value; + } finally { + await file.close(); + } +} +export async function readClusterIdentityToken(path: string): Promise { + const token = (await readBoundedRegularFile(path, 16_384, "cluster identity token")).trim(); + if (new TextEncoder().encode(token).byteLength < 32 || /\s/u.test(token)) throw new Error("cluster identity token file is invalid"); + return token; +} export async function loadKubernetesCredentials(config: ClusterServerConfig): Promise<{ token: string; ca: string }> { - const [token, ca] = await Promise.all([readFile(config.kubernetesTokenPath, "utf8"), readFile(config.kubernetesCaPath, "utf8")]); - if (!token.trim() || token.length > 16_384) throw new Error("Kubernetes token file is invalid"); - if (!ca.includes("BEGIN CERTIFICATE") || ca.length > 1024 * 1024) throw new Error("Kubernetes CA file is invalid"); - return { token: token.trim(), ca }; + const [rawToken, ca] = await Promise.all([ + readBoundedRegularFile(config.kubernetesTokenPath, 16_384, "Kubernetes token"), + readBoundedRegularFile(config.kubernetesCaPath, 1024 * 1024, "Kubernetes CA"), + ]); + const token = rawToken.trim(); + if (!token || /\s/u.test(token)) throw new Error("Kubernetes token file is invalid"); + if (!ca.includes("BEGIN CERTIFICATE")) throw new Error("Kubernetes CA file is invalid"); + return { token, ca }; } diff --git a/packages/cluster-server/src/gateway.ts b/packages/cluster-server/src/gateway.ts index c853f13d..ac75aaed 100644 --- a/packages/cluster-server/src/gateway.ts +++ b/packages/cluster-server/src/gateway.ts @@ -51,7 +51,6 @@ export interface ClusterGatewayOptions { readonly projection: ClusterInfrastructureProjection; readonly connector: PodHostConnector; readonly mutations: GatewayMutationBackend; - readonly internalToken: string; readonly ciProvider?: CiProvider; readonly appserverVersion?: string; readonly appserverBuild?: string; @@ -104,7 +103,6 @@ export class ClusterGateway { this.#ci = options.ciProvider; this.#version = options.appserverVersion ?? "0.1.30"; this.#build = options.appserverBuild ?? "cluster"; - if (options.internalToken.length < 32) throw new Error("cluster internal token is invalid"); } get connectionCount(): number { return this.#connections.size; } get draining(): boolean { return this.#draining; } diff --git a/packages/cluster-server/src/kubernetes-client.ts b/packages/cluster-server/src/kubernetes-client.ts index b60cc502..fc4063d8 100644 --- a/packages/cluster-server/src/kubernetes-client.ts +++ b/packages/cluster-server/src/kubernetes-client.ts @@ -1,4 +1,5 @@ -import { createHash } from "node:crypto"; +import { createHash, timingSafeEqual } from "node:crypto"; +import { isAbsolute } from "node:path"; import type { ClusterSessionCreateArguments, ClusterWorkspaceCreateArguments, @@ -10,9 +11,12 @@ import { type KubernetesResource, type KubernetesWatchEvent, } from "./kubernetes-projection.ts"; +import { readBoundedRegularFile } from "./config.ts"; const API_PREFIX = "/apis/cluster.t4.dev/v1alpha1"; +const TOKEN_REVIEW_PATH = "/apis/authentication.k8s.io/v1/tokenreviews"; const MAX_WATCH_LINE_BYTES = 1024 * 1024; +export const CLUSTER_INTERNAL_AUDIENCE = "t4-cluster-internal"; export interface KubernetesApiClientOptions { readonly baseUrl: string; @@ -180,6 +184,84 @@ export class KubernetesApiClient { } } +export interface KubernetesTokenReviewerOptions { + readonly baseUrl: string; + readonly tokenPath: string; + readonly caPath: string; + readonly namespacePath: string; + readonly serverServiceAccountName: string; + readonly timeoutMs?: number; + readonly fetch?: typeof globalThis.fetch; +} + +/** Validates a projected server identity through the Kubernetes authentication authority. */ +export class KubernetesTokenReviewer { + readonly #baseUrl: string; + readonly #tokenPath: string; + readonly #caPath: string; + readonly #namespacePath: string; + readonly #serverServiceAccountName: string; + readonly #timeoutMs: number; + readonly #fetch: typeof globalThis.fetch; + + constructor(options: KubernetesTokenReviewerOptions) { + this.#baseUrl = exactHttpsBase(options.baseUrl); + if (![options.tokenPath, options.caPath, options.namespacePath].every(isAbsolute)) throw new Error("Kubernetes projected credential paths must be absolute"); + this.#tokenPath = options.tokenPath; + this.#caPath = options.caPath; + this.#namespacePath = options.namespacePath; + if (!/^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u.test(options.serverServiceAccountName)) throw new Error("cluster server ServiceAccount name is invalid"); + this.#serverServiceAccountName = options.serverServiceAccountName; + this.#timeoutMs = options.timeoutMs ?? 5_000; + if (!Number.isSafeInteger(this.#timeoutMs) || this.#timeoutMs < 100 || this.#timeoutMs > 30_000) throw new Error("TokenReview timeout is invalid"); + this.#fetch = options.fetch ?? globalThis.fetch; + } + + async review(presentedToken: string): Promise { + try { + const presentedBytes = new TextEncoder().encode(presentedToken).byteLength; + if (presentedBytes < 32 || presentedBytes > 16_384 || /\s/u.test(presentedToken)) return false; + const [rawReviewerToken, ca, rawNamespace] = await Promise.all([ + readBoundedRegularFile(this.#tokenPath, 16_384, "Kubernetes reviewer token"), + readBoundedRegularFile(this.#caPath, 1024 * 1024, "Kubernetes CA"), + readBoundedRegularFile(this.#namespacePath, 256, "Kubernetes namespace"), + ]); + const reviewerToken = rawReviewerToken.trim(); + const namespace = safeNamespace(rawNamespace.trim()); + if (!reviewerToken || /\s/u.test(reviewerToken) || !ca.includes("BEGIN CERTIFICATE")) return false; + const headers = new Headers({ accept: "application/json", authorization: `Bearer ${reviewerToken}`, "content-type": "application/json" }); + const request = { + method: "POST", + headers, + body: JSON.stringify({ + apiVersion: "authentication.k8s.io/v1", + kind: "TokenReview", + spec: { token: presentedToken, audiences: [CLUSTER_INTERNAL_AUDIENCE] }, + }), + signal: AbortSignal.timeout(this.#timeoutMs), + tls: { ca }, + } as RequestInit & { tls: { ca: string } }; + const response = await this.#fetch(`${this.#baseUrl}${TOKEN_REVIEW_PATH}`, request); + if (!response.ok) return false; + let body: unknown; + try { body = await response.json(); } catch { return false; } + const root = object(body); + if (root.apiVersion !== "authentication.k8s.io/v1" || root.kind !== "TokenReview") return false; + const status = object(root.status); + if (status.authenticated !== true || Object.hasOwn(status, "error")) return false; + const user = object(status.user); + if (typeof user.username !== "string") return false; + const expectedUsername = `system:serviceaccount:${namespace}:${this.#serverServiceAccountName}`; + const expectedBytes = Buffer.from(expectedUsername, "utf8"); + const actualBytes = Buffer.from(user.username, "utf8"); + if (expectedBytes.length !== actualBytes.length || !timingSafeEqual(expectedBytes, actualBytes)) return false; + return Array.isArray(status.audiences) && status.audiences.length === 1 && status.audiences[0] === CLUSTER_INTERNAL_AUDIENCE; + } catch { + return false; + } + } +} + export interface KubernetesGatewayMutationBackendOptions { readonly client: KubernetesApiClient; readonly hostRef: string; diff --git a/packages/cluster-server/src/main.ts b/packages/cluster-server/src/main.ts index bf08288b..39ce43da 100755 --- a/packages/cluster-server/src/main.ts +++ b/packages/cluster-server/src/main.ts @@ -33,6 +33,7 @@ export async function runClusterServer(env: Readonly(); const stop = (): void => stopped.resolve(); let authority: SessionAuthorityRunner | undefined; @@ -40,7 +41,7 @@ export async function runClusterServer(env: Readonly void, onClose?: () => void): Promise; } export interface WebSocketPodHostConnectorOptions { - readonly internalToken: string; + readonly identityTokenFile: string; readonly openTimeoutMs?: number; readonly webSocketFactory?: (url: string) => WebSocket; } export class WebSocketPodHostConnector implements PodHostConnector { - readonly #token: string; + readonly #identityTokenFile: string; readonly #timeoutMs: number; readonly #factory: (url: string) => WebSocket; constructor(options: WebSocketPodHostConnectorOptions) { - this.#token = canonicalInternalDeviceToken(options.internalToken); + this.#identityTokenFile = options.identityTokenFile; this.#timeoutMs = options.openTimeoutMs ?? 10_000; this.#factory = options.webSocketFactory ?? (url => new WebSocket(url)); } @@ -49,15 +49,24 @@ export class WebSocketPodHostConnector implements PodHostConnector { if (!settled) { settled = true; socket.close(1013, "upstream timeout"); reject(new Error("pod host handshake timed out")); } }, this.#timeoutMs); socket.addEventListener("open", () => { - socket.send(JSON.stringify({ - v: "omp-app/1", type: "hello", - protocol: { min: "omp-app/1", max: "omp-app/1" }, - client: { name: "cluster-server", version: "0.1.30", build: "cluster", platform: "linux" }, - requestedFeatures: PROTOCOL_FEATURES.filter(feature => feature !== "cluster.operator"), - savedCursors: [], - capabilities: { client: DEVICE_CAPABILITIES.filter(capability => capability !== "ci.trigger") }, - authentication: { deviceId: "cluster-server", deviceToken: this.#token }, - })); + void readClusterIdentityToken(this.#identityTokenFile).then(token => { + if (settled) return; + socket.send(JSON.stringify({ + v: "omp-app/1", type: "hello", + protocol: { min: "omp-app/1", max: "omp-app/1" }, + client: { name: "cluster-server", version: "0.1.30", build: "cluster", platform: "linux" }, + requestedFeatures: PROTOCOL_FEATURES.filter(feature => feature !== "cluster.operator"), + savedCursors: [], + capabilities: { client: DEVICE_CAPABILITIES.filter(capability => capability !== "ci.trigger") }, + authentication: { deviceId: "cluster-server", deviceToken: token }, + })); + }, () => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.close(1011, "identity unavailable"); + reject(new Error("pod host identity is unavailable")); + }); }); socket.addEventListener("message", event => { try { diff --git a/packages/cluster-server/src/session-host-main.ts b/packages/cluster-server/src/session-host-main.ts index 57bca589..72181224 100755 --- a/packages/cluster-server/src/session-host-main.ts +++ b/packages/cluster-server/src/session-host-main.ts @@ -11,6 +11,7 @@ import { } from "@t4-code/host-service"; import { hostId } from "@t4-code/host-wire"; import { ClusterInternalRemotePolicy, sessionHostConfigFromEnv, type SessionHostConfig } from "./session-host-policy.ts"; +import { KubernetesTokenReviewer } from "./kubernetes-client.ts"; const OMP_VERSION = "17.0.5"; const OMP_COMMIT = "8476f4451ed95c5d5401785d279a93d3c659fac4"; @@ -68,7 +69,13 @@ export async function runSessionHost( rpcChildInvocation: { executable: config.ompExecutable, prefixArgv: [] }, }; const policy = new ClusterInternalRemotePolicy({ - token: config.internalToken, + reviewer: new KubernetesTokenReviewer({ + baseUrl: config.kubernetesBaseUrl, + tokenPath: config.kubernetesTokenPath, + caPath: config.kubernetesCaPath, + namespacePath: config.kubernetesNamespacePath, + serverServiceAccountName: config.serverServiceAccountName, + }), supportedCapabilities: appserverSupportedCapabilities(base), supportedFeatures: appserverSupportedFeatures(base, true), }); diff --git a/packages/cluster-server/src/session-host-policy.ts b/packages/cluster-server/src/session-host-policy.ts index 7d2f2ad8..f99a636e 100644 --- a/packages/cluster-server/src/session-host-policy.ts +++ b/packages/cluster-server/src/session-host-policy.ts @@ -1,6 +1,6 @@ -import { createHash, timingSafeEqual } from "node:crypto"; import { isAbsolute } from "node:path"; import { + decodeClientFrame, requiredCapability, type ClientFrame, type HelloFrame, @@ -13,41 +13,59 @@ import type { import type { RemoteConnection } from "@t4-code/host-service"; const SESSION_NAME = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; +const INTERNAL_TOKEN_PLACEHOLDER = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; +const MAX_PROJECTED_TOKEN_BYTES = 16_384; interface ConnectionGrant { capabilities: Set; features: Set; } +export interface ClusterIdentityReviewer { + review(token: string): Promise; +} export interface ClusterInternalRemotePolicyOptions { - readonly token: string; + readonly reviewer: ClusterIdentityReviewer; readonly supportedCapabilities: readonly string[]; readonly supportedFeatures: readonly string[]; } -/** Converts the mounted secret to the canonical 32-byte base64url token used by omp-app/1. */ -export function canonicalInternalDeviceToken(secret: string): string { - if (new TextEncoder().encode(secret).byteLength < 32 || secret.length > 16_384) throw new Error("cluster internal token is invalid"); - return createHash("sha256").update(secret, "utf8").digest("base64url"); +function projectedToken(value: unknown): string { + if (typeof value !== "string") throw new Error("cluster identity token is invalid"); + const bytes = new TextEncoder().encode(value).byteLength; + if (bytes < 32 || bytes > MAX_PROJECTED_TOKEN_BYTES || /\s/u.test(value)) throw new Error("cluster identity token is invalid"); + return value; } -function sameToken(expectedSecret: string, supplied: string): boolean { - try { - const expected = Buffer.from(canonicalInternalDeviceToken(expectedSecret)); - const canonical = /^[A-Za-z0-9_-]{43}$/u.test(supplied) ? supplied : canonicalInternalDeviceToken(supplied); - const actual = Buffer.from(canonical); - return expected.length === actual.length && timingSafeEqual(expected, actual); - } catch { return false; } +export function decodeClusterInternalClientFrame(input: unknown): ClientFrame { + if (!input || typeof input !== "object" || Array.isArray(input)) return decodeClientFrame(input); + const source = input as Record; + if (source.type !== "hello" || !source.authentication || typeof source.authentication !== "object" || Array.isArray(source.authentication)) + return decodeClientFrame(input); + const authentication = source.authentication as Record; + const token = projectedToken(authentication.deviceToken); + const decoded = decodeClientFrame({ ...source, authentication: { ...authentication, deviceToken: INTERNAL_TOKEN_PLACEHOLDER } }); + if (decoded.type !== "hello" || !decoded.authentication) throw new Error("cluster identity authentication is required"); + return { ...decoded, authentication: { ...decoded.authentication, deviceToken: token } }; } export class ClusterInternalRemotePolicy implements RemoteConnectionPolicy { - readonly #token: string; + readonly #reviewer: ClusterIdentityReviewer; readonly #capabilities: readonly string[]; readonly #features: readonly string[]; readonly #connections = new Map(); constructor(options: ClusterInternalRemotePolicyOptions) { - canonicalInternalDeviceToken(options.token); - this.#token = options.token; + this.#reviewer = options.reviewer; this.#capabilities = [...new Set(options.supportedCapabilities)].filter(value => value !== "ci.trigger"); this.#features = [...new Set(options.supportedFeatures)].filter(value => value !== "cluster.operator"); } + decodeClientFrame(input: unknown): ClientFrame { return decodeClusterInternalClientFrame(input); } async authenticate(connection: RemoteConnection, hello: HelloFrame): Promise { const authentication = hello.authentication; - if (connection.peer.identity.nodeId !== "cluster-server" || authentication?.deviceId !== "cluster-server" || !sameToken(this.#token, authentication.deviceToken)) { + if (connection.peer.identity.nodeId !== "cluster-server" || authentication?.deviceId !== "cluster-server") { + this.#connections.delete(connection.connectionId); + return { authenticated: false, authentication: "denied", grantedCapabilities: [], grantedFeatures: [] }; + } + try { + if (!await this.#reviewer.review(authentication.deviceToken)) { + this.#connections.delete(connection.connectionId); + return { authenticated: false, authentication: "denied", grantedCapabilities: [], grantedFeatures: [] }; + } + } catch { this.#connections.delete(connection.connectionId); return { authenticated: false, authentication: "denied", grantedCapabilities: [], grantedFeatures: [] }; } @@ -74,22 +92,50 @@ export class ClusterInternalRemotePolicy implements RemoteConnectionPolicy { } export interface SessionHostConfig { - readonly internalToken: string; + readonly kubernetesBaseUrl: string; + readonly kubernetesTokenPath: string; + readonly kubernetesCaPath: string; + readonly kubernetesNamespacePath: string; + readonly serverServiceAccountName: string; readonly sessionName: string; readonly ompExecutable: string; readonly stateRoot: string; readonly port: number; } +function required(env: Readonly>, name: string): string { + const value = env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} +function dns(value: string, name: string): string { + if (!SESSION_NAME.test(value)) throw new Error(`${name} is invalid`); + return value; +} +function absolutePath(value: string, name: string): string { + if (!isAbsolute(value)) throw new Error(`${name} must be absolute`); + return value; +} export function sessionHostConfigFromEnv(env: Readonly>): SessionHostConfig { - const internalToken = env.T4_CLUSTER_INTERNAL_TOKEN ?? ""; - canonicalInternalDeviceToken(internalToken); - const sessionName = env.T4_SESSION_NAME ?? ""; - if (!SESSION_NAME.test(sessionName)) throw new Error("T4_SESSION_NAME is invalid"); + const serviceHost = required(env, "KUBERNETES_SERVICE_HOST"); + const servicePort = Number(env.KUBERNETES_SERVICE_PORT_HTTPS ?? env.KUBERNETES_SERVICE_PORT ?? "443"); + if (!Number.isSafeInteger(servicePort) || servicePort < 1 || servicePort > 65_535) throw new Error("KUBERNETES_SERVICE_PORT is invalid"); + const serverServiceAccountName = dns(required(env, "T4_CLUSTER_SERVER_SERVICE_ACCOUNT"), "T4_CLUSTER_SERVER_SERVICE_ACCOUNT"); + const sessionName = dns(required(env, "T4_SESSION_NAME"), "T4_SESSION_NAME"); const ompExecutable = env.T4_OMP_EXECUTABLE ?? "/opt/t4/bin/omp"; if (!isAbsolute(ompExecutable)) throw new Error("T4_OMP_EXECUTABLE must be absolute"); const stateRoot = env.T4_SESSION_STATE_ROOT ?? `/workspace/.t4/sessions/${sessionName}`; if (stateRoot !== `/workspace/.t4/sessions/${sessionName}`) throw new Error("T4_SESSION_STATE_ROOT must select the isolated session directory"); const port = Number(env.T4_SESSION_HOST_PORT ?? "8787"); if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) throw new Error("T4_SESSION_HOST_PORT is invalid"); - return { internalToken, sessionName, ompExecutable, stateRoot, port }; + return { + kubernetesBaseUrl: `https://${serviceHost}:${servicePort}`, + kubernetesTokenPath: absolutePath(env.T4_KUBERNETES_TOKEN_PATH ?? "/var/run/secrets/kubernetes.io/serviceaccount/token", "T4_KUBERNETES_TOKEN_PATH"), + kubernetesCaPath: absolutePath(env.T4_KUBERNETES_CA_PATH ?? "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", "T4_KUBERNETES_CA_PATH"), + kubernetesNamespacePath: absolutePath(env.T4_KUBERNETES_NAMESPACE_PATH ?? "/var/run/secrets/kubernetes.io/serviceaccount/namespace", "T4_KUBERNETES_NAMESPACE_PATH"), + serverServiceAccountName, + sessionName, + ompExecutable, + stateRoot, + port, + }; } diff --git a/packages/cluster-server/test/config.test.ts b/packages/cluster-server/test/config.test.ts index f164dbae..d4f566c6 100644 --- a/packages/cluster-server/test/config.test.ts +++ b/packages/cluster-server/test/config.test.ts @@ -1,5 +1,8 @@ -import { describe, expect, test } from "vite-plus/test" -import { clusterServerConfigFromEnv } from "../src/config.ts"; +import { describe, expect, it } from "vite-plus/test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clusterServerConfigFromEnv, readClusterIdentityToken } from "../src/config.ts"; const BASE_ENV = { POD_NAMESPACE: "cluster-system", @@ -8,11 +11,40 @@ const BASE_ENV = { KUBERNETES_SERVICE_HOST: "10.96.0.1", KUBERNETES_SERVICE_PORT_HTTPS: "443", T4_CLUSTER_HOST_NAME: "default", - T4_CLUSTER_INTERNAL_TOKEN: "x".repeat(32), + T4_CLUSTER_IDENTITY_TOKEN_FILE: "/var/run/secrets/t4-cluster-identity/token", + T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "release-t4-cluster-server", } as const; +describe("cluster server configuration", () => { + it("selects the projected server identity independently from its Kubernetes watch credentials", () => { + const config = clusterServerConfigFromEnv(BASE_ENV); + expect(config).toMatchObject({ + identityTokenPath: "/var/run/secrets/t4-cluster-identity/token", + serverServiceAccountName: "release-t4-cluster-server", + kubernetesTokenPath: "/var/run/secrets/kubernetes.io/serviceaccount/token", + kubernetesCaPath: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + }); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_IDENTITY_TOKEN_FILE: "relative/token" })).toThrow("absolute"); + }); + + it("reads only a bounded regular projected identity file", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-cluster-identity-")); + try { + const path = join(directory, "token"); + const token = `header.payload.${"s".repeat(64)}`; + await writeFile(path, token, { mode: 0o400 }); + expect(await readClusterIdentityToken(path)).toBe(token); + await writeFile(path, "x".repeat(16_385), { mode: 0o400 }); + await expect(readClusterIdentityToken(path)).rejects.toThrow("invalid"); + await expect(readClusterIdentityToken(directory)).rejects.toThrow("invalid"); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); +}); + describe("trusted cluster gateway proxy sources", () => { - test("accepts bounded canonical IPv4 and IPv6 networks", () => { + it("accepts bounded canonical IPv4 and IPv6 networks", () => { const config = clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_ADDRESSES: "10.42.1.7,fd7a:115c:a1e0::1", @@ -22,12 +54,12 @@ describe("trusted cluster gateway proxy sources", () => { expect(config.trustedProxyCidrs).toEqual(["10.42.0.0/16", "fd7a:115c:a1e0::/48"]); }); - test("rejects CIDRs with host bits or non-canonical notation", () => { + it("rejects CIDRs with host bits or non-canonical notation", () => { expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: "10.42.1.7/16" })).toThrow(); expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: "fd7a:115c:a1e0:0::/48" })).toThrow(); }); - test("bounds the trusted CIDR list", () => { + it("bounds the trusted CIDR list", () => { const cidrs = Array.from({ length: 65 }, (_, index) => `10.${index}.0.0/16`).join(","); expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: cidrs })).toThrow(); }); diff --git a/packages/cluster-server/test/gateway.test.ts b/packages/cluster-server/test/gateway.test.ts index cbc17575..052e03e5 100644 --- a/packages/cluster-server/test/gateway.test.ts +++ b/packages/cluster-server/test/gateway.test.ts @@ -1,5 +1,13 @@ -import { describe, expect, test } from "vite-plus/test" -import { hostId, projectId, revision, sessionId, type ClientFrame, type ServerFrame, type SessionRef } from "@t4-code/host-wire"; +import { describe, expect, it } from "vite-plus/test"; +import { + hostId, + projectId, + revision, + sessionId, + type ClientFrame, + type ServerFrame, + type SessionRef, +} from "@t4-code/host-wire"; import { ClusterGateway, type GatewayClient, type GatewayMutationBackend } from "../src/gateway.ts"; import { ClusterInfrastructureProjection } from "../src/kubernetes-projection.ts"; import type { PodHostConnection, PodHostConnector, PodHostRoute } from "../src/pod-host-router.ts"; @@ -91,7 +99,7 @@ function setup(epoch = "replica-uid-1") { projection.setSessionAuthority("session-two", authority("omp-private-two")); const connector = new MemoryConnector(); const mutations = new MemoryMutations(); - const gateway = new ClusterGateway({ projection, connector, mutations, internalToken: "x".repeat(32) }); + const gateway = new ClusterGateway({ projection, connector, mutations }); const client = new MemoryClient(); const connection = gateway.connect(client, PRINCIPAL); return { projection, connector, mutations, gateway, client, connection }; @@ -108,7 +116,7 @@ const hello = { }; describe("stateless omp-app cluster gateway", () => { - test("negotiates cluster.operator, bootstraps one canonical inventory, and changes epoch on replica restart", async () => { + it("negotiates cluster.operator, bootstraps one canonical inventory, and changes epoch on replica restart", async () => { const first = setup("replica-uid-1"); await first.connection.receive(hello); expect(first.client.frames.map(frame => frame.type)).toEqual(["welcome", "sessions"]); @@ -126,7 +134,7 @@ describe("stateless omp-app cluster gateway", () => { expect((second.client.frames[1] as { sessions: unknown[] }).sessions).toHaveLength(2); }); - test("returns workspace bootstrap with its independent cursor and streams each watch revision once", async () => { + it("returns workspace bootstrap with its independent cursor and streams each watch revision once", async () => { const value = setup(); await value.connection.receive(hello); await value.connection.receive({ @@ -145,7 +153,7 @@ describe("stateless omp-app cluster gateway", () => { expect(value.client.frames.filter(frame => frame.type === "workspace.state")).toHaveLength(1); }); - test("routes to exactly one pod host, rewrites only address ids, and preserves attach output order", async () => { + it("routes to exactly one pod host, rewrites only address ids, and preserves attach output order", async () => { const value = setup(); await value.connection.receive(hello); await value.connection.receive({ @@ -178,7 +186,7 @@ describe("stateless omp-app cluster gateway", () => { }); }); - test("denies a preview id learned from another session without opening a second upstream socket", async () => { + it("denies a preview id learned from another session without opening a second upstream socket", async () => { const value = setup(); await value.connection.receive(hello); await value.connection.receive({ @@ -198,7 +206,7 @@ describe("stateless omp-app cluster gateway", () => { expect(value.client.frames.at(-1)).toMatchObject({ type: "response", commandId: "command-preview", ok: false, error: { code: "NOT_AUTHORIZED" } }); }); - test("uses bounded command idempotency while CR mutation identity survives reconnect", async () => { + it("uses bounded command idempotency while CR mutation identity survives reconnect", async () => { const value = setup(); await value.connection.receive(hello); const command = { diff --git a/packages/cluster-server/test/kubernetes-client.test.ts b/packages/cluster-server/test/kubernetes-client.test.ts index f568e536..45fda881 100644 --- a/packages/cluster-server/test/kubernetes-client.test.ts +++ b/packages/cluster-server/test/kubernetes-client.test.ts @@ -1,7 +1,12 @@ -import { describe, expect, test } from "vite-plus/test" +import { describe, expect, it } from "vite-plus/test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { + CLUSTER_INTERNAL_AUDIENCE, KubernetesApiClient, KubernetesGatewayMutationBackend, + KubernetesTokenReviewer, semanticResourceHash, } from "../src/kubernetes-client.ts"; @@ -28,7 +33,7 @@ function conflictFetch(existing: unknown) { } describe("namespaced Kubernetes client", () => { - test("lists and watches only the three cluster.t4.dev resources with bounded resource versions", async () => { + it("lists and watches only the three cluster.t4.dev resources with bounded resource versions", async () => { const values = recordingFetch([ { metadata: { resourceVersion: "20" }, @@ -56,7 +61,7 @@ describe("namespaced Kubernetes client", () => { expect(JSON.stringify(listed)).not.toContain("service-account-token"); }); - test("persists idempotent CR identity as command id plus semantic hash without credentials or arbitrary URLs", async () => { + it("persists idempotent CR identity as command id plus semantic hash without credentials or arbitrary URLs", async () => { const values = recordingFetch([ {}, { kind: "T4Workspace", metadata: { name: "workspace-one" }, spec: { hostRef: "primary", owner: PRINCIPAL } }, @@ -121,7 +126,7 @@ describe("namespaced Kubernetes client", () => { }); }); - test("reuses exact principal-scoped annotations and rejects semantic conflicts", async () => { + it("reuses exact principal-scoped annotations and rejects semantic conflicts", async () => { const args = { displayName: "Created", retentionPolicy: "Delete" as const, capacity: "10Gi" }; const annotations = { "cluster.t4.dev/command-id": "command-one", @@ -148,3 +153,90 @@ describe("namespaced Kubernetes client", () => { await expect(conflictingBackend.createWorkspace("command-one", args, PRINCIPAL)).rejects.toThrow("idempotency conflict"); }); }); + +describe("Kubernetes projected identity review", () => { + it("submits the presented bearer with the fixed audience and requires the exact server ServiceAccount", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-token-review-")); + try { + await writeFile(join(directory, "token"), "reviewer-api-token", { mode: 0o400 }); + await writeFile(join(directory, "ca.crt"), "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n", { mode: 0o400 }); + await writeFile(join(directory, "namespace"), "team\n", { mode: 0o400 }); + const presentedToken = `header.payload.${"s".repeat(64)}`; + const requests: Array<{ url: string; init?: RequestInit }> = []; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + return Response.json({ + apiVersion: "authentication.k8s.io/v1", + kind: "TokenReview", + status: { + authenticated: true, + audiences: [CLUSTER_INTERNAL_AUDIENCE], + user: { username: "system:serviceaccount:team:release-t4-cluster-server" }, + }, + }); + }) as typeof globalThis.fetch; + const reviewer = new KubernetesTokenReviewer({ + baseUrl: "https://kubernetes.default.svc", + tokenPath: join(directory, "token"), + caPath: join(directory, "ca.crt"), + namespacePath: join(directory, "namespace"), + serverServiceAccountName: "release-t4-cluster-server", + fetch, + }); + expect(await reviewer.review(presentedToken)).toBe(true); + expect(requests[0]?.url).toBe("https://kubernetes.default.svc/apis/authentication.k8s.io/v1/tokenreviews"); + expect(new Headers(requests[0]?.init?.headers).get("authorization")).toBe("Bearer reviewer-api-token"); + expect(JSON.parse(String(requests[0]?.init?.body))).toEqual({ + apiVersion: "authentication.k8s.io/v1", + kind: "TokenReview", + spec: { token: presentedToken, audiences: ["t4-cluster-internal"] }, + }); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("denies malformed, rejected, wrong-audience, wrong-username, API-status, and network responses", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-token-review-denied-")); + try { + await writeFile(join(directory, "token"), "reviewer-api-token", { mode: 0o400 }); + await writeFile(join(directory, "ca.crt"), "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n", { mode: 0o400 }); + await writeFile(join(directory, "namespace"), "team", { mode: 0o400 }); + const presentedToken = `header.payload.${"s".repeat(64)}`; + const statuses: unknown[] = [ + { authenticated: false }, + { authenticated: true, audiences: ["other"], user: { username: "system:serviceaccount:team:release-t4-cluster-server" } }, + { authenticated: true, audiences: [CLUSTER_INTERNAL_AUDIENCE], user: { username: "system:serviceaccount:other:release-t4-cluster-server" } }, + { authenticated: true, audiences: [CLUSTER_INTERNAL_AUDIENCE] }, + { authenticated: true, error: "review failed", audiences: [CLUSTER_INTERNAL_AUDIENCE], user: { username: "system:serviceaccount:team:release-t4-cluster-server" } }, + ]; + for (const status of statuses) { + const reviewer = new KubernetesTokenReviewer({ + baseUrl: "https://kubernetes.default.svc", + tokenPath: join(directory, "token"), + caPath: join(directory, "ca.crt"), + namespacePath: join(directory, "namespace"), + serverServiceAccountName: "release-t4-cluster-server", + fetch: (async () => Response.json({ apiVersion: "authentication.k8s.io/v1", kind: "TokenReview", status })) as typeof globalThis.fetch, + }); + expect(await reviewer.review(presentedToken)).toBe(false); + } + const malformed = new KubernetesTokenReviewer({ + baseUrl: "https://kubernetes.default.svc", + tokenPath: join(directory, "token"), caPath: join(directory, "ca.crt"), namespacePath: join(directory, "namespace"), + serverServiceAccountName: "release-t4-cluster-server", + fetch: (async () => new Response("{", { status: 200, headers: { "content-type": "application/json" } })) as typeof globalThis.fetch, + }); + expect(await malformed.review(presentedToken)).toBe(false); + const unavailable = new KubernetesTokenReviewer({ + baseUrl: "https://kubernetes.default.svc", + tokenPath: join(directory, "token"), caPath: join(directory, "ca.crt"), namespacePath: join(directory, "namespace"), + serverServiceAccountName: "release-t4-cluster-server", + fetch: (async () => { throw new Error("network unavailable"); }) as typeof globalThis.fetch, + }); + expect(await unavailable.review(presentedToken)).toBe(false); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cluster-server/test/kubernetes-projection.test.ts b/packages/cluster-server/test/kubernetes-projection.test.ts index f0ca764a..ab2c9e08 100644 --- a/packages/cluster-server/test/kubernetes-projection.test.ts +++ b/packages/cluster-server/test/kubernetes-projection.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "vite-plus/test" +import { describe, expect, it } from "vite-plus/test"; import { hostId, projectId, revision, sessionId, type SessionRef } from "@t4-code/host-wire"; import { CLUSTER_MAX_SESSIONS, @@ -70,7 +70,7 @@ const session = { }; describe("Kubernetes infrastructure projection", () => { - test("derives a stable host id from the T4ClusterHost UID and bounded list state", () => { + it("derives a stable host id from the T4ClusterHost UID and bounded list state", () => { expect(clusterHostIdFromUid(host.metadata.uid)).toBe("cluster:24e7bcb1-c694-4ba4-85c4-70a829f7996b"); expect(CLUSTER_MAX_WORKSPACES).toBe(256); expect(CLUSTER_MAX_SESSIONS).toBe(1_000); @@ -92,7 +92,8 @@ describe("Kubernetes infrastructure projection", () => { expect(JSON.stringify(projection.workspaceList())).not.toContain("pvcRef"); expect(JSON.stringify(projection.workspaceList())).not.toContain("repositoryId"); }); - test("keeps workspace cursors separate and reconnect replacement idempotent", () => { + + it("keeps workspace cursors separate and reconnect replacement idempotent", () => { const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development" }); projection.replace({ host, workspaces: [workspace], sessions: [session], resourceVersion: "102" }); const seen: unknown[] = []; @@ -114,7 +115,7 @@ describe("Kubernetes infrastructure projection", () => { stop(); }); - test("projects routable pod authority and removes deleted sessions without local truth", () => { + it("projects routable pod authority and removes deleted sessions without local truth", () => { const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development" }); projection.replace({ host, workspaces: [workspace], sessions: [session], resourceVersion: "102" }); projection.setSessionAuthority("session-one", authority("omp-session-private")); @@ -137,7 +138,7 @@ describe("Kubernetes infrastructure projection", () => { expect(projection.sessionRefs()).toEqual([]); }); - test("fails closed at explicit projection limits", () => { + it("fails closed at explicit projection limits", () => { const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development", diff --git a/packages/cluster-server/test/observability.test.ts b/packages/cluster-server/test/observability.test.ts index 97282d30..40816ab8 100644 --- a/packages/cluster-server/test/observability.test.ts +++ b/packages/cluster-server/test/observability.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "vite-plus/test" +import { describe, expect, it } from "vite-plus/test"; import { ClusterMetrics, ClusterServerHealth, @@ -9,7 +9,7 @@ import { import { isLoopbackAddress } from "../src/server.ts"; describe("cluster-server observability", () => { - test("redacts credentials, prompt content, pairing data, and private paths recursively", () => { + it("redacts credentials, prompt content, pairing data, and private paths recursively", () => { const redacted = redactStructuredValue({ component: "cluster-server", session: "session-one", @@ -35,7 +35,7 @@ describe("cluster-server observability", () => { expect(JSON.parse(lines[0] ?? "{}")).toMatchObject({ level: "info", message: "ci lookup", provider: "woodpecker", token: "[REDACTED]" }); }); - test("serves separate health/readiness and bounded metrics without a remote drain route", async () => { + it("serves separate health/readiness and bounded metrics without a remote drain route", async () => { const health = new ClusterServerHealth(); const metrics = new ClusterMetrics({ component: "cluster-server", version: "test" }); const handler = createAdminHandler({ health, metrics }); @@ -57,7 +57,7 @@ describe("cluster-server observability", () => { expect((await handler(new Request("http://admin/readyz"))).status).toBe(200); }); - test("recognizes only kernel loopback sources for the preStop drain route", () => { + it("recognizes only kernel loopback sources for the preStop drain route", () => { expect(isLoopbackAddress("127.0.0.1")).toBe(true); expect(isLoopbackAddress("127.42.0.7")).toBe(true); expect(isLoopbackAddress("::1")).toBe(true); @@ -65,7 +65,7 @@ describe("cluster-server observability", () => { expect(isLoopbackAddress("fd7a:115c:a1e0::1")).toBe(false); }); - test("rejects unbounded or secret-like metric labels", () => { + it("rejects unbounded or secret-like metric labels", () => { const metrics = new ClusterMetrics({ component: "cluster-server", version: "test" }); expect(() => metrics.increment("bad-name", {})).toThrow("metric name"); expect(() => metrics.increment("valid_metric", { token: "secret" })).toThrow("metric label"); diff --git a/packages/cluster-server/test/pod-host-router.test.ts b/packages/cluster-server/test/pod-host-router.test.ts new file mode 100644 index 00000000..a5547455 --- /dev/null +++ b/packages/cluster-server/test/pod-host-router.test.ts @@ -0,0 +1,62 @@ +import { expect, it } from "vite-plus/test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { WebSocketPodHostConnector } from "../src/pod-host-router.ts"; + +class MemoryWebSocket { + readyState = WebSocket.CONNECTING; + readonly sent = Promise.withResolvers(); + readonly #listeners: Record void>> = {}; + addEventListener(type: string, listener: EventListenerOrEventListenerObject): void { + const callback = typeof listener === "function" ? listener : event => listener.handleEvent(event as Event); + (this.#listeners[type] ??= []).push(callback); + } + send(value: string | ArrayBufferLike | Blob | ArrayBufferView): void { + this.sent.resolve(String(value)); + } + close(): void { this.readyState = WebSocket.CLOSED; } + emit(type: string, event: unknown = {}): void { + if (type === "open") this.readyState = WebSocket.OPEN; + for (const listener of this.#listeners[type] ?? []) listener(event); + } +} + +it("pod connector reads the current projected identity and presents it in the existing hello authentication field", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-pod-identity-")); + try { + const path = join(directory, "token"); + const token = `header.payload.${"s".repeat(64)}`; + await writeFile(path, token, { mode: 0o400 }); + const socket = new MemoryWebSocket(); + const connector = new WebSocketPodHostConnector({ + identityTokenFile: path, + webSocketFactory: () => socket as unknown as WebSocket, + }); + const pending = connector.connect({ clusterSessionId: "session-one", url: "ws://session-one:8787/v1/ws" }, () => undefined); + socket.emit("open"); + const hello = JSON.parse(await socket.sent.promise); + expect(hello.authentication).toEqual({ deviceId: "cluster-server", deviceToken: token }); + socket.emit("message", { data: JSON.stringify({ + v: "omp-app/1", + type: "welcome", + selectedProtocol: "omp-app/1", + hostId: "host-a", + ompVersion: "17.0.5", + ompBuild: "test", + appserverVersion: "0.1.30", + appserverBuild: "cluster-session", + epoch: "pod-epoch", + grantedCapabilities: [], + grantedFeatures: [], + negotiatedLimits: {}, + authentication: "paired", + resumed: false, + }) }); + const connection = await pending; + expect(connection.hostId).toBe("host-a"); + connection.close(); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/packages/cluster-server/test/session-host-policy.test.ts b/packages/cluster-server/test/session-host-policy.test.ts index c759d14b..068154fc 100644 --- a/packages/cluster-server/test/session-host-policy.test.ts +++ b/packages/cluster-server/test/session-host-policy.test.ts @@ -1,7 +1,13 @@ -import { describe, expect, test } from "vite-plus/test" +import { describe, expect, it } from "vite-plus/test"; import type { RemoteConnection } from "@t4-code/host-service"; -import { ClusterInternalRemotePolicy, sessionHostConfigFromEnv } from "../src/session-host-policy.ts"; +import { + ClusterInternalRemotePolicy, + decodeClusterInternalClientFrame, + sessionHostConfigFromEnv, + type ClusterIdentityReviewer, +} from "../src/session-host-policy.ts"; +const SERVER_TOKEN = `header.payload.${"s".repeat(64)}`; const connection = { connectionId: "connection-one", peer: { @@ -19,63 +25,84 @@ const hello = { requestedFeatures: ["resume", "session.state", "cluster.operator"], savedCursors: [], capabilities: { client: ["sessions.read", "sessions.prompt", "preview.control", "ci.trigger"] }, - authentication: { deviceId: "cluster-server", deviceToken: "s".repeat(32) }, + authentication: { deviceId: "cluster-server", deviceToken: SERVER_TOKEN }, }; +class MemoryReviewer implements ClusterIdentityReviewer { + allowed = true; + failure = false; + readonly tokens: string[] = []; + async review(token: string): Promise { + this.tokens.push(token); + if (this.failure) throw new Error("unavailable"); + return this.allowed; + } +} +function policy(reviewer: ClusterIdentityReviewer): ClusterInternalRemotePolicy { + return new ClusterInternalRemotePolicy({ + reviewer, + supportedCapabilities: ["sessions.read", "sessions.prompt", "preview.control"], + supportedFeatures: ["resume", "session.state"], + }); +} describe("one-session pod host authority", () => { - test("accepts only the mounted internal credential and never grants cluster-server-only names upstream", async () => { - const policy = new ClusterInternalRemotePolicy({ - token: "s".repeat(32), - supportedCapabilities: ["sessions.read", "sessions.prompt", "preview.control"], - supportedFeatures: ["resume", "session.state"], - }); - expect(await policy.authenticate(connection, hello)).toEqual({ + it("preserves the existing hello field while admitting a bounded projected bearer only on the internal policy", () => { + expect(decodeClusterInternalClientFrame(hello)).toMatchObject({ authentication: { deviceId: "cluster-server", deviceToken: SERVER_TOKEN } }); + expect(() => decodeClusterInternalClientFrame({ ...hello, authentication: { ...hello.authentication, deviceToken: "short" } })).toThrow("token"); + }); + + it("accepts only the TokenReview-authorized fixed server peer and never grants cluster-server-only names upstream", async () => { + const reviewer = new MemoryReviewer(); + const remotePolicy = policy(reviewer); + expect(await remotePolicy.authenticate(connection, hello)).toEqual({ authenticated: true, authentication: "paired", deviceId: "cluster-server", grantedCapabilities: ["sessions.read", "sessions.prompt", "preview.control"], grantedFeatures: ["resume", "session.state"], }); - expect( - await policy.authenticate(connection, { - ...hello, - authentication: { deviceId: "cluster-server", deviceToken: "x".repeat(32) }, - }), - ).toMatchObject({ authenticated: false, authentication: "denied" }); + expect(reviewer.tokens).toEqual([SERVER_TOKEN]); + reviewer.allowed = false; + expect(await remotePolicy.authenticate(connection, hello)).toMatchObject({ authenticated: false, authentication: "denied" }); + reviewer.failure = true; + expect(await remotePolicy.authenticate(connection, hello)).toMatchObject({ authenticated: false, authentication: "denied" }); + expect(await remotePolicy.authenticate({ ...connection, peer: { ...connection.peer, identity: { ...connection.peer.identity, nodeId: "other" } } }, hello)).toMatchObject({ authenticated: false }); }); - test("authorizes only negotiated command capabilities on an authenticated connection", async () => { - const policy = new ClusterInternalRemotePolicy({ - token: "s".repeat(32), - supportedCapabilities: ["sessions.read"], - supportedFeatures: ["resume"], - }); - await policy.authenticate(connection, hello); - expect(await policy.authorize(connection, { v: "omp-app/1", type: "ping", nonce: "one" }, { connectionId: "connection-one", peer: connection.peer })).toBe(true); - expect(await policy.authorize(connection, { + it("authorizes only negotiated command capabilities on an authenticated connection", async () => { + const remotePolicy = policy(new MemoryReviewer()); + await remotePolicy.authenticate(connection, hello); + expect(await remotePolicy.authorize(connection, { v: "omp-app/1", type: "ping", nonce: "one" }, { connectionId: "connection-one", peer: connection.peer })).toBe(true); + expect(await remotePolicy.authorize(connection, { v: "omp-app/1", type: "command", requestId: "r1", commandId: "c1", hostId: "pod-host", sessionId: "private-session", command: "session.attach", args: {}, }, { connectionId: "connection-one", peer: connection.peer })).toBe(true); - expect(await policy.authorize(connection, { + expect(await remotePolicy.authorize(connection, { v: "omp-app/1", type: "command", requestId: "r2", commandId: "c2", hostId: "pod-host", sessionId: "private-session", command: "session.prompt", args: { message: "hello" }, }, { connectionId: "connection-one", peer: connection.peer })).toBe(false); }); - test("parses a fixed, path-safe session host environment", () => { + it("parses fixed Kubernetes reviewer and isolated session paths", () => { expect(sessionHostConfigFromEnv({ - T4_CLUSTER_INTERNAL_TOKEN: "s".repeat(32), + KUBERNETES_SERVICE_HOST: "10.96.0.1", + KUBERNETES_SERVICE_PORT_HTTPS: "443", + T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "release-t4-cluster-server", T4_SESSION_NAME: "session-one", T4_OMP_EXECUTABLE: "/opt/t4/bin/omp", T4_SESSION_STATE_ROOT: "/workspace/.t4/sessions/session-one", T4_SESSION_HOST_PORT: "8787", })).toEqual({ - internalToken: "s".repeat(32), + kubernetesBaseUrl: "https://10.96.0.1:443", + kubernetesTokenPath: "/var/run/secrets/kubernetes.io/serviceaccount/token", + kubernetesCaPath: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + kubernetesNamespacePath: "/var/run/secrets/kubernetes.io/serviceaccount/namespace", + serverServiceAccountName: "release-t4-cluster-server", sessionName: "session-one", ompExecutable: "/opt/t4/bin/omp", stateRoot: "/workspace/.t4/sessions/session-one", port: 8787, }); - expect(() => sessionHostConfigFromEnv({ ...process.env, T4_CLUSTER_INTERNAL_TOKEN: "short" })).toThrow("token"); + expect(() => sessionHostConfigFromEnv({ KUBERNETES_SERVICE_HOST: "10.96.0.1", T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "server", T4_SESSION_NAME: "bad/name" })).toThrow("T4_SESSION_NAME"); }); }); diff --git a/packages/cluster-server/test/woodpecker.test.ts b/packages/cluster-server/test/woodpecker.test.ts index cf503f24..fb335288 100644 --- a/packages/cluster-server/test/woodpecker.test.ts +++ b/packages/cluster-server/test/woodpecker.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "vite-plus/test" +import { describe, expect, it } from "vite-plus/test"; import { WoodpeckerProvider, mapWoodpeckerPipeline } from "../src/woodpecker.ts"; const correlation = { @@ -33,7 +33,7 @@ function provider(fetch: typeof globalThis.fetch) { } describe("bounded Woodpecker provider", () => { - test("maps only allowlisted categorical pipeline state and canonical HTTPS links", () => { + it("maps only allowlisted categorical pipeline state and canonical HTTPS links", () => { expect(mapWoodpeckerPipeline(pipeline, correlation, "https://ci.example.test/repos/owner/t4-code/pipeline/42")).toEqual({ provider: "woodpecker", correlation: "exact", @@ -53,7 +53,7 @@ describe("bounded Woodpecker provider", () => { ] as const) expect(mapWoodpeckerPipeline({ ...pipeline, status: raw }, correlation).status).toBe(mapped); }); - test("queries exact repository/ref/commit/session correlation and deduplicates before trigger", async () => { + it("queries exact repository/ref/commit/session correlation and deduplicates before trigger", async () => { const requests: Array<{ url: string; init?: RequestInit }> = []; const fetch = (async (input: string | URL | Request, init?: RequestInit) => { requests.push({ url: String(input), init }); @@ -67,7 +67,7 @@ describe("bounded Woodpecker provider", () => { expect(JSON.stringify(result)).not.toContain("secret-from-kubernetes"); }); - test("ignores approximate matches, triggers once with server-resolved URL, and re-queries before retry", async () => { + it("ignores approximate matches, triggers once with server-resolved URL, and re-queries before retry", async () => { const requests: Array<{ url: string; init?: RequestInit }> = []; let query = 0; const fetch = (async (input: string | URL | Request, init?: RequestInit) => { @@ -94,7 +94,7 @@ describe("bounded Woodpecker provider", () => { expect(requests.filter(request => request.init?.method === "POST")).toHaveLength(1); }); - test("fails closed for unconfigured repositories, insecure provider URLs, oversized replies, and unknown correlation", async () => { + it("fails closed for unconfigured repositories, insecure provider URLs, oversized replies, and unknown correlation", async () => { expect(() => new WoodpeckerProvider({ baseUrl: "http://ci.example.test", token: "secret", repositories: {} })).toThrow("HTTPS"); const fetch = (async () => Response.json(Array.from({ length: 101 }, () => pipeline))) as typeof globalThis.fetch; await expect(provider(fetch).query(correlation)).rejects.toThrow("pipeline response limit"); diff --git a/packages/host-service/src/server.ts b/packages/host-service/src/server.ts index 55449819..f3714f01 100644 --- a/packages/host-service/src/server.ts +++ b/packages/host-service/src/server.ts @@ -3479,7 +3479,10 @@ export class LocalAppserver implements AppserverHandle { return; } if (typeof raw !== "string") throw new Error("binary websocket frames are not supported"); - const frame = decodeClientFrame(parseBounded(raw)); + const input = parseBounded(raw); + const frame = ws.remote && this.#remotePolicy?.decodeClientFrame + ? this.#remotePolicy.decodeClientFrame(input) + : decodeClientFrame(input); if (frame.type === "command" && frame.command === "session.attach") attachingSessionId = frame.sessionId; if (frame.type === "hello") { if (this.#hello.has(ws)) throw new Error("hello already received"); diff --git a/packages/host-service/src/types.ts b/packages/host-service/src/types.ts index bce7a9a6..c4c44283 100644 --- a/packages/host-service/src/types.ts +++ b/packages/host-service/src/types.ts @@ -55,6 +55,8 @@ export interface RemoteAuthorizationContext { readonly sessionRevision?: Revision; } export interface RemoteConnectionPolicy { + /** Optional decoder used only by a policy-bound internal listener; public remote listeners retain the canonical wire decoder. */ + decodeClientFrame?(input: unknown): ClientFrame; authenticate(connection: RemoteConnection, hello: HelloFrame): RemoteHelloDecision | Promise; pairStart?( connection: RemoteConnection, From c99df7db464af78a427b36c1421579988f1dbf05 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 16:33:24 +0000 Subject: [PATCH 17/98] test(cluster): cover integration safety gaps --- .../charttests/chart_contract_test.go | 16 ++++ .../controllers/reconciler_test.go | 87 +++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go index dd4179c6..3ca54344 100644 --- a/packages/cluster-operator/charttests/chart_contract_test.go +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -49,6 +49,15 @@ func TestEnabledChartRendersHARestrictedWorkloads(t *testing.T) { "coordination.k8s.io", "resources:", ) + server := documentContainingKind(t, output, "Deployment", "name: release-name-t4-cluster-server") + assertContains(t, server, + "automountServiceAccountToken: false", + "name: T4_CLUSTER_TRUSTED_PROXY_CIDRS", + "value: 192.0.2.0/24", + "name: kubernetes-api-access", + "audience: https://kubernetes.default.svc", + "expirationSeconds: 3600", + ) if strings.Contains(output, "privileged: true") || strings.Contains(output, "hostNetwork: true") || strings.Contains(output, "hostPID: true") { t.Fatal("enabled chart contains a privileged shortcut") } @@ -158,6 +167,12 @@ func TestImageContractsArePinnedAndAuthorityCompatible(t *testing.T) { "chromium", "Xvfb", ) + for name, content := range map[string]string{"server": server, "session": session} { + assertContains(t, content, "pnpm install --frozen-lockfile") + if strings.Contains(content, "bun install --ignore-scripts --lockfile-only") { + t.Fatalf("%s image synthesizes an uncommitted dependency lock", name) + } + } assertContains(t, server, "packages/cluster-server/src/main.ts") entrypoint := mustRead(t, filepath.Join(root, "cluster", "images", "session-runtime", "session-entrypoint.sh")) assertContains(t, entrypoint, @@ -187,6 +202,7 @@ func enabledValues() []string { "--set", "images.controller.digest=" + fakeDigest, "--set", "images.server.digest=" + fakeDigest, "--set", "images.sessionRuntime.digest=" + fakeDigest, + "--set", "server.trustedProxyCIDRs[0]=192.0.2.0/24", } } diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 15080a33..21a4b20e 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -58,6 +58,27 @@ func TestWorkspaceReconcileIsIdempotentAcrossDuplicateEvents(t *testing.T) { } } +func TestRetainWorkspaceCreatesPVCWithoutGarbageCollectableOwner(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) + workspace.UID = "workspace-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), rwxStorageClass(), workspace).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) + return err + }) + var pvc corev1.PersistentVolumeClaim + if err := c.Get(context.Background(), types.NamespacedName{Namespace: workspace.Namespace, Name: controllers.WorkspacePVCName(workspace)}, &pvc); err != nil { + t.Fatal(err) + } + if len(pvc.OwnerReferences) != 0 || pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] != string(workspace.UID) { + t.Fatalf("retained PVC is exposed to owner garbage collection: %#v", pvc.ObjectMeta) + } +} + func TestWorkspaceStorageFailsClosedWhenClassMissingOrNotRWX(t *testing.T) { for _, test := range []struct { name string @@ -255,6 +276,72 @@ func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) } } +func TestSessionRejectsUnownedDeterministicResources(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: "team"}} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), workspace, pvc, session, service).Build() + r := &controllers.SessionReconciler{Client: c, Scheme: scheme, RuntimeImage: "registry.example/session@sha256:deadbeef"} + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { t.Fatal(err) } + var got clusterv1alpha1.T4Session + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &got); err != nil { t.Fatal(err) } + condition := findCondition(got.Status.Conditions, "Available") + if condition == nil || condition.Reason != "ServiceOwnershipConflict" || got.Status.Phase != clusterv1alpha1.InfrastructureFailed { + t.Fatalf("collision status = %#v/%q", condition, got.Status.Phase) + } + var pods corev1.PodList + if err := c.List(context.Background(), &pods, client.InNamespace("team")); err != nil { t.Fatal(err) } + if len(pods.Items) != 0 { t.Fatalf("collision created %d pods", len(pods.Items)) } +} + +func TestSessionRecreatesPodWhenImmutableDesiredStateChanges(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := &controllers.SessionReconciler{Client: c, Scheme: scheme, RuntimeImage: "registry.example/session@sha256:old"} + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + var original corev1.Pod + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)}, &original); err != nil { t.Fatal(err) } + originalHash := original.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] + r.RuntimeImage = "registry.example/session@sha256:new" + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { t.Fatal(err) } + var deleted corev1.Pod + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)}, &deleted); !apierrors.IsNotFound(err) { + t.Fatalf("outdated pod remains after immutable desired state changed: %v", err) + } + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + var replacement corev1.Pod + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)}, &replacement); err != nil { t.Fatal(err) } + if replacement.Spec.Containers[0].Image != r.RuntimeImage || replacement.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] == originalHash { + t.Fatalf("replacement pod did not converge: image=%q annotations=%#v", replacement.Spec.Containers[0].Image, replacement.Annotations) + } +} + func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { scheme := testScheme(t) session := testSession() From d222c0bd4f9f7628b1c36789661005538dcc9cd9 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 16:38:17 +0000 Subject: [PATCH 18/98] fix(cluster): harden portable runtime reconciliation --- cluster/images/cluster-server/Dockerfile | 10 +- cluster/images/session-runtime/Dockerfile | 6 +- .../crds/t4clusterhosts.cluster.t4.dev.yaml | 4 +- .../crds/t4workspaces.cluster.t4.dev.yaml | 2 +- .../t4-cluster/templates/deployments.yaml | 22 ++- deploy/charts/t4-cluster/values.schema.json | 12 +- deploy/charts/t4-cluster/values.yaml | 5 +- .../cluster-operator/api/v1alpha1/types.go | 28 ++-- .../controllers/reconciler_test.go | 60 +++++--- .../controllers/session_controller.go | 83 ++++++++--- .../controllers/workspace_controller.go | 135 +++++++++++++----- 11 files changed, 275 insertions(+), 92 deletions(-) diff --git a/cluster/images/cluster-server/Dockerfile b/cluster/images/cluster-server/Dockerfile index 6608f17a..ba6e1513 100644 --- a/cluster/images/cluster-server/Dockerfile +++ b/cluster/images/cluster-server/Dockerfile @@ -1,13 +1,17 @@ # syntax=docker/dockerfile:1.7 +FROM docker.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 AS dependencies +WORKDIR /opt/t4 +COPY . /opt/t4 +RUN corepack enable \ + && pnpm install --frozen-lockfile --ignore-scripts + FROM docker.io/oven/bun:1.3.14-debian@sha256:9dba1a1b43ce28c9d7931bfc4eb00feb63b0114720a0277a8f939ae4dfc9db6f USER root RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl tini \ && rm -rf /var/lib/apt/lists/* WORKDIR /opt/t4 -COPY . /opt/t4 -RUN bun install --ignore-scripts --lockfile-only \ - && bun install --frozen-lockfile --ignore-scripts +COPY --from=dependencies /opt/t4 /opt/t4 ENV NODE_ENV=production \ HOME=/tmp \ T4_CLUSTER_SERVER_PORT=8080 \ diff --git a/cluster/images/session-runtime/Dockerfile b/cluster/images/session-runtime/Dockerfile index 087d526d..a8c4c910 100644 --- a/cluster/images/session-runtime/Dockerfile +++ b/cluster/images/session-runtime/Dockerfile @@ -22,11 +22,11 @@ RUN bun install --frozen-lockfile --ignore-scripts \ && grep -R -F -q 't4-omp-authority/1' packages/coding-agent \ && rm -rf .git -FROM ${BUN_IMAGE} AS t4-build +FROM docker.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 AS t4-build WORKDIR /opt/t4 COPY . /opt/t4 -RUN bun install --ignore-scripts --lockfile-only \ - && bun install --frozen-lockfile --ignore-scripts +RUN corepack enable \ + && pnpm install --frozen-lockfile --ignore-scripts FROM ${BUN_IMAGE} USER root diff --git a/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml b/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml index a4876f87..83ab3078 100644 --- a/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml +++ b/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml @@ -66,8 +66,8 @@ spec: type: integer format: int32 minimum: 1 - maximum: 10000 - default: 1000 + maximum: 256 + default: 256 resyncSeconds: type: integer format: int32 diff --git a/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml b/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml index 078d7543..033c00c3 100644 --- a/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml +++ b/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml @@ -56,7 +56,7 @@ spec: owner: type: string minLength: 1 - maxLength: 128 + maxLength: 256 pattern: '^[A-Za-z0-9][A-Za-z0-9._:@/-]*$' repository: type: object diff --git a/deploy/charts/t4-cluster/templates/deployments.yaml b/deploy/charts/t4-cluster/templates/deployments.yaml index 5e06f8ae..f453d688 100644 --- a/deploy/charts/t4-cluster/templates/deployments.yaml +++ b/deploy/charts/t4-cluster/templates/deployments.yaml @@ -155,7 +155,7 @@ spec: app.kubernetes.io/part-of: t4-cluster spec: serviceAccountName: {{ include "t4-cluster.fullname" . }}-server - automountServiceAccountToken: true + automountServiceAccountToken: false terminationGracePeriodSeconds: {{ .Values.server.terminationGracePeriodSeconds }} securityContext: runAsNonRoot: true @@ -202,6 +202,10 @@ spec: value: "8080" - name: T4_CLUSTER_ADMIN_PORT value: "9090" + - name: T4_CLUSTER_TRUSTED_PROXY_ADDRESSES + value: {{ join "," .Values.server.trustedProxyAddresses | quote }} + - name: T4_CLUSTER_TRUSTED_PROXY_CIDRS + value: {{ join "," .Values.server.trustedProxyCIDRs | quote }} - name: T4_CLUSTER_HOST_NAME value: {{ .Values.clusterHost.name }} - name: T4_CLUSTER_IDENTITY_TOKEN_FILE @@ -273,6 +277,9 @@ spec: - name: cluster-identity mountPath: /var/run/secrets/t4-cluster-identity readOnly: true + - name: kubernetes-api-access + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true volumes: - name: cluster-identity projected: @@ -282,6 +289,19 @@ spec: audience: t4-cluster-internal expirationSeconds: 600 path: token + - name: kubernetes-api-access + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: + audience: https://kubernetes.default.svc + expirationSeconds: 3600 + path: token + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt - name: temporary emptyDir: sizeLimit: 256Mi diff --git a/deploy/charts/t4-cluster/values.schema.json b/deploy/charts/t4-cluster/values.schema.json index e693624f..f12cab16 100644 --- a/deploy/charts/t4-cluster/values.schema.json +++ b/deploy/charts/t4-cluster/values.schema.json @@ -15,7 +15,7 @@ "type": "object", "required": ["maxWorkspaces", "resyncSeconds"], "properties": { - "maxWorkspaces": { "type": "integer", "minimum": 1, "maximum": 10000 }, + "maxWorkspaces": { "type": "integer", "minimum": 1, "maximum": 256 }, "resyncSeconds": { "type": "integer", "minimum": 10, "maximum": 3600 } } } @@ -41,10 +41,12 @@ }, "server": { "type": "object", - "required": ["replicas", "terminationGracePeriodSeconds", "resources"], + "required": ["replicas", "terminationGracePeriodSeconds", "trustedProxyAddresses", "trustedProxyCIDRs", "resources"], "properties": { "replicas": { "type": "integer", "minimum": 2, "maximum": 9 }, "terminationGracePeriodSeconds": { "type": "integer", "minimum": 30, "maximum": 300 }, + "trustedProxyAddresses": { "type": "array", "maxItems": 64, "uniqueItems": true, "items": { "type": "string", "anyOf": [{ "format": "ipv4" }, { "format": "ipv6" }] } }, + "trustedProxyCIDRs": { "type": "array", "maxItems": 64, "uniqueItems": true, "items": { "type": "string", "pattern": "^[0-9A-Fa-f:.]+/(?:[0-9]|[1-9][0-9]|1[01][0-9]|12[0-8])$", "maxLength": 64 } }, "resources": { "type": "object" } } }, @@ -84,6 +86,12 @@ "if": { "properties": { "enabled": { "const": true } } }, "then": { "properties": { + "server": { + "anyOf": [ + { "properties": { "trustedProxyAddresses": { "minItems": 1 } } }, + { "properties": { "trustedProxyCIDRs": { "minItems": 1 } } } + ] + }, "storage": { "properties": { "adminRWXStorageClass": { "minLength": 1 } } }, "images": { "properties": { diff --git a/deploy/charts/t4-cluster/values.yaml b/deploy/charts/t4-cluster/values.yaml index 5b722052..e8006b7a 100644 --- a/deploy/charts/t4-cluster/values.yaml +++ b/deploy/charts/t4-cluster/values.yaml @@ -6,7 +6,7 @@ clusterHost: - default allowedOrigins: [] projection: - maxWorkspaces: 1000 + maxWorkspaces: 256 resyncSeconds: 60 images: @@ -38,6 +38,9 @@ controller: server: replicas: 3 terminationGracePeriodSeconds: 60 + # At least one trusted Tailscale proxy address or canonical CIDR is required for client access. + trustedProxyAddresses: [] + trustedProxyCIDRs: [] resources: requests: cpu: 250m diff --git a/packages/cluster-operator/api/v1alpha1/types.go b/packages/cluster-operator/api/v1alpha1/types.go index 7749fc47..49e18b77 100644 --- a/packages/cluster-operator/api/v1alpha1/types.go +++ b/packages/cluster-operator/api/v1alpha1/types.go @@ -7,10 +7,12 @@ import ( ) const ( - WorkspaceFinalizer = "cluster.t4.dev/workspace-protection" - SessionFinalizer = "cluster.t4.dev/session-cleanup" - RWXStorageClassAnnotation = "cluster.t4.dev/access-modes" - RetainedPVCAnnotation = "cluster.t4.dev/retained" + WorkspaceFinalizer = "cluster.t4.dev/workspace-protection" + SessionFinalizer = "cluster.t4.dev/session-cleanup" + RWXStorageClassAnnotation = "cluster.t4.dev/access-modes" + RetainedPVCAnnotation = "cluster.t4.dev/retained" + WorkspaceUIDAnnotation = "cluster.t4.dev/workspace-uid" + SessionPodSpecHashAnnotation = "cluster.t4.dev/pod-spec-hash" ) type RetentionPolicy string @@ -105,11 +107,11 @@ type T4WorkspaceSpec struct { type T4WorkspaceStatus struct { ObservedGeneration int64 `json:"observedGeneration,omitempty"` - PVCName string `json:"pvcName,omitempty"` - PVCPhase corev1.ClaimPhase `json:"pvcPhase,omitempty"` - Capacity resource.Quantity `json:"capacity,omitempty"` - Phase InfrastructurePhase `json:"phase,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` + PVCName string `json:"pvcName,omitempty"` + PVCPhase corev1.ClaimPhase `json:"pvcPhase,omitempty"` + Capacity resource.Quantity `json:"capacity,omitempty"` + Phase InfrastructurePhase `json:"phase,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` } // +kubebuilder:object:root=true @@ -150,10 +152,10 @@ type T4SessionSpec struct { type T4SessionStatus struct { ObservedGeneration int64 `json:"observedGeneration,omitempty"` - PodName string `json:"podName,omitempty"` - ServiceName string `json:"serviceName,omitempty"` - Phase InfrastructurePhase `json:"phase,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` + PodName string `json:"podName,omitempty"` + ServiceName string `json:"serviceName,omitempty"` + Phase InfrastructurePhase `json:"phase,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` } // +kubebuilder:object:root=true diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 21a4b20e..1445df54 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -171,16 +171,28 @@ func TestWorkspaceDeletionWaitsForSessionResources(t *testing.T) { session := testSession() session.Spec.WorkspaceRef = workspace.Name c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc, session).Build() - if err := c.Delete(context.Background(), workspace); err != nil { t.Fatal(err) } + if err := c.Delete(context.Background(), workspace); err != nil { + t.Fatal(err) + } r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} - if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { t.Fatal(err) } + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } var waiting clusterv1alpha1.T4Workspace - if err := c.Get(context.Background(), client.ObjectKeyFromObject(workspace), &waiting); err != nil { t.Fatalf("workspace deletion did not wait: %v", err) } + if err := c.Get(context.Background(), client.ObjectKeyFromObject(workspace), &waiting); err != nil { + t.Fatalf("workspace deletion did not wait: %v", err) + } condition := findCondition(waiting.Status.Conditions, "Ready") - if condition == nil || condition.Reason != "SessionsRemain" { t.Fatalf("Ready = %#v, want SessionsRemain", condition) } + if condition == nil || condition.Reason != "SessionsRemain" { + t.Fatalf("Ready = %#v, want SessionsRemain", condition) + } var retained corev1.PersistentVolumeClaim - if err := c.Get(context.Background(), client.ObjectKeyFromObject(pvc), &retained); err != nil { t.Fatalf("workspace PVC changed during wait: %v", err) } - if len(retained.OwnerReferences) != 1 { t.Fatalf("workspace PVC was orphaned before sessions exited: %#v", retained.OwnerReferences) } + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pvc), &retained); err != nil { + t.Fatalf("workspace PVC changed during wait: %v", err) + } + if len(retained.OwnerReferences) != 1 { + t.Fatalf("workspace PVC was orphaned before sessions exited: %#v", retained.OwnerReferences) + } } func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) { @@ -282,8 +294,8 @@ func TestSessionRejectsUnownedDeterministicResources(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, - Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: "team"}} @@ -291,16 +303,24 @@ func TestSessionRejectsUnownedDeterministicResources(t *testing.T) { WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}). WithObjects(testHost(), workspace, pvc, session, service).Build() r := &controllers.SessionReconciler{Client: c, Scheme: scheme, RuntimeImage: "registry.example/session@sha256:deadbeef"} - if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { t.Fatal(err) } + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } var got clusterv1alpha1.T4Session - if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &got); err != nil { t.Fatal(err) } + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } condition := findCondition(got.Status.Conditions, "Available") if condition == nil || condition.Reason != "ServiceOwnershipConflict" || got.Status.Phase != clusterv1alpha1.InfrastructureFailed { t.Fatalf("collision status = %#v/%q", condition, got.Status.Phase) } var pods corev1.PodList - if err := c.List(context.Background(), &pods, client.InNamespace("team")); err != nil { t.Fatal(err) } - if len(pods.Items) != 0 { t.Fatalf("collision created %d pods", len(pods.Items)) } + if err := c.List(context.Background(), &pods, client.InNamespace("team")); err != nil { + t.Fatal(err) + } + if len(pods.Items) != 0 { + t.Fatalf("collision created %d pods", len(pods.Items)) + } } func TestSessionRecreatesPodWhenImmutableDesiredStateChanges(t *testing.T) { @@ -309,8 +329,8 @@ func TestSessionRecreatesPodWhenImmutableDesiredStateChanges(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, - Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() session.UID = "session-uid" @@ -323,10 +343,14 @@ func TestSessionRecreatesPodWhenImmutableDesiredStateChanges(t *testing.T) { return err }) var original corev1.Pod - if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)}, &original); err != nil { t.Fatal(err) } + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)}, &original); err != nil { + t.Fatal(err) + } originalHash := original.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] r.RuntimeImage = "registry.example/session@sha256:new" - if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { t.Fatal(err) } + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } var deleted corev1.Pod if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)}, &deleted); !apierrors.IsNotFound(err) { t.Fatalf("outdated pod remains after immutable desired state changed: %v", err) @@ -336,7 +360,9 @@ func TestSessionRecreatesPodWhenImmutableDesiredStateChanges(t *testing.T) { return err }) var replacement corev1.Pod - if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)}, &replacement); err != nil { t.Fatal(err) } + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)}, &replacement); err != nil { + t.Fatal(err) + } if replacement.Spec.Containers[0].Image != r.RuntimeImage || replacement.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] == originalHash { t.Fatalf("replacement pod did not converge: image=%q annotations=%#v", replacement.Spec.Containers[0].Image, replacement.Annotations) } diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index ac0738e7..6a1be61a 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -2,6 +2,8 @@ package controllers import ( "context" + "crypto/sha256" + "encoding/json" "fmt" "reflect" "strings" @@ -99,37 +101,61 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) "app.kubernetes.io/part-of": "t4-cluster", "cluster.t4.dev/session": podName, } + desiredService := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: session.Namespace, Labels: labels}, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: labels, + Ports: []corev1.ServicePort{{Name: "host", Port: 8787, TargetPort: intstr.FromString("host"), Protocol: corev1.ProtocolTCP}}, + }, + } + if err := controllerutil.SetControllerReference(&session, &desiredService, r.Scheme); err != nil { + return ctrl.Result{}, err + } var service corev1.Service if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: serviceName}, &service); apierrors.IsNotFound(err) { - service = corev1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: session.Namespace, Labels: labels}, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Selector: labels, - Ports: []corev1.ServicePort{{Name: "host", Port: 8787, TargetPort: intstr.FromString("host"), Protocol: corev1.ProtocolTCP}}, - }, - } - if err := controllerutil.SetControllerReference(&session, &service, r.Scheme); err != nil { - return ctrl.Result{}, err - } + service = desiredService if err := r.Create(ctx, &service); err != nil && !apierrors.IsAlreadyExists(err) { return ctrl.Result{}, err } } else if err != nil { return ctrl.Result{}, err + } else if !metav1.IsControlledBy(&service, &session) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "Available", "ServiceOwnershipConflict", "deterministic session Service is not controlled by this session") + } else if !reflect.DeepEqual(service.Spec.Selector, desiredService.Spec.Selector) || !reflect.DeepEqual(service.Spec.Ports, desiredService.Spec.Ports) || !reflect.DeepEqual(service.Labels, desiredService.Labels) { + service.Spec.Selector = desiredService.Spec.Selector + service.Spec.Ports = desiredService.Spec.Ports + service.Labels = desiredService.Labels + if err := r.Update(ctx, &service); err != nil { + return ctrl.Result{}, err + } } + desiredPod, err := r.desiredPod(&session, workspace.Status.PVCName, podName, labels) + if err != nil { + return ctrl.Result{}, err + } + if err := controllerutil.SetControllerReference(&session, &desiredPod, r.Scheme); err != nil { + return ctrl.Result{}, err + } var pod corev1.Pod if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: podName}, &pod); apierrors.IsNotFound(err) { - pod = r.desiredPod(&session, workspace.Status.PVCName, podName, labels) - if err := controllerutil.SetControllerReference(&session, &pod, r.Scheme); err != nil { - return ctrl.Result{}, err - } + pod = desiredPod if err := r.Create(ctx, &pod); err != nil && !apierrors.IsAlreadyExists(err) { return ctrl.Result{}, err } } else if err != nil { return ctrl.Result{}, err + } else if !metav1.IsControlledBy(&pod, &session) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "Available", "PodOwnershipConflict", "deterministic session Pod is not controlled by this session") + } else if pod.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] != desiredPod.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] { + if err := r.Delete(ctx, &pod); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err := r.updateSessionPending(ctx, &session, podName, serviceName, "PodSpecChanged", "session Pod is being recreated to apply immutable desired state"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Second}, nil } original := session.Status @@ -161,7 +187,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{}, nil } -func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcName, podName string, labels map[string]string) corev1.Pod { +func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcName, podName string, labels map[string]string) (corev1.Pod, error) { falseValue := false trueValue := true runAsUser := int64(10001) @@ -245,7 +271,7 @@ func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcNa container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: "initial-prompt", MountPath: "/run/t4-initial-prompt", ReadOnly: true}) container.Env = append(container.Env, corev1.EnvVar{Name: "T4_INITIAL_PROMPT_FILE", Value: "/run/t4-initial-prompt/prompt"}) } - return corev1.Pod{ + pod := corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: session.Namespace, Labels: labels}, Spec: corev1.PodSpec{ AutomountServiceAccountToken: &falseValue, @@ -257,6 +283,29 @@ func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcNa Containers: []corev1.Container{container}, Volumes: volumes, }, } + spec, err := json.Marshal(pod.Spec) + if err != nil { + return corev1.Pod{}, fmt.Errorf("serialize desired session Pod: %w", err) + } + hash := sha256.Sum256(spec) + pod.Annotations = map[string]string{clusterv1alpha1.SessionPodSpecHashAnnotation: fmt.Sprintf("%x", hash)} + return pod, nil +} + +func (r *SessionReconciler) updateSessionPending(ctx context.Context, session *clusterv1alpha1.T4Session, podName, serviceName, reason, message string) error { + original := session.Status + if session.Status.Conditions != nil { + original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } + session.Status.ObservedGeneration = session.Generation + session.Status.PodName = podName + session.Status.ServiceName = serviceName + session.Status.Phase = clusterv1alpha1.InfrastructurePending + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, reason, message, session.Generation)) + if reflect.DeepEqual(original, session.Status) { + return nil + } + return r.Status().Update(ctx, session) } func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *clusterv1alpha1.T4Session) (ctrl.Result, error) { diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index 4c1dea76..a3ca0239 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -9,10 +9,10 @@ import ( corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - meta "k8s.io/apimachinery/pkg/api/meta" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -34,7 +34,9 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return r.reconcileDelete(ctx, &workspace) } if controllerutil.AddFinalizer(&workspace, clusterv1alpha1.WorkspaceFinalizer) { - if err := r.Update(ctx, &workspace); err != nil { return ctrl.Result{}, err } + if err := r.Update(ctx, &workspace); err != nil { + return ctrl.Result{}, err + } } var host clusterv1alpha1.T4ClusterHost @@ -45,7 +47,9 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{}, err } storageClassName := workspace.Spec.StorageClassName - if storageClassName == "" { storageClassName = host.Spec.StorageClassName } + if storageClassName == "" { + storageClassName = host.Spec.StorageClassName + } var storageClass storagev1.StorageClass if err := r.Get(ctx, types.NamespacedName{Name: storageClassName}, &storageClass); err != nil { if apierrors.IsNotFound(err) { @@ -67,27 +71,45 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques Name: pvcName, Namespace: workspace.Namespace, Labels: map[string]string{ "app.kubernetes.io/part-of": "t4-cluster", - "cluster.t4.dev/workspace": workspace.Name, + "cluster.t4.dev/workspace": workspace.Name, }, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, }, Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, StorageClassName: &storageClassName, - VolumeMode: &volumeMode, - Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: workspace.Spec.Size.DeepCopy()}}, + VolumeMode: &volumeMode, + Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: workspace.Spec.Size.DeepCopy()}}, }, } - if err := controllerutil.SetControllerReference(&workspace, &pvc, r.Scheme); err != nil { return ctrl.Result{}, err } - if err := r.Create(ctx, &pvc); err != nil && !apierrors.IsAlreadyExists(err) { return ctrl.Result{}, err } + if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyDelete { + if err := controllerutil.SetControllerReference(&workspace, &pvc, r.Scheme); err != nil { + return ctrl.Result{}, err + } + } + if err := r.Create(ctx, &pvc); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } } else if err != nil { return ctrl.Result{}, err - } else if !metav1.IsControlledBy(&pvc, &workspace) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "deterministic workspace PVC is not controlled by this workspace") + } else if !workspaceOwnsPVC(&workspace, &pvc) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "deterministic workspace PVC does not belong to this workspace") + } else if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyRetain && metav1.IsControlledBy(&pvc, &workspace) { + before := pvc.DeepCopy() + pvc.OwnerReferences = removeWorkspaceOwnerReference(pvc.OwnerReferences, workspace.UID) + if !reflect.DeepEqual(before.OwnerReferences, pvc.OwnerReferences) { + if err := r.Update(ctx, &pvc); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } } original := workspace.Status original.Capacity = workspace.Status.Capacity.DeepCopy() - if workspace.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) } + if workspace.Status.Conditions != nil { + original.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) + } workspace.Status.ObservedGeneration = workspace.Generation workspace.Status.PVCName = pvcName workspace.Status.PVCPhase = pvc.Status.Phase @@ -111,34 +133,73 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCBinding", "workspace PVC is waiting to bind", workspace.Generation)) } if !reflect.DeepEqual(original, workspace.Status) { - if err := r.Status().Update(ctx, &workspace); err != nil { return ctrl.Result{}, err } + if err := r.Status().Update(ctx, &workspace); err != nil { + return ctrl.Result{}, err + } + } + if pvc.Status.Phase != corev1.ClaimBound { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } - if pvc.Status.Phase != corev1.ClaimBound { return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } return ctrl.Result{}, nil } +func workspaceOwnsPVC(workspace *clusterv1alpha1.T4Workspace, pvc *corev1.PersistentVolumeClaim) bool { + if pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] != string(workspace.UID) { + return false + } + controller := metav1.GetControllerOf(pvc) + if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyDelete { + return controller != nil && controller.UID == workspace.UID + } + return controller == nil || controller.UID == workspace.UID +} + +func removeWorkspaceOwnerReference(references []metav1.OwnerReference, uid types.UID) []metav1.OwnerReference { + output := references[:0] + for _, reference := range references { + if reference.UID != uid { + output = append(output, reference) + } + } + return output +} + func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *clusterv1alpha1.T4Workspace) (ctrl.Result, error) { - if !controllerutil.ContainsFinalizer(workspace, clusterv1alpha1.WorkspaceFinalizer) { return ctrl.Result{}, nil } + if !controllerutil.ContainsFinalizer(workspace, clusterv1alpha1.WorkspaceFinalizer) { + return ctrl.Result{}, nil + } originalStatus := workspace.Status - if workspace.Status.Conditions != nil { originalStatus.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) } + if workspace.Status.Conditions != nil { + originalStatus.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) + } workspace.Status.ObservedGeneration = workspace.Generation workspace.Status.Phase = clusterv1alpha1.InfrastructureTerminating meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "Terminating", "workspace infrastructure is terminating", workspace.Generation)) if !reflect.DeepEqual(originalStatus, workspace.Status) { - if err := r.Status().Update(ctx, workspace); err != nil { return ctrl.Result{}, err } + if err := r.Status().Update(ctx, workspace); err != nil { + return ctrl.Result{}, err + } } var sessions clusterv1alpha1.T4SessionList - if err := r.List(ctx, &sessions, client.InNamespace(workspace.Namespace)); err != nil { return ctrl.Result{}, err } + if err := r.List(ctx, &sessions, client.InNamespace(workspace.Namespace)); err != nil { + return ctrl.Result{}, err + } remainingSessions := 0 for i := range sessions.Items { - if sessions.Items[i].Spec.WorkspaceRef == workspace.Name { remainingSessions++ } + if sessions.Items[i].Spec.WorkspaceRef == workspace.Name { + remainingSessions++ + } } if remainingSessions > 0 { before := workspace.Status - if workspace.Status.Conditions != nil { before.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) } + if workspace.Status.Conditions != nil { + before.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) + } meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "SessionsRemain", fmt.Sprintf("workspace deletion is waiting for %d session resources", remainingSessions), workspace.Generation)) if !reflect.DeepEqual(before, workspace.Status) { - if err := r.Status().Update(ctx, workspace); err != nil { return ctrl.Result{}, err } + if err := r.Status().Update(ctx, workspace); err != nil { + return ctrl.Result{}, err + } } return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } @@ -148,23 +209,29 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyRetain { if err == nil { before := pvc.DeepCopy() - refs := pvc.OwnerReferences[:0] - for _, ref := range pvc.OwnerReferences { - if ref.UID != workspace.UID { refs = append(refs, ref) } + pvc.OwnerReferences = removeWorkspaceOwnerReference(pvc.OwnerReferences, workspace.UID) + if pvc.Annotations == nil { + pvc.Annotations = map[string]string{} } - pvc.OwnerReferences = refs - if pvc.Annotations == nil { pvc.Annotations = map[string]string{} } pvc.Annotations[clusterv1alpha1.RetainedPVCAnnotation] = "true" if !reflect.DeepEqual(before.ObjectMeta, pvc.ObjectMeta) { - if err := r.Update(ctx, &pvc); err != nil { return ctrl.Result{}, err } + if err := r.Update(ctx, &pvc); err != nil { + return ctrl.Result{}, err + } } - } else if !apierrors.IsNotFound(err) { return ctrl.Result{}, err } + } else if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } } else { if err == nil { - if err := r.Delete(ctx, &pvc); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } + if err := r.Delete(ctx, &pvc); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: time.Second}, nil } - if !apierrors.IsNotFound(err) { return ctrl.Result{}, err } + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } } controllerutil.RemoveFinalizer(workspace, clusterv1alpha1.WorkspaceFinalizer) return ctrl.Result{}, r.Update(ctx, workspace) @@ -173,11 +240,15 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl func (r *WorkspaceReconciler) updateWorkspaceFailure(ctx context.Context, workspace *clusterv1alpha1.T4Workspace, conditionType, reason, message string) error { original := workspace.Status original.Capacity = workspace.Status.Capacity.DeepCopy() - if workspace.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) } + if workspace.Status.Conditions != nil { + original.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) + } workspace.Status.ObservedGeneration = workspace.Generation workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed meta.SetStatusCondition(&workspace.Status.Conditions, condition(conditionType, metav1.ConditionFalse, reason, message, workspace.Generation)) - if reflect.DeepEqual(original, workspace.Status) { return nil } + if reflect.DeepEqual(original, workspace.Status) { + return nil + } return r.Status().Update(ctx, workspace) } From b11370cef6b153246d8b5f3382892a284059340f Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 16:39:45 +0000 Subject: [PATCH 19/98] test(cluster): require rotating CI identity --- .../charttests/chart_contract_test.go | 17 +++++++++++ packages/cluster-server/test/config.test.ts | 20 +++++++++++++ .../cluster-server/test/woodpecker.test.ts | 29 +++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go index 3ca54344..ad171bc4 100644 --- a/packages/cluster-operator/charttests/chart_contract_test.go +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -129,6 +129,23 @@ func TestNetworkPoliciesDefaultDenyAndAllowOnlyDeclaredFlows(t *testing.T) { } } +func TestWoodpeckerCanUseRotatingProjectedServiceAccountIdentity(t *testing.T) { + values := append(enabledValues(), + "--set", "woodpecker.configMap=woodpecker-config", + "--set", "woodpecker.serviceAccountAudience=woodpecker-ci-trigger", + ) + output := helmTemplate(t, values...) + server := documentContainingKind(t, output, "Deployment", "name: release-name-t4-cluster-server") + assertContains(t, server, + "name: T4_WOODPECKER_TOKEN_FILE", + "/var/run/secrets/t4-ci/token", + "audience: woodpecker-ci-trigger", + "expirationSeconds: 600", + ) + host := documentContainingKind(t, output, "T4ClusterHost", "name: t4-cluster") + assertContains(t, host, "serviceAccountAudience: woodpecker-ci-trigger", "name: woodpecker-config") +} + func TestCRDsRemainExplicitAcrossUpgradeAndUninstall(t *testing.T) { withoutCRDs := helmTemplate(t, enabledValues()...) if strings.Contains(withoutCRDs, "kind: CustomResourceDefinition") { diff --git a/packages/cluster-server/test/config.test.ts b/packages/cluster-server/test/config.test.ts index d4f566c6..1254f9c8 100644 --- a/packages/cluster-server/test/config.test.ts +++ b/packages/cluster-server/test/config.test.ts @@ -41,6 +41,26 @@ describe("cluster server configuration", () => { await rm(directory, { recursive: true, force: true }); } }); + it("accepts exactly one server-side Woodpecker credential source", () => { + const common = { + ...BASE_ENV, + T4_WOODPECKER_BASE_URL: "https://ci.example.test", + T4_WOODPECKER_REPOSITORIES: '{"t4-code":{"slug":"owner/t4-code"}}', + }; + expect(clusterServerConfigFromEnv({ ...common, T4_WOODPECKER_TOKEN_FILE: "/var/run/secrets/t4-ci/token" }).woodpecker).toMatchObject({ + tokenFile: "/var/run/secrets/t4-ci/token", + }); + expect(clusterServerConfigFromEnv({ ...common, T4_WOODPECKER_TOKEN: "secret-from-kubernetes" }).woodpecker).toMatchObject({ + token: "secret-from-kubernetes", + }); + expect(() => clusterServerConfigFromEnv(common)).toThrow("complete"); + expect(() => clusterServerConfigFromEnv({ + ...common, + T4_WOODPECKER_TOKEN: "secret-from-kubernetes", + T4_WOODPECKER_TOKEN_FILE: "/var/run/secrets/t4-ci/token", + })).toThrow("exactly one"); + }); + }); describe("trusted cluster gateway proxy sources", () => { diff --git a/packages/cluster-server/test/woodpecker.test.ts b/packages/cluster-server/test/woodpecker.test.ts index fb335288..ce462045 100644 --- a/packages/cluster-server/test/woodpecker.test.ts +++ b/packages/cluster-server/test/woodpecker.test.ts @@ -1,4 +1,7 @@ import { describe, expect, it } from "vite-plus/test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { WoodpeckerProvider, mapWoodpeckerPipeline } from "../src/woodpecker.ts"; const correlation = { @@ -94,8 +97,34 @@ describe("bounded Woodpecker provider", () => { expect(requests.filter(request => request.init?.method === "POST")).toHaveLength(1); }); + it("re-reads a bounded projected credential for every provider request", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-woodpecker-token-")); + try { + const tokenFile = join(directory, "token"); + const seen: string[] = []; + const fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + seen.push(new Headers(init?.headers).get("authorization") ?? ""); + return Response.json([]); + }) as typeof globalThis.fetch; + await writeFile(tokenFile, "a".repeat(40), { mode: 0o400 }); + const woodpecker = new WoodpeckerProvider({ + baseUrl: "https://ci.example.test", + tokenFile, + repositories: { "t4-code": { slug: "owner/t4-code" } }, + fetch, + }); + await woodpecker.query(correlation); + await writeFile(tokenFile, "b".repeat(40), { mode: 0o400 }); + await woodpecker.query(correlation); + expect(seen).toEqual([`Bearer ${"a".repeat(40)}`, `Bearer ${"b".repeat(40)}`]); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + it("fails closed for unconfigured repositories, insecure provider URLs, oversized replies, and unknown correlation", async () => { expect(() => new WoodpeckerProvider({ baseUrl: "http://ci.example.test", token: "secret", repositories: {} })).toThrow("HTTPS"); + expect(() => new WoodpeckerProvider({ baseUrl: "https://ci.example.test", token: "secret", tokenFile: "/run/token", repositories: {} })).toThrow("exactly one"); const fetch = (async () => Response.json(Array.from({ length: 101 }, () => pipeline))) as typeof globalThis.fetch; await expect(provider(fetch).query(correlation)).rejects.toThrow("pipeline response limit"); await expect(provider(fetch).query({ ...correlation, repositoryId: "not-allowed" })).rejects.toThrow("not allowlisted"); From b5cb89540876ac21c8c8059d3ff276b7138a8720 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 16:41:45 +0000 Subject: [PATCH 20/98] feat(cluster): support projected CI identity --- .../crds/t4clusterhosts.cluster.t4.dev.yaml | 11 ++- .../t4-cluster/templates/clusterhost.yaml | 6 +- .../t4-cluster/templates/deployments.yaml | 18 +++++ deploy/charts/t4-cluster/values.schema.json | 19 ++++- deploy/charts/t4-cluster/values.yaml | 3 + .../cluster-operator/api/v1alpha1/types.go | 6 +- .../api/v1alpha1/zz_generated.deepcopy.go | 72 ++++++++++++++----- .../controllers/clusterhost_controller.go | 35 ++++++--- packages/cluster-server/src/config.ts | 19 +++-- packages/cluster-server/src/woodpecker.ts | 23 ++++-- 10 files changed, 168 insertions(+), 44 deletions(-) diff --git a/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml b/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml index 83ab3078..ea26aefa 100644 --- a/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml +++ b/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml @@ -39,8 +39,8 @@ spec: type: object required: [storageClassName, runtimeProfiles] x-kubernetes-validations: - - rule: "!has(self.ciProvider) || (self.ciProvider.secretRef.name != '' && self.ciProvider.configMapRef.name != '')" - message: CI provider requires non-empty Secret and ConfigMap references + - rule: "!has(self.ciProvider) || (self.ciProvider.configMapRef.name != '' && (has(self.ciProvider.secretRef) != has(self.ciProvider.serviceAccountAudience)))" + message: CI provider requires a ConfigMap and exactly one Secret or projected ServiceAccount identity - rule: "!has(self.allowedOrigins) || self.allowedOrigins.all(o, o.startsWith('https://') && !o.contains('@'))" message: allowed origins must be credential-free HTTPS origins properties: @@ -76,7 +76,7 @@ spec: default: 60 ciProvider: type: object - required: [secretRef, configMapRef] + required: [configMapRef] properties: secretRef: type: object @@ -87,6 +87,11 @@ spec: minLength: 1 maxLength: 253 pattern: '^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$' + serviceAccountAudience: + type: string + minLength: 1 + maxLength: 253 + pattern: '^[A-Za-z0-9][A-Za-z0-9._:/-]*$' configMapRef: type: object required: [name] diff --git a/deploy/charts/t4-cluster/templates/clusterhost.yaml b/deploy/charts/t4-cluster/templates/clusterhost.yaml index 14c37f38..88e9091f 100644 --- a/deploy/charts/t4-cluster/templates/clusterhost.yaml +++ b/deploy/charts/t4-cluster/templates/clusterhost.yaml @@ -14,10 +14,14 @@ spec: resyncSeconds: {{ .Values.clusterHost.projection.resyncSeconds }} allowedOrigins: {{- toYaml .Values.clusterHost.allowedOrigins | nindent 4 }} - {{- if and .Values.woodpecker.existingSecret .Values.woodpecker.configMap }} + {{- if and .Values.woodpecker.configMap (or .Values.woodpecker.existingSecret .Values.woodpecker.serviceAccountAudience) }} ciProvider: + {{- if .Values.woodpecker.existingSecret }} secretRef: name: {{ .Values.woodpecker.existingSecret }} + {{- else }} + serviceAccountAudience: {{ .Values.woodpecker.serviceAccountAudience | quote }} + {{- end }} configMapRef: name: {{ .Values.woodpecker.configMap }} {{- end }} diff --git a/deploy/charts/t4-cluster/templates/deployments.yaml b/deploy/charts/t4-cluster/templates/deployments.yaml index f453d688..55758460 100644 --- a/deploy/charts/t4-cluster/templates/deployments.yaml +++ b/deploy/charts/t4-cluster/templates/deployments.yaml @@ -228,6 +228,9 @@ spec: secretKeyRef: name: {{ .Values.woodpecker.existingSecret }} key: {{ .Values.woodpecker.tokenKey }} + {{- else if .Values.woodpecker.serviceAccountAudience }} + - name: T4_WOODPECKER_TOKEN_FILE + value: /var/run/secrets/t4-ci/token {{- end }} {{- if .Values.woodpecker.configMap }} - name: T4_WOODPECKER_BASE_URL @@ -280,6 +283,11 @@ spec: - name: kubernetes-api-access mountPath: /var/run/secrets/kubernetes.io/serviceaccount readOnly: true + {{- if .Values.woodpecker.serviceAccountAudience }} + - name: ci-identity + mountPath: /var/run/secrets/t4-ci + readOnly: true + {{- end }} volumes: - name: cluster-identity projected: @@ -302,6 +310,16 @@ spec: items: - key: ca.crt path: ca.crt + {{- if .Values.woodpecker.serviceAccountAudience }} + - name: ci-identity + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: + audience: {{ .Values.woodpecker.serviceAccountAudience | quote }} + expirationSeconds: {{ .Values.woodpecker.tokenExpirationSeconds }} + path: token + {{- end }} - name: temporary emptyDir: sizeLimit: 256Mi diff --git a/deploy/charts/t4-cluster/values.schema.json b/deploy/charts/t4-cluster/values.schema.json index f12cab16..222302af 100644 --- a/deploy/charts/t4-cluster/values.schema.json +++ b/deploy/charts/t4-cluster/values.schema.json @@ -65,7 +65,24 @@ "required": ["adminRWXStorageClass"], "properties": { "adminRWXStorageClass": { "type": "string", "maxLength": 63 } } }, - "woodpecker": { "type": "object" }, + "woodpecker": { + "type": "object", + "required": ["existingSecret", "tokenKey", "configMap", "baseURLKey", "repositoriesKey", "serviceAccountAudience", "tokenExpirationSeconds"], + "properties": { + "existingSecret": { "type": "string", "maxLength": 253 }, + "tokenKey": { "type": "string", "minLength": 1, "maxLength": 253 }, + "configMap": { "type": "string", "maxLength": 253 }, + "baseURLKey": { "type": "string", "minLength": 1, "maxLength": 253 }, + "repositoriesKey": { "type": "string", "minLength": 1, "maxLength": 253 }, + "serviceAccountAudience": { "type": "string", "maxLength": 253, "pattern": "^[A-Za-z0-9]*[A-Za-z0-9._:/-]*$" }, + "tokenExpirationSeconds": { "type": "integer", "minimum": 600, "maximum": 3600 } + }, + "oneOf": [ + { "properties": { "existingSecret": { "maxLength": 0 }, "configMap": { "maxLength": 0 }, "serviceAccountAudience": { "maxLength": 0 } } }, + { "properties": { "existingSecret": { "minLength": 1 }, "configMap": { "minLength": 1 }, "serviceAccountAudience": { "maxLength": 0 } } }, + { "properties": { "existingSecret": { "maxLength": 0 }, "configMap": { "minLength": 1 }, "serviceAccountAudience": { "minLength": 1 } } } + ] + }, "ingress": { "type": "object" }, "networkPolicy": { "type": "object" }, "observability": { "type": "object" } diff --git a/deploy/charts/t4-cluster/values.yaml b/deploy/charts/t4-cluster/values.yaml index e8006b7a..1d140dea 100644 --- a/deploy/charts/t4-cluster/values.yaml +++ b/deploy/charts/t4-cluster/values.yaml @@ -69,6 +69,9 @@ storage: woodpecker: existingSecret: "" tokenKey: token + # Alternative to existingSecret: a short-lived token accepted by an administrator-owned CI adapter. + serviceAccountAudience: "" + tokenExpirationSeconds: 600 configMap: "" baseURLKey: baseURL repositoriesKey: repositories diff --git a/packages/cluster-operator/api/v1alpha1/types.go b/packages/cluster-operator/api/v1alpha1/types.go index 49e18b77..aeafa45e 100644 --- a/packages/cluster-operator/api/v1alpha1/types.go +++ b/packages/cluster-operator/api/v1alpha1/types.go @@ -52,10 +52,10 @@ type ClusterServerProjectionSpec struct { } type CIProviderReferences struct { - SecretRef corev1.LocalObjectReference `json:"secretRef"` - ConfigMapRef corev1.LocalObjectReference `json:"configMapRef"` + SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"` + ConfigMapRef corev1.LocalObjectReference `json:"configMapRef"` + ServiceAccountAudience string `json:"serviceAccountAudience,omitempty"` } - type T4ClusterHostSpec struct { StorageClassName string `json:"storageClassName"` RuntimeProfiles []string `json:"runtimeProfiles"` diff --git a/packages/cluster-operator/api/v1alpha1/zz_generated.deepcopy.go b/packages/cluster-operator/api/v1alpha1/zz_generated.deepcopy.go index 820483b0..6befe822 100644 --- a/packages/cluster-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/packages/cluster-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -15,7 +15,9 @@ func (in *T4ClusterHost) DeepCopyInto(out *T4ClusterHost) { } func (in *T4ClusterHost) DeepCopy() *T4ClusterHost { - if in == nil { return nil } + if in == nil { + return nil + } out := new(T4ClusterHost) in.DeepCopyInto(out) return out @@ -28,12 +30,16 @@ func (in *T4ClusterHostList) DeepCopyInto(out *T4ClusterHostList) { in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { out.Items = make([]T4ClusterHost, len(in.Items)) - for i := range in.Items { in.Items[i].DeepCopyInto(&out.Items[i]) } + for i := range in.Items { + in.Items[i].DeepCopyInto(&out.Items[i]) + } } } func (in *T4ClusterHostList) DeepCopy() *T4ClusterHostList { - if in == nil { return nil } + if in == nil { + return nil + } out := new(T4ClusterHostList) in.DeepCopyInto(out) return out @@ -43,14 +49,26 @@ func (in *T4ClusterHostList) DeepCopyObject() runtime.Object { return in.DeepCop func (in *T4ClusterHostSpec) DeepCopyInto(out *T4ClusterHostSpec) { *out = *in - if in.RuntimeProfiles != nil { out.RuntimeProfiles = append([]string(nil), in.RuntimeProfiles...) } - if in.AllowedOrigins != nil { out.AllowedOrigins = append([]string(nil), in.AllowedOrigins...) } - if in.CIProvider != nil { out.CIProvider = &CIProviderReferences{SecretRef: in.CIProvider.SecretRef, ConfigMapRef: in.CIProvider.ConfigMapRef} } + if in.RuntimeProfiles != nil { + out.RuntimeProfiles = append([]string(nil), in.RuntimeProfiles...) + } + if in.AllowedOrigins != nil { + out.AllowedOrigins = append([]string(nil), in.AllowedOrigins...) + } + if in.CIProvider != nil { + out.CIProvider = &CIProviderReferences{ConfigMapRef: in.CIProvider.ConfigMapRef, ServiceAccountAudience: in.CIProvider.ServiceAccountAudience} + if in.CIProvider.SecretRef != nil { + secret := *in.CIProvider.SecretRef + out.CIProvider.SecretRef = &secret + } + } } func (in *T4ClusterHostStatus) DeepCopyInto(out *T4ClusterHostStatus) { *out = *in - if in.Conditions != nil { out.Conditions = append([]metav1.Condition(nil), in.Conditions...) } + if in.Conditions != nil { + out.Conditions = append([]metav1.Condition(nil), in.Conditions...) + } } func (in *T4Workspace) DeepCopyInto(out *T4Workspace) { @@ -61,7 +79,9 @@ func (in *T4Workspace) DeepCopyInto(out *T4Workspace) { } func (in *T4Workspace) DeepCopy() *T4Workspace { - if in == nil { return nil } + if in == nil { + return nil + } out := new(T4Workspace) in.DeepCopyInto(out) return out @@ -74,12 +94,16 @@ func (in *T4WorkspaceList) DeepCopyInto(out *T4WorkspaceList) { in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { out.Items = make([]T4Workspace, len(in.Items)) - for i := range in.Items { in.Items[i].DeepCopyInto(&out.Items[i]) } + for i := range in.Items { + in.Items[i].DeepCopyInto(&out.Items[i]) + } } } func (in *T4WorkspaceList) DeepCopy() *T4WorkspaceList { - if in == nil { return nil } + if in == nil { + return nil + } out := new(T4WorkspaceList) in.DeepCopyInto(out) return out @@ -89,14 +113,18 @@ func (in *T4WorkspaceList) DeepCopyObject() runtime.Object { return in.DeepCopy( func (in *T4WorkspaceSpec) DeepCopyInto(out *T4WorkspaceSpec) { *out = *in - if in.Repository != nil { out.Repository = &RepositoryMetadata{RepositoryID: in.Repository.RepositoryID, Ref: in.Repository.Ref, Commit: in.Repository.Commit} } + if in.Repository != nil { + out.Repository = &RepositoryMetadata{RepositoryID: in.Repository.RepositoryID, Ref: in.Repository.Ref, Commit: in.Repository.Commit} + } out.Size = in.Size.DeepCopy() } func (in *T4WorkspaceStatus) DeepCopyInto(out *T4WorkspaceStatus) { *out = *in out.Capacity = in.Capacity.DeepCopy() - if in.Conditions != nil { out.Conditions = append([]metav1.Condition(nil), in.Conditions...) } + if in.Conditions != nil { + out.Conditions = append([]metav1.Condition(nil), in.Conditions...) + } } func (in *T4Session) DeepCopyInto(out *T4Session) { @@ -107,7 +135,9 @@ func (in *T4Session) DeepCopyInto(out *T4Session) { } func (in *T4Session) DeepCopy() *T4Session { - if in == nil { return nil } + if in == nil { + return nil + } out := new(T4Session) in.DeepCopyInto(out) return out @@ -120,12 +150,16 @@ func (in *T4SessionList) DeepCopyInto(out *T4SessionList) { in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { out.Items = make([]T4Session, len(in.Items)) - for i := range in.Items { in.Items[i].DeepCopyInto(&out.Items[i]) } + for i := range in.Items { + in.Items[i].DeepCopyInto(&out.Items[i]) + } } } func (in *T4SessionList) DeepCopy() *T4SessionList { - if in == nil { return nil } + if in == nil { + return nil + } out := new(T4SessionList) in.DeepCopyInto(out) return out @@ -139,10 +173,14 @@ func (in *T4SessionSpec) DeepCopyInto(out *T4SessionSpec) { copy := *in.InitialPromptSecretRef out.InitialPromptSecretRef = © } - if in.CI != nil { out.CI = &SessionCIMetadata{RepositoryID: in.CI.RepositoryID, Ref: in.CI.Ref, Commit: in.CI.Commit} } + if in.CI != nil { + out.CI = &SessionCIMetadata{RepositoryID: in.CI.RepositoryID, Ref: in.CI.Ref, Commit: in.CI.Commit} + } } func (in *T4SessionStatus) DeepCopyInto(out *T4SessionStatus) { *out = *in - if in.Conditions != nil { out.Conditions = append([]metav1.Condition(nil), in.Conditions...) } + if in.Conditions != nil { + out.Conditions = append([]metav1.Condition(nil), in.Conditions...) + } } diff --git a/packages/cluster-operator/controllers/clusterhost_controller.go b/packages/cluster-operator/controllers/clusterhost_controller.go index 92e22250..1acc6c98 100644 --- a/packages/cluster-operator/controllers/clusterhost_controller.go +++ b/packages/cluster-operator/controllers/clusterhost_controller.go @@ -28,13 +28,17 @@ func (r *ClusterHostReconciler) Reconcile(ctx context.Context, request ctrl.Requ return ctrl.Result{}, client.IgnoreNotFound(err) } original := host.Status - if host.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), host.Status.Conditions...) } + if host.Status.Conditions != nil { + original.Conditions = append([]metav1.Condition(nil), host.Status.Conditions...) + } host.Status.ObservedGeneration = host.Generation var storageClass storagev1.StorageClass storageReady := true if err := r.Get(ctx, types.NamespacedName{Name: host.Spec.StorageClassName}, &storageClass); err != nil { - if !apierrors.IsNotFound(err) { return ctrl.Result{}, err } + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } storageReady = false meta.SetStatusCondition(&host.Status.Conditions, condition("StorageReady", metav1.ConditionFalse, ReasonStorageClassNotFound, "selected StorageClass does not exist", host.Generation)) } else if !storageClassAllowsRWX(storageClass.Annotations) { @@ -44,15 +48,24 @@ func (r *ClusterHostReconciler) Reconcile(ctx context.Context, request ctrl.Requ meta.SetStatusCondition(&host.Status.Conditions, condition("StorageReady", metav1.ConditionTrue, ReasonStorageReady, "selected StorageClass supports ReadWriteMany", host.Generation)) } - ciReady := host.Spec.CIProvider == nil || host.Spec.CIProvider.SecretRef.Name != "" && host.Spec.CIProvider.ConfigMapRef.Name != "" + ciReady := true + ciReason := "NotConfigured" + ciMessage := "CI provider is optional and not configured" + if host.Spec.CIProvider != nil { + hasSecret := host.Spec.CIProvider.SecretRef != nil && host.Spec.CIProvider.SecretRef.Name != "" + hasProjectedIdentity := host.Spec.CIProvider.ServiceAccountAudience != "" + ciReady = host.Spec.CIProvider.ConfigMapRef.Name != "" && hasSecret != hasProjectedIdentity + if ciReady { + ciReason, ciMessage = "ReferencesConfigured", "CI provider configuration and one server-side credential source are configured" + } else { + ciReason, ciMessage = "ReferencesIncomplete", "CI provider requires a ConfigMap and exactly one Secret or projected ServiceAccount identity" + } + } + status := metav1.ConditionFalse if ciReady { - reason := "NotConfigured" - message := "CI provider is optional and not configured" - if host.Spec.CIProvider != nil { reason, message = "ReferencesConfigured", "CI provider Secret and ConfigMap references are configured" } - meta.SetStatusCondition(&host.Status.Conditions, condition("CIReady", metav1.ConditionTrue, reason, message, host.Generation)) - } else { - meta.SetStatusCondition(&host.Status.Conditions, condition("CIReady", metav1.ConditionFalse, "ReferencesIncomplete", "CI provider requires both Secret and ConfigMap references", host.Generation)) + status = metav1.ConditionTrue } + meta.SetStatusCondition(&host.Status.Conditions, condition("CIReady", status, ciReason, ciMessage, host.Generation)) available := storageReady && len(host.Spec.RuntimeProfiles) > 0 if available { @@ -61,7 +74,9 @@ func (r *ClusterHostReconciler) Reconcile(ctx context.Context, request ctrl.Requ meta.SetStatusCondition(&host.Status.Conditions, condition("Available", metav1.ConditionFalse, "ConfigurationNotReady", "cluster host infrastructure configuration is not ready", host.Generation)) } if !reflect.DeepEqual(original, host.Status) { - if err := r.Status().Update(ctx, &host); err != nil { return ctrl.Result{}, err } + if err := r.Status().Update(ctx, &host); err != nil { + return ctrl.Result{}, err + } } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } diff --git a/packages/cluster-server/src/config.ts b/packages/cluster-server/src/config.ts index 9b4f3f71..cd89ea44 100644 --- a/packages/cluster-server/src/config.ts +++ b/packages/cluster-server/src/config.ts @@ -19,8 +19,9 @@ export interface ClusterServerConfig { readonly trustedProxyCidrs: readonly string[]; readonly woodpecker?: { readonly baseUrl: string; - readonly token: string; readonly repositories: Readonly>; + readonly token?: string; + readonly tokenFile?: string; }; } function required(env: Readonly>, name: string): string { @@ -109,8 +110,14 @@ export function clusterServerConfigFromEnv(env: Readonly>; readonly fetch?: typeof globalThis.fetch; } @@ -108,14 +111,20 @@ export function mapWoodpeckerPipeline( export class WoodpeckerProvider implements CiProvider { readonly name = "woodpecker" as const; readonly #baseUrl: string; - readonly #token: string; + readonly #token?: string; + readonly #tokenFile?: string; readonly #repositories: Readonly>; readonly #fetch: typeof globalThis.fetch; readonly #inflight = new Map>(); constructor(options: WoodpeckerProviderOptions) { this.#baseUrl = httpsBase(options.baseUrl); - this.#token = bounded(options.token, "Woodpecker token", 16_384); + if (Boolean(options.token) === Boolean(options.tokenFile)) throw new Error("Woodpecker provider requires exactly one credential source"); + if (options.token) this.#token = bounded(options.token, "Woodpecker token", 16_384); + if (options.tokenFile) { + if (!isAbsolute(options.tokenFile)) throw new Error("Woodpecker token file must be absolute"); + this.#tokenFile = options.tokenFile; + } const repositories: Record = {}; for (const [id, config] of Object.entries(options.repositories)) { bounded(id, "Woodpecker repository id", 256); @@ -182,8 +191,14 @@ export class WoodpeckerProvider implements CiProvider { bounded(value.ref, "CI ref", 512); bounded(value.commit, "CI commit", 512); } + async #credential(): Promise { + if (this.#token) return this.#token; + const token = (await readBoundedRegularFile(this.#tokenFile!, 16_384, "Woodpecker token")).trim(); + if (new TextEncoder().encode(token).byteLength < 32 || /\s/u.test(token)) throw new Error("Woodpecker token file is invalid"); + return token; + } async #json(path: string, init: RequestInit): Promise { - const headers = { Authorization: `Bearer ${this.#token}`, ...Object.fromEntries(new Headers(init.headers).entries()) }; + const headers = { Authorization: `Bearer ${await this.#credential()}`, ...Object.fromEntries(new Headers(init.headers).entries()) }; const response = await this.#fetch(`${this.#baseUrl}${path}`, { ...init, headers }); const declaredLength = Number(response.headers.get("content-length")); if (Number.isFinite(declaredLength) && declaredLength > WOODPECKER_MAX_RESPONSE_BYTES) throw new Error("Woodpecker response exceeds byte limit"); From a1e8e75fe1fd2372465a09dabd1d88a40a24b260 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 16:49:26 +0000 Subject: [PATCH 21/98] fix(cluster): separate CI API and web endpoints --- .../t4-cluster/templates/deployments.yaml | 5 +++++ deploy/charts/t4-cluster/values.schema.json | 3 ++- deploy/charts/t4-cluster/values.yaml | 1 + packages/cluster-server/src/config.ts | 3 +++ packages/cluster-server/src/woodpecker.ts | 17 +++++++++++++---- packages/cluster-server/test/woodpecker.test.ts | 17 +++++++++++++++++ 6 files changed, 41 insertions(+), 5 deletions(-) diff --git a/deploy/charts/t4-cluster/templates/deployments.yaml b/deploy/charts/t4-cluster/templates/deployments.yaml index 55758460..2be5fe83 100644 --- a/deploy/charts/t4-cluster/templates/deployments.yaml +++ b/deploy/charts/t4-cluster/templates/deployments.yaml @@ -238,6 +238,11 @@ spec: configMapKeyRef: name: {{ .Values.woodpecker.configMap }} key: {{ .Values.woodpecker.baseURLKey }} + - name: T4_WOODPECKER_WEB_BASE_URL + valueFrom: + configMapKeyRef: + name: {{ .Values.woodpecker.configMap }} + key: {{ .Values.woodpecker.webBaseURLKey }} - name: T4_WOODPECKER_REPOSITORIES valueFrom: configMapKeyRef: diff --git a/deploy/charts/t4-cluster/values.schema.json b/deploy/charts/t4-cluster/values.schema.json index 222302af..67c62273 100644 --- a/deploy/charts/t4-cluster/values.schema.json +++ b/deploy/charts/t4-cluster/values.schema.json @@ -67,13 +67,14 @@ }, "woodpecker": { "type": "object", - "required": ["existingSecret", "tokenKey", "configMap", "baseURLKey", "repositoriesKey", "serviceAccountAudience", "tokenExpirationSeconds"], + "required": ["existingSecret", "tokenKey", "configMap", "baseURLKey", "webBaseURLKey", "repositoriesKey", "serviceAccountAudience", "tokenExpirationSeconds"], "properties": { "existingSecret": { "type": "string", "maxLength": 253 }, "tokenKey": { "type": "string", "minLength": 1, "maxLength": 253 }, "configMap": { "type": "string", "maxLength": 253 }, "baseURLKey": { "type": "string", "minLength": 1, "maxLength": 253 }, "repositoriesKey": { "type": "string", "minLength": 1, "maxLength": 253 }, + "webBaseURLKey": { "type": "string", "minLength": 1, "maxLength": 253 }, "serviceAccountAudience": { "type": "string", "maxLength": 253, "pattern": "^[A-Za-z0-9]*[A-Za-z0-9._:/-]*$" }, "tokenExpirationSeconds": { "type": "integer", "minimum": 600, "maximum": 3600 } }, diff --git a/deploy/charts/t4-cluster/values.yaml b/deploy/charts/t4-cluster/values.yaml index 1d140dea..00cfcb75 100644 --- a/deploy/charts/t4-cluster/values.yaml +++ b/deploy/charts/t4-cluster/values.yaml @@ -74,6 +74,7 @@ woodpecker: tokenExpirationSeconds: 600 configMap: "" baseURLKey: baseURL + webBaseURLKey: webBaseURL repositoriesKey: repositories ingress: diff --git a/packages/cluster-server/src/config.ts b/packages/cluster-server/src/config.ts index cd89ea44..c5f34c53 100644 --- a/packages/cluster-server/src/config.ts +++ b/packages/cluster-server/src/config.ts @@ -19,6 +19,7 @@ export interface ClusterServerConfig { readonly trustedProxyCidrs: readonly string[]; readonly woodpecker?: { readonly baseUrl: string; + readonly webBaseUrl?: string; readonly repositories: Readonly>; readonly token?: string; readonly tokenFile?: string; @@ -111,6 +112,7 @@ export function clusterServerConfigFromEnv(env: Readonly>; @@ -38,11 +39,17 @@ interface WoodpeckerPipeline { readonly variables?: unknown; readonly stages?: unknown; } - function httpsBase(value: string): string { const url = new URL(value); if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash) - throw new Error("Woodpecker base URL must use HTTPS without credentials"); + throw new Error("Woodpecker web base URL must use HTTPS without credentials"); + return url.href.replace(/\/$/u, ""); +} +function apiBase(value: string): string { + const url = new URL(value); + const inCluster = url.protocol === "http:" && url.hostname.endsWith(".svc.cluster.local"); + if (!(url.protocol === "https:" || inCluster) || url.username || url.password || url.search || url.hash || !["", "/"].includes(url.pathname)) + throw new Error("Woodpecker API base URL must use HTTPS or an exact in-cluster Service URL without credentials"); return url.href.replace(/\/$/u, ""); } function bounded(value: unknown, name: string, max: number): string { @@ -111,6 +118,7 @@ export function mapWoodpeckerPipeline( export class WoodpeckerProvider implements CiProvider { readonly name = "woodpecker" as const; readonly #baseUrl: string; + readonly #webBaseUrl: string; readonly #token?: string; readonly #tokenFile?: string; readonly #repositories: Readonly>; @@ -118,7 +126,8 @@ export class WoodpeckerProvider implements CiProvider { readonly #inflight = new Map>(); constructor(options: WoodpeckerProviderOptions) { - this.#baseUrl = httpsBase(options.baseUrl); + this.#baseUrl = apiBase(options.baseUrl); + this.#webBaseUrl = options.webBaseUrl ? httpsBase(options.webBaseUrl) : httpsBase(options.baseUrl); if (Boolean(options.token) === Boolean(options.tokenFile)) throw new Error("Woodpecker provider requires exactly one credential source"); if (options.token) this.#token = bounded(options.token, "Woodpecker token", 16_384); if (options.tokenFile) { @@ -147,7 +156,7 @@ export class WoodpeckerProvider implements CiProvider { ref: correlation.ref, commit: correlation.commit, }; const number = positiveInteger(exact.number); - return mapWoodpeckerPipeline(exact, correlation, number ? `${this.#baseUrl}/repos/${repository.slug}/pipeline/${number}` : undefined); + return mapWoodpeckerPipeline(exact, correlation, number ? `${this.#webBaseUrl}/repos/${repository.slug}/pipeline/${number}` : undefined); } async run(correlation: CiCorrelation, signal?: AbortSignal): Promise { diff --git a/packages/cluster-server/test/woodpecker.test.ts b/packages/cluster-server/test/woodpecker.test.ts index ce462045..23c85f7e 100644 --- a/packages/cluster-server/test/woodpecker.test.ts +++ b/packages/cluster-server/test/woodpecker.test.ts @@ -122,6 +122,23 @@ describe("bounded Woodpecker provider", () => { } }); + it("permits only an exact in-cluster HTTP adapter while keeping deep links on HTTPS", async () => { + const woodpecker = new WoodpeckerProvider({ + baseUrl: "http://woodpecker-ci-trigger.linkedin-ci.svc.cluster.local:8080", + webBaseUrl: "https://woodpecker-ci-dev.tail.example.test", + token: "server-side-token", + repositories: { "t4-code": { slug: "owner/t4-code" } }, + fetch: (async () => Response.json([pipeline])) as typeof globalThis.fetch, + }); + expect((await woodpecker.query(correlation)).link).toBe("https://woodpecker-ci-dev.tail.example.test/repos/owner/t4-code/pipeline/42"); + expect(() => new WoodpeckerProvider({ + baseUrl: "http://ci.example.test", + webBaseUrl: "https://ci.example.test", + token: "token", + repositories: {}, + })).toThrow("in-cluster"); + }); + it("fails closed for unconfigured repositories, insecure provider URLs, oversized replies, and unknown correlation", async () => { expect(() => new WoodpeckerProvider({ baseUrl: "http://ci.example.test", token: "secret", repositories: {} })).toThrow("HTTPS"); expect(() => new WoodpeckerProvider({ baseUrl: "https://ci.example.test", token: "secret", tokenFile: "/run/token", repositories: {} })).toThrow("exactly one"); From 7e78dd9789443ed6b1e1027d8ee4bc95e1f3961c Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 16:57:26 +0000 Subject: [PATCH 22/98] fix(cluster): harden session state and replay boundaries --- .dockerignore | 14 +++++++++++++ .../session-runtime/session-entrypoint.sh | 21 ++++++++++--------- .../crds/t4workspaces.cluster.t4.dev.yaml | 7 ------- packages/client/src/omp-client-cursor.ts | 6 +++++- packages/client/src/omp-client-runtime.ts | 2 ++ .../cluster-operator/api/v1alpha1/types.go | 13 ++++++------ .../controllers/session_controller.go | 1 + .../controllers/workspace_controller.go | 5 +---- packages/cluster-server/src/gateway.ts | 11 ++++++++-- .../cluster-server/src/kubernetes-client.ts | 1 - .../src/kubernetes-projection.ts | 4 ++-- .../cluster-server/src/session-host-policy.ts | 5 +++-- packages/cluster-server/test/gateway.test.ts | 13 ++++++++++++ .../test/kubernetes-client.test.ts | 1 - .../test/kubernetes-projection.test.ts | 1 - .../test/session-host-policy.test.ts | 4 ++-- packages/host-service/src/server.ts | 1 + 17 files changed, 70 insertions(+), 40 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..30750596 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +.gitignore +.cluster-ci +artifacts +**/artifacts +node_modules +**/node_modules +.env +.env.* +*.log +coverage +**/coverage +test-results +playwright-report diff --git a/cluster/images/session-runtime/session-entrypoint.sh b/cluster/images/session-runtime/session-entrypoint.sh index 60f7389c..82f5e039 100755 --- a/cluster/images/session-runtime/session-entrypoint.sh +++ b/cluster/images/session-runtime/session-entrypoint.sh @@ -2,6 +2,7 @@ set -euo pipefail umask 077 +: "${T4_SESSION_STATE_ROOT:?T4_SESSION_STATE_ROOT is required}" : "${T4_AUTHORITY_STATE_DIR:?T4_AUTHORITY_STATE_DIR is required}" : "${T4_BROWSER_STATE_DIR:?T4_BROWSER_STATE_DIR is required}" : "${T4_CLUSTER_SERVER_SERVICE_ACCOUNT:?T4_CLUSTER_SERVER_SERVICE_ACCOUNT is required}" @@ -12,18 +13,18 @@ for projected_file in "${T4_KUBERNETES_TOKEN_PATH}" "${T4_KUBERNETES_CA_PATH}" " [[ -f "${projected_file}" && -r "${projected_file}" ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"kubernetes_api_projection"}' >&2; exit 64; } done -case "${T4_AUTHORITY_STATE_DIR}" in - /workspace/.t4/sessions/*/authority) ;; - *) echo '{"component":"session-runtime","result":"invalid_config","condition":"authority_state_path"}' >&2; exit 64 ;; -esac -case "${T4_BROWSER_STATE_DIR}" in - /workspace/.t4/sessions/*/browser) ;; - *) echo '{"component":"session-runtime","result":"invalid_config","condition":"browser_state_path"}' >&2; exit 64 ;; -esac +if [[ ! "${T4_SESSION_STATE_ROOT}" =~ ^/workspace/\.t4/sessions/[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ ]]; then + echo '{"component":"session-runtime","result":"invalid_config","condition":"session_state_path"}' >&2 + exit 64 +fi +[[ "${T4_AUTHORITY_STATE_DIR}" == "${T4_SESSION_STATE_ROOT}/authority" ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"authority_state_path"}' >&2; exit 64; } +[[ "${T4_BROWSER_STATE_DIR}" == "${T4_SESSION_STATE_ROOT}/browser" ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"browser_state_path"}' >&2; exit 64; } mkdir -p "${T4_AUTHORITY_STATE_DIR}" "${T4_BROWSER_STATE_DIR}" /run/t4 /tmp/t4 export HOME="${T4_AUTHORITY_STATE_DIR}" export DISPLAY="${DISPLAY:-:99}" +[[ "${DISPLAY}" =~ ^:([0-9]{1,3})$ ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"display"}' >&2; exit 64; } +display_socket="/tmp/.X11-unix/X${BASH_REMATCH[1]}" export T4_OMP_EXECUTABLE=/opt/t4/bin/omp export T4_WORKSPACE_ROOT=/workspace @@ -39,11 +40,11 @@ trap stop_children TERM INT EXIT Xvfb "${DISPLAY}" -screen 0 1920x1080x24 -nolisten tcp -ac & children+=("$!") for _ in $(seq 1 50); do - [[ -S /tmp/.X11-unix/X99 ]] && break + [[ -S "${display_socket}" ]] && break kill -0 "${children[0]}" 2>/dev/null || { echo '{"component":"session-runtime","result":"startup_failed","condition":"xvfb"}' >&2; exit 70; } sleep 0.1 done -[[ -S /tmp/.X11-unix/X99 ]] || { echo '{"component":"session-runtime","result":"startup_timeout","condition":"xvfb"}' >&2; exit 70; } +[[ -S "${display_socket}" ]] || { echo '{"component":"session-runtime","result":"startup_timeout","condition":"xvfb"}' >&2; exit 70; } fluxbox -display "${DISPLAY}" & children+=("$!") diff --git a/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml b/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml index 033c00c3..cdfa0396 100644 --- a/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml +++ b/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml @@ -41,8 +41,6 @@ spec: x-kubernetes-validations: - rule: "!has(self.repository) || !has(self.repository.commit) || (has(self.repository.ref) && self.repository.ref != '')" message: repository commit requires an explicit ref - - rule: "!has(self.storageClassName) || self.storageClassName != ''" - message: storageClassName must be omitted rather than empty properties: hostRef: type: string @@ -76,11 +74,6 @@ spec: minLength: 7 maxLength: 64 pattern: '^[0-9a-fA-F]+$' - storageClassName: - type: string - minLength: 1 - maxLength: 63 - pattern: '^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$' size: type: string minLength: 2 diff --git a/packages/client/src/omp-client-cursor.ts b/packages/client/src/omp-client-cursor.ts index 46ec95bd..a874d3d5 100644 --- a/packages/client/src/omp-client-cursor.ts +++ b/packages/client/src/omp-client-cursor.ts @@ -50,7 +50,11 @@ export class CursorJournal { this.bySession.set(key, cursor); if (this.records.size > MAX_SAVED) { const oldest = this.records.keys().next().value; - if (typeof oldest === "string") this.records.delete(oldest); + if (typeof oldest === "string") { + this.records.delete(oldest); + this.bySession.delete(oldest); + this.saved.delete(oldest); + } } if (this.store === undefined) return; const queue = this.queues.get(key) ?? { latest: undefined, running: false }; diff --git a/packages/client/src/omp-client-runtime.ts b/packages/client/src/omp-client-runtime.ts index 33d6e4e3..c728a26b 100644 --- a/packages/client/src/omp-client-runtime.ts +++ b/packages/client/src/omp-client-runtime.ts @@ -998,6 +998,8 @@ export class OmpClient { private acceptSnapshot(event: PublicEvent<"snapshot">): void { const frame = event.payload; const currentKey = sessionKey(String(frame.hostId), String(frame.sessionId)); + const previous = this.cursorJournal.bySession.get(currentKey); + if (previous?.epoch === frame.cursor.epoch && frame.cursor.seq <= previous.seq) return; this.desyncedSessions.delete(currentKey); this.epochValue = frame.cursor.epoch; this.cursorValue = frame.cursor; diff --git a/packages/cluster-operator/api/v1alpha1/types.go b/packages/cluster-operator/api/v1alpha1/types.go index aeafa45e..c1f37b6e 100644 --- a/packages/cluster-operator/api/v1alpha1/types.go +++ b/packages/cluster-operator/api/v1alpha1/types.go @@ -96,13 +96,12 @@ type RepositoryMetadata struct { } type T4WorkspaceSpec struct { - HostRef string `json:"hostRef"` - DisplayName string `json:"displayName"` - Owner string `json:"owner"` - Repository *RepositoryMetadata `json:"repository,omitempty"` - StorageClassName string `json:"storageClassName,omitempty"` - Size resource.Quantity `json:"size"` - RetentionPolicy RetentionPolicy `json:"retentionPolicy"` + HostRef string `json:"hostRef"` + DisplayName string `json:"displayName"` + Owner string `json:"owner"` + Repository *RepositoryMetadata `json:"repository,omitempty"` + Size resource.Quantity `json:"size"` + RetentionPolicy RetentionPolicy `json:"retentionPolicy"` } type T4WorkspaceStatus struct { diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 6a1be61a..feba2a8d 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -231,6 +231,7 @@ func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcNa {Name: "T4_KUBERNETES_NAMESPACE_PATH", Value: "/var/run/secrets/kubernetes.io/serviceaccount/namespace"}, {Name: "T4_SESSION_NAME", Value: session.Name}, {Name: "T4_WORKSPACE_ROOT", Value: "/workspace"}, + {Name: "T4_SESSION_STATE_ROOT", Value: "/workspace/.t4/sessions/" + stateID}, {Name: "T4_AUTHORITY_STATE_DIR", Value: "/workspace/.t4/sessions/" + stateID + "/authority"}, {Name: "T4_BROWSER_STATE_DIR", Value: "/workspace/.t4/sessions/" + stateID + "/browser"}, {Name: "T4_GUI_ENABLED", Value: fmt.Sprintf("%t", session.Spec.GUIEnabled)}, diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index a3ca0239..f284982b 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -46,10 +46,7 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques } return ctrl.Result{}, err } - storageClassName := workspace.Spec.StorageClassName - if storageClassName == "" { - storageClassName = host.Spec.StorageClassName - } + storageClassName := host.Spec.StorageClassName var storageClass storagev1.StorageClass if err := r.Get(ctx, types.NamespacedName{Name: storageClassName}, &storageClass); err != nil { if apierrors.IsNotFound(err) { diff --git a/packages/cluster-server/src/gateway.ts b/packages/cluster-server/src/gateway.ts index ac75aaed..bbe1beb1 100644 --- a/packages/cluster-server/src/gateway.ts +++ b/packages/cluster-server/src/gateway.ts @@ -16,7 +16,7 @@ import { type ServerFrame, } from "@t4-code/host-wire"; import type { Revision } from "@t4-code/host-wire"; -import { IdempotencyStore, type CommandOutcome } from "@t4-code/host-service"; +import { commandFeature, IdempotencyStore, type CommandOutcome } from "@t4-code/host-service"; import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; import { rewriteClientAddress, @@ -272,6 +272,11 @@ export class ClusterGateway { if (frame.hostId !== this.#projection.hostId) { client.send(errorResult(frame, "NOT_FOUND", "cluster host was not found")); return; } const capability = requiredCapability(frame.command); if (!capability || !grantedCapabilities.has(capability)) { client.send(errorResult(frame, "NOT_AUTHORIZED", "command capability was not granted")); return; } + if (frame.command === "session.list") { + const sessions = this.#projection.sessionRefs(principal); + client.send(successResult(frame, { cursor: this.#projection.sessionCursor, sessions, totalCount: sessions.length, truncated: false })); + return; + } if (frame.command === "workspace.list") { client.send(successResult(frame, this.#projection.workspaceList(principal))); return; } if (frame.command === "workspace.create") { await idempotent(frame, async () => { @@ -279,7 +284,7 @@ export class ClusterGateway { const created = await this.#mutations.createWorkspace(frame.commandId, args, principal); return successResult(frame, { workspace: { id: created.id, displayName: args.displayName, phase: "Pending", retentionPolicy: args.retentionPolicy, - ...(args.storageClass ? { storageClass: args.storageClass } : {}), capacity: args.capacity, + capacity: args.capacity, accessMode: "ReadWriteMany", revision: created.revision, } }); }); @@ -328,6 +333,8 @@ export class ClusterGateway { }); return; } + const feature = commandFeature(frame.command); + if (feature && !grantedFeatures.has(feature)) { client.send(errorResult(frame, "UNSUPPORTED_FEATURE", "command feature was not negotiated")); return; } if (frame.command.startsWith("preview.")) { const preview = typeof frame.args.previewId === "string" ? frame.args.previewId : undefined; if (preview && previewOwners.has(preview) && previewOwners.get(preview) !== frame.sessionId) { diff --git a/packages/cluster-server/src/kubernetes-client.ts b/packages/cluster-server/src/kubernetes-client.ts index fc4063d8..e8fa1325 100644 --- a/packages/cluster-server/src/kubernetes-client.ts +++ b/packages/cluster-server/src/kubernetes-client.ts @@ -288,7 +288,6 @@ export class KubernetesGatewayMutationBackend { displayName: args.displayName, retentionPolicy: args.retentionPolicy, size: args.capacity, - ...(args.storageClass ? { storageClassName: args.storageClass } : {}), ...(args.repository ? { repository: { repositoryId: args.repository.repositoryId, ...(args.repository.ref ? { ref: args.repository.ref } : {}), ...(args.repository.commit ? { commit: args.repository.commit } : {}) } } : {}), }, }; diff --git a/packages/cluster-server/src/kubernetes-projection.ts b/packages/cluster-server/src/kubernetes-projection.ts index b0a136bd..5fcc9780 100644 --- a/packages/cluster-server/src/kubernetes-projection.ts +++ b/packages/cluster-server/src/kubernetes-projection.ts @@ -331,7 +331,7 @@ export class ClusterInfrastructureProjection { } else return; const key = `${resource.kind}/${name}`; const version = text(resource.metadata.resourceVersion); - if (version && this.#versions.get(key) === version) return; + if (event.type !== "DELETED" && version && this.#versions.get(key) === version) return; if (version) { this.#versions.set(key, version); this.#resourceVersion = version; @@ -445,7 +445,7 @@ export class ClusterInfrastructureProjection { displayName: text(spec.displayName, resource.metadata.name).slice(0, 256), phase: resource.metadata.deletionTimestamp ? "Terminating" : categorical(status.phase, ["Pending", "Ready", "Failed", "Terminating", "Unknown"] as const, "Unknown"), retentionPolicy: spec.retentionPolicy === "Delete" ? "Delete" : "Retain", - ...(typeof spec.storageClassName === "string" ? { storageClass: spec.storageClassName.slice(0, 128) } : {}), + ...(typeof record(this.#host?.spec).storageClassName === "string" ? { storageClass: String(record(this.#host?.spec).storageClassName).slice(0, 128) } : {}), ...(typeof status.capacity === "string" ? { capacity: status.capacity.slice(0, 64) } : typeof spec.size === "string" ? { capacity: spec.size.slice(0, 64) } : {}), accessMode: "ReadWriteMany", revision: resourceRevision(resource), diff --git a/packages/cluster-server/src/session-host-policy.ts b/packages/cluster-server/src/session-host-policy.ts index f99a636e..9e201a26 100644 --- a/packages/cluster-server/src/session-host-policy.ts +++ b/packages/cluster-server/src/session-host-policy.ts @@ -123,8 +123,9 @@ export function sessionHostConfigFromEnv(env: Readonly 65_535) throw new Error("T4_SESSION_HOST_PORT is invalid"); return { diff --git a/packages/cluster-server/test/gateway.test.ts b/packages/cluster-server/test/gateway.test.ts index 052e03e5..b90d3492 100644 --- a/packages/cluster-server/test/gateway.test.ts +++ b/packages/cluster-server/test/gateway.test.ts @@ -134,6 +134,19 @@ describe("stateless omp-app cluster gateway", () => { expect((second.client.frames[1] as { sessions: unknown[] }).sessions).toHaveLength(2); }); + it("answers the host-scoped session.list bootstrap from the Kubernetes projection", async () => { + const value = setup(); + await value.connection.receive(hello); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "r-sessions", commandId: "c-sessions", hostId: "cluster:host-uid", + command: "session.list", args: {}, + }); + expect(value.client.frames.at(-1)).toMatchObject({ + type: "response", commandId: "c-sessions", ok: true, command: "session.list", + result: { cursor: { epoch: "replica-uid-1" }, totalCount: 2, truncated: false, sessions: [{ sessionId: "session-one" }, { sessionId: "session-two" }] }, + }); + }); + it("returns workspace bootstrap with its independent cursor and streams each watch revision once", async () => { const value = setup(); await value.connection.receive(hello); diff --git a/packages/cluster-server/test/kubernetes-client.test.ts b/packages/cluster-server/test/kubernetes-client.test.ts index 45fda881..4d7a25aa 100644 --- a/packages/cluster-server/test/kubernetes-client.test.ts +++ b/packages/cluster-server/test/kubernetes-client.test.ts @@ -103,7 +103,6 @@ describe("namespaced Kubernetes client", () => { displayName: "Created workspace", retentionPolicy: "Retain", size: "20Gi", - storageClassName: "t4-workspaces-rwx", repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, }, }); diff --git a/packages/cluster-server/test/kubernetes-projection.test.ts b/packages/cluster-server/test/kubernetes-projection.test.ts index ab2c9e08..8aa8c77a 100644 --- a/packages/cluster-server/test/kubernetes-projection.test.ts +++ b/packages/cluster-server/test/kubernetes-projection.test.ts @@ -36,7 +36,6 @@ const workspace = { owner: PRINCIPAL, displayName: "T4 code", retentionPolicy: "Retain", - storageClassName: "t4-workspaces-rwx", size: "20Gi", repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, }, diff --git a/packages/cluster-server/test/session-host-policy.test.ts b/packages/cluster-server/test/session-host-policy.test.ts index 068154fc..6ad253bc 100644 --- a/packages/cluster-server/test/session-host-policy.test.ts +++ b/packages/cluster-server/test/session-host-policy.test.ts @@ -90,7 +90,7 @@ describe("one-session pod host authority", () => { T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "release-t4-cluster-server", T4_SESSION_NAME: "session-one", T4_OMP_EXECUTABLE: "/opt/t4/bin/omp", - T4_SESSION_STATE_ROOT: "/workspace/.t4/sessions/session-one", + T4_SESSION_STATE_ROOT: "/workspace/.t4/sessions/a1b2c3d4", T4_SESSION_HOST_PORT: "8787", })).toEqual({ kubernetesBaseUrl: "https://10.96.0.1:443", @@ -100,7 +100,7 @@ describe("one-session pod host authority", () => { serverServiceAccountName: "release-t4-cluster-server", sessionName: "session-one", ompExecutable: "/opt/t4/bin/omp", - stateRoot: "/workspace/.t4/sessions/session-one", + stateRoot: "/workspace/.t4/sessions/a1b2c3d4", port: 8787, }); expect(() => sessionHostConfigFromEnv({ KUBERNETES_SERVICE_HOST: "10.96.0.1", T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "server", T4_SESSION_NAME: "bad/name" })).toThrow("T4_SESSION_NAME"); diff --git a/packages/host-service/src/server.ts b/packages/host-service/src/server.ts index f3714f01..c482ab5d 100644 --- a/packages/host-service/src/server.ts +++ b/packages/host-service/src/server.ts @@ -4403,6 +4403,7 @@ export class LocalAppserver implements AppserverHandle { const sends: Array> = []; for (const client of this.#clients) { if (!this.#hello.has(client) || !this.#clientCapabilities.get(client)?.has("sessions.read")) continue; + if (frame.type === "session.delta" && !this.#clientFeatures.get(client)?.has("session.delta")) continue; sends.push(this.#sendFrame(client, frame)); } await Promise.all(sends); From 1a7487a6bc1e2d9d58069db7095d928aea1b5339 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 17:20:07 +0000 Subject: [PATCH 23/98] fix(cluster): secure HA projections and idempotency --- .../src/ci-projection-runner.ts | 101 ++++++++++++++++++ packages/cluster-server/src/gateway.ts | 29 ++++- packages/cluster-server/src/index.ts | 1 + .../cluster-server/src/kubernetes-client.ts | 12 ++- .../src/kubernetes-projection.ts | 92 ++++++++++++++-- packages/cluster-server/src/main.ts | 20 +++- .../cluster-server/src/pod-host-router.ts | 28 ++++- packages/cluster-server/src/server.ts | 2 +- packages/cluster-server/src/woodpecker.ts | 42 ++++---- .../test/ci-projection-runner.test.ts | 90 ++++++++++++++++ packages/cluster-server/test/gateway.test.ts | 44 +++++++- .../test/kubernetes-client.test.ts | 16 ++- .../test/kubernetes-projection.test.ts | 38 ++++++- .../test/pod-host-router.test.ts | 87 +++++++++++---- .../cluster-server/test/woodpecker.test.ts | 44 ++++++-- packages/host-wire/src/cluster.ts | 58 ++++++---- .../host-wire/test/cluster-operator.test.ts | 18 +++- 17 files changed, 627 insertions(+), 95 deletions(-) create mode 100644 packages/cluster-server/src/ci-projection-runner.ts create mode 100644 packages/cluster-server/test/ci-projection-runner.test.ts diff --git a/packages/cluster-server/src/ci-projection-runner.ts b/packages/cluster-server/src/ci-projection-runner.ts new file mode 100644 index 00000000..dd0aec3a --- /dev/null +++ b/packages/cluster-server/src/ci-projection-runner.ts @@ -0,0 +1,101 @@ +import type { SessionCiState } from "@t4-code/host-wire"; +import type { ClusterInfrastructureProjection, SessionCiCorrelation } from "./kubernetes-projection.ts"; +import type { CiProvider } from "./woodpecker.ts"; + +export interface CiProjectionRunnerOptions { + readonly projection: ClusterInfrastructureProjection; + readonly provider: CiProvider; + readonly pollMs?: number; + readonly onError?: (error: unknown) => void; +} + +/** Reconstructs bounded CI observations from the configured provider on every replica. */ +export class CiProjectionRunner { + readonly #projection: ClusterInfrastructureProjection; + readonly #provider: CiProvider; + readonly #pollMs: number; + readonly #onError?: (error: unknown) => void; + #timer: ReturnType | undefined; + #inflight: Promise | undefined; + #unsubscribe: (() => void) | undefined; + #correlationFingerprint = ""; + #started = false; + + constructor(options: CiProjectionRunnerOptions) { + this.#projection = options.projection; + this.#provider = options.provider; + this.#pollMs = options.pollMs ?? 5_000; + if (!Number.isSafeInteger(this.#pollMs) || this.#pollMs < 10 || this.#pollMs > 60_000) + throw new Error("CI projection poll interval is invalid"); + this.#onError = options.onError; + } + + start(): void { + if (this.#started) throw new Error("CI projection runner already started"); + this.#started = true; + this.#unsubscribe = this.#projection.subscribeSessions(() => this.#scheduleWhenCorrelationsChange()); + this.#correlationFingerprint = this.#fingerprint(this.#projection.sessionCiCorrelations()); + this.#schedule(0); + } + + async stop(): Promise { + if (!this.#started) return; + this.#started = false; + this.#unsubscribe?.(); + this.#unsubscribe = undefined; + clearTimeout(this.#timer); + this.#timer = undefined; + await this.#inflight; + } + + #fingerprint(correlations: readonly SessionCiCorrelation[]): string { + return JSON.stringify([...correlations].sort((left, right) => left.sessionId.localeCompare(right.sessionId))); + } + + #scheduleWhenCorrelationsChange(): void { + const fingerprint = this.#fingerprint(this.#projection.sessionCiCorrelations()); + if (fingerprint === this.#correlationFingerprint) return; + this.#correlationFingerprint = fingerprint; + this.#schedule(0); + } + + #schedule(delayMs: number): void { + if (!this.#started) return; + clearTimeout(this.#timer); + this.#timer = setTimeout(() => { + this.#timer = undefined; + if (this.#inflight) return; + const operation = this.#refresh(); + this.#inflight = operation; + const finish = (): void => { + if (this.#inflight === operation) this.#inflight = undefined; + this.#schedule(this.#pollMs); + }; + void operation.then(finish, error => { + this.#onError?.(error); + finish(); + }); + }, delayMs); + } + + async #refresh(): Promise { + const correlations = this.#projection.sessionCiCorrelations(); + this.#correlationFingerprint = this.#fingerprint(correlations); + await Promise.all(correlations.map(async correlation => { + let state: SessionCiState; + try { + state = await this.#provider.query(correlation); + this.#projection.setSessionCiState(correlation.sessionId, state); + } catch (error) { + this.#projection.setSessionCiState(correlation.sessionId, { + provider: "woodpecker", + correlation: "unknown", + repositoryId: correlation.repositoryId, + ref: correlation.ref, + commit: correlation.commit, + }); + this.#onError?.(error); + } + })); + } +} diff --git a/packages/cluster-server/src/gateway.ts b/packages/cluster-server/src/gateway.ts index bbe1beb1..a88317c2 100644 --- a/packages/cluster-server/src/gateway.ts +++ b/packages/cluster-server/src/gateway.ts @@ -26,6 +26,7 @@ import { type PodHostRoute, } from "./pod-host-router.ts"; import type { CiProvider } from "./woodpecker.ts"; +import { KubernetesApiError } from "./kubernetes-client.ts"; const GATEWAY_CAPABILITIES = [ "sessions.read", "sessions.prompt", "sessions.control", "sessions.manage", "agents.control", @@ -164,7 +165,11 @@ export class ClusterGateway { } let output: ServerFrame; try { output = await operation(); } - catch { output = errorResult(command, "UPSTREAM_UNAVAILABLE", "cluster operation failed"); } + catch (error) { + output = error instanceof KubernetesApiError && error.status === 422 + ? errorResult(command, "INVALID_FRAME", "cluster request did not satisfy the Kubernetes API contract") + : errorResult(command, "UPSTREAM_UNAVAILABLE", "cluster operation failed"); + } const outcome: CommandOutcome = { frame: output }; this.#idempotency.complete(scopedId, command, outcome); client.send(output); @@ -221,7 +226,10 @@ export class ClusterGateway { } await idempotent(command, async () => { const session = command.sessionId!; - if (!this.#projection.ownsSession(session, principal)) return errorResult(command, "NOT_AUTHORIZED", "session is unavailable for this identity"); + if (!this.#projection.ownsSession(session, principal)) + return this.#projection.sessionExists(session) + ? errorResult(command, "NOT_AUTHORIZED", "session is unavailable for this identity") + : successResult(command, { deleted: true }); if (this.#projection.sessionRevision(session, principal) !== command.expectedRevision) return errorResult(command, "stale_revision", "session revision changed before deletion"); return successResult(command, await this.#mutations.deleteSession(command.commandId, session, principal)); @@ -303,7 +311,13 @@ export class ClusterGateway { return; } if (frame.command === "session.delete") { - if (!frame.sessionId || !this.#projection.ownsSession(frame.sessionId, principal)) { client.send(errorResult(frame, "NOT_AUTHORIZED", "session is unavailable for this identity")); return; } + if (!frame.sessionId) { client.send(errorResult(frame, "INVALID_FRAME", "session route is required")); return; } + if (!this.#projection.ownsSession(frame.sessionId, principal)) { + client.send(this.#projection.sessionExists(frame.sessionId) + ? errorResult(frame, "NOT_AUTHORIZED", "session is unavailable for this identity") + : successResult(frame, { deleted: true })); + return; + } if (this.#projection.sessionRevision(frame.sessionId, principal) !== frame.expectedRevision) { client.send(errorResult(frame, "stale_revision", "session revision changed before confirmation")); return; } if (frame.confirmationId !== undefined) { client.send(errorResult(frame, "confirmation_invalid", "command confirmation must use a confirm frame")); return; } for (const [id, pending] of challenges) if (pending.expiresAt < Date.now()) challenges.delete(id); @@ -329,13 +343,20 @@ export class ClusterGateway { const allowed = this.#projection.sessionCiSelection(frame.sessionId, principal); if (!allowed || allowed.repositoryId !== args.repositoryId || allowed.ref !== args.ref || allowed.commit !== args.commit) return errorResult(frame, "NOT_AUTHORIZED", "CI correlation is not declared by this session"); - return successResult(frame, await this.#ci.run({ sessionId: frame.sessionId, repositoryId: args.repositoryId, ref: args.ref, commit: args.commit })); + return successResult(frame, await this.#ci.run({ commandId: frame.commandId, sessionId: frame.sessionId, repositoryId: args.repositoryId, ref: args.ref, commit: args.commit })); }); return; } const feature = commandFeature(frame.command); if (feature && !grantedFeatures.has(feature)) { client.send(errorResult(frame, "UNSUPPORTED_FEATURE", "command feature was not negotiated")); return; } if (frame.command.startsWith("preview.")) { + if (!frame.sessionId) { client.send(errorResult(frame, "INVALID_FRAME", "session route is required")); return; } + const guiState = this.#projection.sessionGuiState(frame.sessionId, principal); + if (guiState === undefined) { client.send(errorResult(frame, "NOT_AUTHORIZED", "session is unavailable for this identity")); return; } + if (guiState !== "Ready") { + client.send(errorResult(frame, guiState === "Unavailable" ? "UNSUPPORTED_FEATURE" : "UPSTREAM_UNAVAILABLE", guiState === "Unavailable" ? "GUI is disabled for this session" : "session GUI is not ready")); + return; + } const preview = typeof frame.args.previewId === "string" ? frame.args.previewId : undefined; if (preview && previewOwners.has(preview) && previewOwners.get(preview) !== frame.sessionId) { client.send(errorResult(frame, "NOT_AUTHORIZED", "preview belongs to another session")); diff --git a/packages/cluster-server/src/index.ts b/packages/cluster-server/src/index.ts index 50b04ea9..4f81c495 100644 --- a/packages/cluster-server/src/index.ts +++ b/packages/cluster-server/src/index.ts @@ -1,3 +1,4 @@ +export * from "./ci-projection-runner.ts"; export * from "./config.ts"; export * from "./gateway.ts"; export * from "./kubernetes-client.ts"; diff --git a/packages/cluster-server/src/kubernetes-client.ts b/packages/cluster-server/src/kubernetes-client.ts index e8fa1325..d01f4d09 100644 --- a/packages/cluster-server/src/kubernetes-client.ts +++ b/packages/cluster-server/src/kubernetes-client.ts @@ -320,14 +320,22 @@ export class KubernetesGatewayMutationBackend { async deleteSession(_commandId: string, sessionId: string, principal: string): Promise<{ deleted: true }> { this.#validatePrincipal(principal); - const session = await this.#client.get("t4sessions", sessionId); + let session: KubernetesResource; + try { session = await this.#client.get("t4sessions", sessionId); } + catch (error) { + if (error instanceof KubernetesApiError && error.status === 404) return { deleted: true }; + throw error; + } if (object(session.spec).hostRef !== this.#hostRef) throw new Error("session belongs to another cluster host"); const workspaceRef = object(session.spec).workspaceRef; if (typeof workspaceRef !== "string") throw new Error("session workspace reference is invalid"); const workspace = await this.#client.get("t4workspaces", workspaceRef); this.#assertWorkspaceOwner(workspace, principal); if (!session.metadata.uid || !session.metadata.resourceVersion) throw new Error("session delete precondition is unavailable"); - await this.#client.delete("t4sessions", sessionId, { uid: session.metadata.uid, resourceVersion: session.metadata.resourceVersion }); + try { await this.#client.delete("t4sessions", sessionId, { uid: session.metadata.uid, resourceVersion: session.metadata.resourceVersion }); } + catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 404) throw error; + } return { deleted: true }; } diff --git a/packages/cluster-server/src/kubernetes-projection.ts b/packages/cluster-server/src/kubernetes-projection.ts index 5fcc9780..b0d3e019 100644 --- a/packages/cluster-server/src/kubernetes-projection.ts +++ b/packages/cluster-server/src/kubernetes-projection.ts @@ -1,6 +1,7 @@ import { CLUSTER_MAX_WORKSPACES as WIRE_MAX_WORKSPACES, decodeClusterCondition, + decodeSessionCiState, decodeSessionRef, decodeWorkspaceInfrastructureProjection, hostId, @@ -9,6 +10,7 @@ import { type Cursor, type HostId, type SessionRef, + type SessionCiState, type WorkspaceInfrastructureProjection, type WorkspaceListResult, type WorkspaceStateFrame, @@ -55,6 +57,12 @@ export interface ClusterInfrastructureProjectionOptions { type SessionListener = () => void; interface WorkspaceReplayItem { readonly frame: WorkspaceStateFrame; readonly owner?: string; } interface WorkspaceSubscription { readonly listener: (frame: WorkspaceStateFrame) => void; readonly principal?: string; } +export interface SessionCiCorrelation { + readonly sessionId: string; + readonly repositoryId: string; + readonly ref: string; + readonly commit: string; +} interface SessionAuthorityWaiter { readonly resolve: (value: SessionRef) => void; readonly reject: (reason: Error) => void; readonly timer: ReturnType; } function record(value: unknown): Record { @@ -96,6 +104,11 @@ function workspaceOwner(resource: KubernetesResource): string | undefined { if (typeof owner !== "string" || owner.length === 0 || new TextEncoder().encode(owner).byteLength > 256 || /[\u0000-\u001f\u007f]/u.test(owner)) return undefined; return owner; } +function ciSelection(resource: KubernetesResource): { readonly repositoryId: string; readonly ref: string; readonly commit: string } | undefined { + const ci = record(record(resource.spec).ci); + if (typeof ci.repositoryId !== "string" || typeof ci.ref !== "string" || typeof ci.commit !== "string") return undefined; + return { repositoryId: ci.repositoryId, ref: ci.ref, commit: ci.commit }; +} export function clusterHostIdFromUid(uid: string): HostId { if (!/^[A-Za-z0-9][A-Za-z0-9-]{0,127}$/u.test(uid)) throw new Error("T4ClusterHost UID is invalid"); @@ -112,6 +125,7 @@ export class ClusterInfrastructureProjection { #workspaces = new Map(); #sessions = new Map(); #authoritativeSessions = new Map(); + #ciStates = new Map(); #authorityWaiters = new Map>(); #versions = new Map(); #workspaceSequence = 0; @@ -163,6 +177,11 @@ export class ClusterInfrastructureProjection { this.#workspaces = nextWorkspaces; this.#sessions = nextSessions; for (const name of this.#authoritativeSessions.keys()) if (!nextSessions.has(name)) this.#authoritativeSessions.delete(name); + for (const [name, state] of this.#ciStates) { + const selected = nextSessions.get(name); + const ci = selected ? ciSelection(selected) : undefined; + if (!ci || ci.repositoryId !== state.repositoryId || ci.ref !== state.ref || ci.commit !== state.commit) this.#ciStates.delete(name); + } this.#versions.clear(); for (const resource of [input.host, ...input.workspaces, ...input.sessions]) this.#versions.set(`${resource.kind ?? "unknown"}/${resource.metadata.name}`, text(resource.metadata.resourceVersion)); @@ -232,7 +251,7 @@ export class ClusterInfrastructureProjection { if (typeof value !== "string") continue; try { const url = new URL(value); - if ((url.protocol === "https:" || url.protocol === "http:") && !url.username && !url.password && url.pathname === "/" && !url.search && !url.hash) + if (url.protocol === "https:" && !url.username && !url.password && url.pathname === "/" && !url.search && !url.hash) origins.push(url.origin); } catch {} } @@ -247,6 +266,18 @@ export class ClusterInfrastructureProjection { const resource = this.#sessions.get(session); return resource !== undefined && this.#ownsSessionResource(resource, principal); } + sessionExists(session: string): boolean { + return this.#sessions.has(session); + } + sessionGuiState(session: string, principal?: string): "Unavailable" | "Starting" | "Ready" | "Failed" | undefined { + const resource = this.#sessions.get(session); + if (!resource || principal !== undefined && !this.#ownsSessionResource(resource, principal)) return undefined; + if (record(resource.spec).guiEnabled !== true) return "Unavailable"; + const phase = record(resource.status).phase; + if (phase === "Running") return "Ready"; + if (phase === "Failed" || phase === "Terminating") return "Failed"; + return "Starting"; + } sessionEndpoints(): PodHostEndpoint[] { return [...this.#sessions.values()].flatMap(resource => { const service = serviceName(record(resource.status).serviceName); @@ -314,9 +345,26 @@ export class ClusterInfrastructureProjection { sessionCiSelection(session: string, principal?: string): { repositoryId: string; ref: string; commit: string } | undefined { const resource = this.#sessions.get(session); if (!resource || principal !== undefined && !this.#ownsSessionResource(resource, principal)) return undefined; - const ci = record(record(resource.spec).ci); - if (typeof ci.repositoryId !== "string" || typeof ci.ref !== "string" || typeof ci.commit !== "string") return undefined; - return { repositoryId: ci.repositoryId, ref: ci.ref, commit: ci.commit }; + return ciSelection(resource); + } + sessionCiCorrelations(): SessionCiCorrelation[] { + return [...this.#sessions.values()].flatMap(resource => { + const selected = ciSelection(resource); + return selected ? [{ sessionId: resource.metadata.name, ...selected }] : []; + }); + } + setSessionCiState(session: string, value: SessionCiState): void { + const resource = this.#sessions.get(session); + const selected = resource ? ciSelection(resource) : undefined; + if (!selected) return; + const state = decodeSessionCiState(value, `ci.${session}`); + if (state.repositoryId !== selected.repositoryId || state.ref !== selected.ref || state.commit !== selected.commit) + throw new Error("CI provider state does not match the declared session correlation"); + const previous = this.#ciStates.get(session); + if (previous && JSON.stringify(previous) === JSON.stringify(state)) return; + this.#ciStates.set(session, state); + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); } applyWatch(event: KubernetesWatchEvent): void { @@ -327,7 +375,24 @@ export class ClusterInfrastructureProjection { if (event.type === "DELETED") throw new Error("selected T4ClusterHost was deleted"); if (!resource.metadata.uid || clusterHostIdFromUid(resource.metadata.uid) !== this.hostId) throw new Error("selected T4ClusterHost identity changed"); } else if (resource.kind === "T4Workspace" || resource.kind === "T4Session") { - if (!this.#host || record(resource.spec).hostRef !== this.#host.metadata.name) return; + if (!this.#host) return; + if (record(resource.spec).hostRef !== this.#host.metadata.name) { + if (resource.kind === "T4Session") { + if (!this.#sessions.delete(name)) return; + this.#authoritativeSessions.delete(name); + this.#ciStates.delete(name); + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + return; + } + const existing = this.#workspaces.get(name); + if (!existing) return; + this.#workspaces.delete(name); + this.#publishWorkspace(existing, true, workspaceOwner(existing)); + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + return; + } } else return; const key = `${resource.kind}/${name}`; const version = text(resource.metadata.resourceVersion); @@ -343,9 +408,14 @@ export class ClusterInfrastructureProjection { if (event.type === "DELETED") { if (!this.#sessions.delete(name)) return; this.#authoritativeSessions.delete(name); + this.#ciStates.delete(name); } else { if (!this.#sessions.has(name) && this.#sessions.size >= this.maxSessions) throw new Error("session projection limit exceeded"); this.#sessions.set(name, resource); + const state = this.#ciStates.get(name); + const selected = ciSelection(resource); + if (state && (!selected || state.repositoryId !== selected.repositoryId || state.ref !== selected.ref || state.commit !== selected.commit)) + this.#ciStates.delete(name); } this.#sessionSequence++; for (const listener of this.#sessionListeners) listener(); @@ -405,7 +475,12 @@ export class ClusterInfrastructureProjection { const spec = record(resource.spec); const status = record(resource.status); const workspaceId = text(spec.workspaceRef, "unknown-workspace"); - const ci = record(spec.ci); + const selectedCi = ciSelection(resource); + const projectedCi = this.#ciStates.get(resource.metadata.name) ?? (selectedCi ? { + provider: "woodpecker" as const, + correlation: "unknown" as const, + ...selectedCi, + } : undefined); const condition = firstCondition(status); const guiEnabled = spec.guiEnabled === true; const infrastructurePhase = categorical(status.phase, ["Pending", "Running", "Failed", "Terminating", "Unknown"] as const, "Unknown"); @@ -429,10 +504,7 @@ export class ClusterInfrastructureProjection { ...(condition ? { condition } : {}), gui: { state: guiState }, }, - ...(ci.repositoryId && ci.ref && ci.commit ? { ci: { - provider: "woodpecker", correlation: "unknown", repositoryId: ci.repositoryId, - ref: ci.ref, commit: ci.commit, - } } : {}), + ...(projectedCi ? { ci: projectedCi } : {}), }, }, `session.${resource.metadata.name}`); } diff --git a/packages/cluster-server/src/main.ts b/packages/cluster-server/src/main.ts index 39ce43da..862c6ce4 100755 --- a/packages/cluster-server/src/main.ts +++ b/packages/cluster-server/src/main.ts @@ -1,4 +1,5 @@ #!/usr/bin/env bun +import { CiProjectionRunner } from "./ci-projection-runner.ts"; import { clusterServerConfigFromEnv, loadKubernetesCredentials } from "./config.ts"; import { ClusterGateway } from "./gateway.ts"; import { KubernetesApiClient, KubernetesGatewayMutationBackend } from "./kubernetes-client.ts"; @@ -37,6 +38,7 @@ export async function runClusterServer(env: Readonly(); const stop = (): void => stopped.resolve(); let authority: SessionAuthorityRunner | undefined; + let ciProjection: CiProjectionRunner | undefined; let servers: ClusterHttpServers | undefined; let signalsInstalled = false; try { @@ -49,6 +51,14 @@ export async function runClusterServer(env: Readonly logger.warn("CI projection refresh failed", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }), + }); + ciProjection.start(); + } const gateway = new ClusterGateway({ projection, connector, @@ -79,12 +89,16 @@ export async function runClusterServer(env: Readonly WebSocket; } @@ -34,9 +35,13 @@ export class WebSocketPodHostConnector implements PodHostConnector { readonly #identityTokenFile: string; readonly #timeoutMs: number; readonly #factory: (url: string) => WebSocket; + readonly #keepAliveMs: number; constructor(options: WebSocketPodHostConnectorOptions) { this.#identityTokenFile = options.identityTokenFile; this.#timeoutMs = options.openTimeoutMs ?? 10_000; + this.#keepAliveMs = options.keepAliveMs ?? 30_000; + if (!Number.isSafeInteger(this.#keepAliveMs) || this.#keepAliveMs < 10 || this.#keepAliveMs > 60_000) + throw new Error("pod host keepalive interval is invalid"); this.#factory = options.webSocketFactory ?? (url => new WebSocket(url)); } connect(endpoint: PodHostEndpoint, onFrame: (frame: ServerFrame) => void, onClose?: () => void): Promise { @@ -45,6 +50,8 @@ export class WebSocketPodHostConnector implements PodHostConnector { return new Promise((resolve, reject) => { let settled = false; let upstreamHostId: string | undefined; + let heartbeat: ReturnType | undefined; + let heartbeatNonce: string | undefined; const timer = setTimeout(() => { if (!settled) { settled = true; socket.close(1013, "upstream timeout"); reject(new Error("pod host handshake timed out")); } }, this.#timeoutMs); @@ -77,27 +84,42 @@ export class WebSocketPodHostConnector implements PodHostConnector { upstreamHostId = frame.hostId; settled = true; clearTimeout(timer); + heartbeat = setInterval(() => { + if (socket.readyState !== WebSocket.OPEN) return; + heartbeatNonce = `cluster-${Date.now().toString(36)}`; + try { socket.send(JSON.stringify({ v: "omp-app/1", type: "ping", nonce: heartbeatNonce, timestamp: new Date().toISOString() })); } + catch { socket.close(1011, "upstream keepalive failed"); } + }, this.#keepAliveMs); resolve({ get hostId() { return upstreamHostId; }, send: value => { if (socket.readyState !== WebSocket.OPEN) throw new Error("pod host connection is closed"); socket.send(JSON.stringify(value)); }, - close: (code, reason) => socket.close(code, reason), + close: (code, reason) => { + clearInterval(heartbeat); + socket.close(code, reason); + }, }); return; } if (!settled) throw new Error("pod host frame arrived before welcome"); + if (frame.type === "pong" && frame.nonce === heartbeatNonce) return; onFrame(frame); } catch (error) { - if (!settled) { settled = true; clearTimeout(timer); reject(error); } - else socket.close(1002, "invalid upstream frame"); + if (!settled) { + settled = true; + clearTimeout(timer); + socket.close(1002, "invalid upstream frame"); + reject(error); + } else socket.close(1002, "invalid upstream frame"); } }); socket.addEventListener("error", () => { if (!settled) { settled = true; clearTimeout(timer); reject(new Error("pod host connection failed")); } }); socket.addEventListener("close", () => { + clearInterval(heartbeat); if (!settled) { settled = true; clearTimeout(timer); reject(new Error("pod host connection closed")); } else onClose?.(); }); diff --git a/packages/cluster-server/src/server.ts b/packages/cluster-server/src/server.ts index 35565c69..229927e4 100644 --- a/packages/cluster-server/src/server.ts +++ b/packages/cluster-server/src/server.ts @@ -61,7 +61,7 @@ export function startClusterHttpServers(options: ClusterHttpServersOptions): Clu port: options.gatewayPort, fetch(request, server) { const url = new URL(request.url); - if (url.pathname !== "/omp-app/v1") return new Response("not found", { status: 404 }); + if (url.pathname !== "/v1/ws") return new Response("not found", { status: 404 }); if (request.method !== "GET") return new Response("method not allowed", { status: 405 }); if (draining) return new Response("draining", { status: 503 }); const remoteAddress = server.requestIP(request)?.address; diff --git a/packages/cluster-server/src/woodpecker.ts b/packages/cluster-server/src/woodpecker.ts index 74d245c0..8189bb02 100644 --- a/packages/cluster-server/src/woodpecker.ts +++ b/packages/cluster-server/src/woodpecker.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { isAbsolute } from "node:path"; import { readBoundedRegularFile } from "./config.ts"; import type { CiRunResult, SessionCiState } from "@t4-code/host-wire"; @@ -13,10 +14,11 @@ export interface CiCorrelation { readonly ref: string; readonly commit: string; } +export interface CiRunCorrelation extends CiCorrelation { readonly commandId: string; } export interface CiProvider { readonly name: "woodpecker"; query(correlation: CiCorrelation, signal?: AbortSignal): Promise; - run(correlation: CiCorrelation, signal?: AbortSignal): Promise; + run(correlation: CiRunCorrelation, signal?: AbortSignal): Promise; } export interface WoodpeckerRepositoryConfig { readonly slug: string; } export interface WoodpeckerProviderOptions { @@ -81,9 +83,11 @@ function timestamp(value: unknown): string | undefined { function object(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; } -function exactPipeline(pipeline: WoodpeckerPipeline, correlation: CiCorrelation): boolean { +function exactPipeline(pipeline: WoodpeckerPipeline, correlation: CiCorrelation, idempotencyKey?: string): boolean { const variables = object(pipeline.variables); - return pipeline.ref === correlation.ref && pipeline.commit === correlation.commit && variables.T4_SESSION_ID === correlation.sessionId; + return pipeline.ref === correlation.ref && pipeline.commit === correlation.commit && + variables.T4_SESSION_ID === correlation.sessionId && + (idempotencyKey === undefined || variables.T4_IDEMPOTENCY_KEY === idempotencyKey); } export function mapWoodpeckerPipeline( @@ -142,15 +146,18 @@ export class WoodpeckerProvider implements CiProvider { this.#repositories = Object.freeze(repositories); this.#fetch = options.fetch ?? globalThis.fetch; } - async query(correlation: CiCorrelation, signal?: AbortSignal): Promise { + return await this.#queryExact(correlation, signal); + } + + async #queryExact(correlation: CiCorrelation, signal?: AbortSignal, idempotencyKey?: string): Promise { const repository = this.#repository(correlation.repositoryId); this.#validateCorrelation(correlation); const query = new URLSearchParams({ ref: correlation.ref, commit: correlation.commit, limit: String(WOODPECKER_MAX_PIPELINES) }); const pipelines = await this.#json(`/api/repos/${encodeURIComponent(repository.slug)}/pipelines?${query}`, { signal }); if (!Array.isArray(pipelines)) throw new Error("Woodpecker pipeline response is invalid"); if (pipelines.length > WOODPECKER_MAX_PIPELINES) throw new Error("Woodpecker pipeline response limit exceeded"); - const exact = (pipelines as WoodpeckerPipeline[]).find(value => exactPipeline(value, correlation)); + const exact = (pipelines as WoodpeckerPipeline[]).find(value => exactPipeline(value, correlation, idempotencyKey)); if (!exact) return { provider: "woodpecker", correlation: "unknown", repositoryId: correlation.repositoryId, ref: correlation.ref, commit: correlation.commit, @@ -158,9 +165,8 @@ export class WoodpeckerProvider implements CiProvider { const number = positiveInteger(exact.number); return mapWoodpeckerPipeline(exact, correlation, number ? `${this.#webBaseUrl}/repos/${repository.slug}/pipeline/${number}` : undefined); } - - async run(correlation: CiCorrelation, signal?: AbortSignal): Promise { - const key = `${correlation.repositoryId}\u0000${correlation.ref}\u0000${correlation.commit}\u0000${correlation.sessionId}`; + async run(correlation: CiRunCorrelation, signal?: AbortSignal): Promise { + const key = `${correlation.repositoryId}\u0000${correlation.ref}\u0000${correlation.commit}\u0000${correlation.sessionId}\u0000${correlation.commandId}`; const pending = this.#inflight.get(key); if (pending) return await pending; if (this.#inflight.size >= WOODPECKER_MAX_INFLIGHT_TRIGGERS) throw new Error("Woodpecker trigger concurrency limit exceeded"); @@ -169,18 +175,22 @@ export class WoodpeckerProvider implements CiProvider { try { return await operation; } finally { if (this.#inflight.get(key) === operation) this.#inflight.delete(key); } } - - async #runExact(correlation: CiCorrelation, signal?: AbortSignal): Promise { - const existing = await this.query(correlation, signal); + async #runExact(correlation: CiRunCorrelation, signal?: AbortSignal): Promise { + const idempotencyKey = createHash("sha256") + .update(correlation.sessionId).update("\u0000").update(correlation.commandId) + .digest("base64url"); + const existing = await this.#queryExact(correlation, signal, idempotencyKey); if (existing.correlation === "exact" && existing.pipelineNumber !== undefined) return { triggered: false, pipelineNumber: existing.pipelineNumber, ...(existing.status ? { status: existing.status } : {}) }; const repository = this.#repository(correlation.repositoryId); const created = object(await this.#json(`/api/repos/${encodeURIComponent(repository.slug)}/pipelines`, { method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ ref: correlation.ref, commit: correlation.commit, event: "manual", variables: { T4_SESSION_ID: correlation.sessionId } }), + headers: { "content-type": "application/json", "idempotency-key": `t4-${idempotencyKey}` }, + body: JSON.stringify({ ref: correlation.ref, commit: correlation.commit, event: "manual", variables: { T4_SESSION_ID: correlation.sessionId, T4_IDEMPOTENCY_KEY: idempotencyKey } }), signal, })); + if (created.idempotencyKey !== idempotencyKey) + throw new Error("Woodpecker adapter did not confirm provider-side idempotency"); const mapped = mapWoodpeckerPipeline(created, correlation); return { triggered: true, @@ -189,16 +199,12 @@ export class WoodpeckerProvider implements CiProvider { }; } - #repository(id: string): WoodpeckerRepositoryConfig { - const repository = this.#repositories[id]; - if (!repository) throw new Error("Woodpecker repository is not allowlisted"); - return repository; - } #validateCorrelation(value: CiCorrelation): void { bounded(value.sessionId, "CI session id", 256); bounded(value.repositoryId, "CI repository id", 256); bounded(value.ref, "CI ref", 512); bounded(value.commit, "CI commit", 512); + if ("commandId" in value) bounded(value.commandId, "CI command id", 256); } async #credential(): Promise { if (this.#token) return this.#token; diff --git a/packages/cluster-server/test/ci-projection-runner.test.ts b/packages/cluster-server/test/ci-projection-runner.test.ts new file mode 100644 index 00000000..1466eefa --- /dev/null +++ b/packages/cluster-server/test/ci-projection-runner.test.ts @@ -0,0 +1,90 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; +import { hostId, projectId, revision, sessionId, type SessionCiState, type SessionRef } from "@t4-code/host-wire"; +import { CiProjectionRunner } from "../src/ci-projection-runner.ts"; +import { ClusterInfrastructureProjection } from "../src/kubernetes-projection.ts"; +import type { CiProvider } from "../src/woodpecker.ts"; + +const correlation = { + sessionId: "session-one", + repositoryId: "t4-code", + ref: "refs/heads/agent/t4-cluster-operator", + commit: "0123456789abcdef0123456789abcdef01234567", +}; +const exactState: SessionCiState = { + provider: "woodpecker", + correlation: "exact", + repositoryId: correlation.repositoryId, + ref: correlation.ref, + commit: correlation.commit, + pipelineNumber: 42, + status: "running", + currentStage: "cluster-server-tests", +}; +const authority: SessionRef = { + hostId: hostId("pod-host"), + sessionId: sessionId("omp-session"), + project: { projectId: projectId("workspace-one") }, + revision: revision("authority-r1"), + title: "Cluster session", + status: "active", + updatedAt: "2026-07-20T00:00:00.000Z", +}; + +function projection(): ClusterInfrastructureProjection { + const value = new ClusterInfrastructureProjection({ epoch: "replica-one", namespace: "development" }); + value.replace({ + host: { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4ClusterHost", metadata: { name: "primary", uid: "host-uid", resourceVersion: "1" }, spec: {} }, + workspaces: [{ apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", metadata: { name: "workspace-one", resourceVersion: "2" }, spec: { hostRef: "primary", owner: "owner@example.com", displayName: "Workspace", retentionPolicy: "Retain", size: "20Gi" }, status: { phase: "Ready" } }], + sessions: [{ apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", metadata: { name: correlation.sessionId, resourceVersion: "3" }, spec: { hostRef: "primary", workspaceRef: "workspace-one", title: "Session", runtimeProfile: "default", guiEnabled: true, ci: { repositoryId: correlation.repositoryId, ref: correlation.ref, commit: correlation.commit } }, status: { phase: "Running", serviceName: "session-one" } }], + resourceVersion: "3", + }); + value.setSessionAuthority(correlation.sessionId, authority); + return value; +} + +afterEach(() => vi.useRealTimers()); + +describe("CI projection runner", () => { + it("reconstructs exact provider state and refreshes stage transitions", async () => { + vi.useFakeTimers(); + const state = projection(); + let current = exactState; + const provider: CiProvider = { + name: "woodpecker", + query: async () => current, + run: async () => ({ triggered: false }), + }; + const runner = new CiProjectionRunner({ projection: state, provider, pollMs: 10 }); + runner.start(); + await vi.advanceTimersByTimeAsync(0); + expect(state.sessionRef(correlation.sessionId)?.liveState?.ci).toEqual(exactState); + current = { ...exactState, status: "success", currentStage: "publish" }; + await vi.advanceTimersByTimeAsync(10); + expect(state.sessionRef(correlation.sessionId)?.liveState?.ci).toMatchObject({ status: "success", currentStage: "publish" }); + await runner.stop(); + }); + + it("fails closed to unknown correlation when the provider is unavailable", async () => { + vi.useFakeTimers(); + const state = projection(); + state.setSessionCiState(correlation.sessionId, exactState); + const errors: unknown[] = []; + const provider: CiProvider = { + name: "woodpecker", + query: async () => { throw new Error("provider unavailable"); }, + run: async () => ({ triggered: false }), + }; + const runner = new CiProjectionRunner({ projection: state, provider, pollMs: 10, onError: error => errors.push(error) }); + runner.start(); + await vi.advanceTimersByTimeAsync(0); + expect(state.sessionRef(correlation.sessionId)?.liveState?.ci).toEqual({ + provider: "woodpecker", + correlation: "unknown", + repositoryId: correlation.repositoryId, + ref: correlation.ref, + commit: correlation.commit, + }); + expect(errors).toHaveLength(1); + await runner.stop(); + }); +}); diff --git a/packages/cluster-server/test/gateway.test.ts b/packages/cluster-server/test/gateway.test.ts index b90d3492..9c607f94 100644 --- a/packages/cluster-server/test/gateway.test.ts +++ b/packages/cluster-server/test/gateway.test.ts @@ -10,6 +10,7 @@ import { } from "@t4-code/host-wire"; import { ClusterGateway, type GatewayClient, type GatewayMutationBackend } from "../src/gateway.ts"; import { ClusterInfrastructureProjection } from "../src/kubernetes-projection.ts"; +import { KubernetesApiError } from "../src/kubernetes-client.ts"; import type { PodHostConnection, PodHostConnector, PodHostRoute } from "../src/pod-host-router.ts"; const PRINCIPAL = "owner@example.com"; @@ -204,7 +205,7 @@ describe("stateless omp-app cluster gateway", () => { await value.connection.receive(hello); await value.connection.receive({ v: "omp-app/1", type: "command", requestId: "request-owner", commandId: "command-owner", - hostId: "cluster:host-uid", sessionId: "session-one", command: "preview.state", args: { previewId: "preview-one" }, + hostId: "cluster:host-uid", sessionId: "session-one", command: "preview.state", args: {}, }); value.connector.onFrame?.({ v: "omp-app/1", type: "preview.state", hostId: "upstream" as never, sessionId: "omp-private-one" as never, @@ -213,7 +214,7 @@ describe("stateless omp-app cluster gateway", () => { }); await value.connection.receive({ v: "omp-app/1", type: "command", requestId: "request-preview", commandId: "command-preview", - hostId: "cluster:host-uid", sessionId: "session-two", command: "preview.state", args: { previewId: "preview-one" }, + hostId: "cluster:host-uid", sessionId: "session-two", command: "preview.activate", args: { previewId: "preview-one" }, }); expect(value.connector.routes).toHaveLength(1); expect(value.client.frames.at(-1)).toMatchObject({ type: "response", commandId: "command-preview", ok: false, error: { code: "NOT_AUTHORIZED" } }); @@ -235,4 +236,43 @@ describe("stateless omp-app cluster gateway", () => { expect.objectContaining({ type: "response", commandId: "command-create", requestId: "request-create-retry", ok: true }), ]); }); + + it("fails closed when the session infrastructure does not authorize GUI access", async () => { + const value = setup(); + value.projection.applyWatch({ + type: "MODIFIED", + object: { ...session("session-one", "omp-private-one"), metadata: { ...session("session-one", "omp-private-one").metadata, resourceVersion: "30" }, spec: { ...session("session-one", "omp-private-one").spec, guiEnabled: false } }, + }); + await value.connection.receive(hello); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "request-disabled-gui", commandId: "command-disabled-gui", + hostId: "cluster:host-uid", sessionId: "session-one", command: "preview.state", args: {}, + }); + expect(value.connector.routes).toHaveLength(0); + expect(value.client.frames.at(-1)).toMatchObject({ type: "response", ok: false, error: { code: "UNSUPPORTED_FEATURE", message: "GUI is disabled for this session" } }); + }); + + it("treats an already absent session delete as an idempotent success", async () => { + const value = setup(); + value.projection.applyWatch({ type: "DELETED", object: session("session-one", "omp-private-one") }); + await value.connection.receive(hello); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "request-delete-replay", commandId: "command-delete-replay", + hostId: "cluster:host-uid", sessionId: "session-one", command: "session.delete", expectedRevision: "authority-r1", args: {}, + }); + expect(value.client.frames.at(-1)).toMatchObject({ type: "response", ok: true, result: { deleted: true } }); + expect(value.mutations.sessionDeletes).toBe(0); + }); + + it("reports Kubernetes schema rejection as a client contract error", async () => { + const value = setup(); + value.mutations.createWorkspace = async () => { throw new KubernetesApiError(422, "invalid"); }; + await value.connection.receive(hello); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "request-invalid", commandId: "command-invalid", + hostId: "cluster:host-uid", command: "workspace.create", + args: { displayName: "Created", retentionPolicy: "Retain", capacity: "20Gi" }, + }); + expect(value.client.frames.at(-1)).toMatchObject({ type: "response", ok: false, error: { code: "INVALID_FRAME" } }); + }); }); diff --git a/packages/cluster-server/test/kubernetes-client.test.ts b/packages/cluster-server/test/kubernetes-client.test.ts index 4d7a25aa..b3685941 100644 --- a/packages/cluster-server/test/kubernetes-client.test.ts +++ b/packages/cluster-server/test/kubernetes-client.test.ts @@ -77,8 +77,7 @@ describe("namespaced Kubernetes client", () => { displayName: "Created workspace", retentionPolicy: "Retain" as const, capacity: "20Gi", - storageClass: "t4-workspaces-rwx", - repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, + repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, }; await backend.createWorkspace("command-create-workspace", workspaceArgs, PRINCIPAL); const workspaceBody = JSON.parse(String(values.requests[0]?.init?.body)); @@ -103,7 +102,7 @@ describe("namespaced Kubernetes client", () => { displayName: "Created workspace", retentionPolicy: "Retain", size: "20Gi", - repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, + repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, }, }); expect(JSON.stringify(workspaceBody)).not.toContain("token"); @@ -114,7 +113,7 @@ describe("namespaced Kubernetes client", () => { title: "Task", runtimeProfile: "omp-17.0.5", guiEnabled: true, - ci: { provider: "woodpecker", repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, + ci: { provider: "woodpecker", repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, }, PRINCIPAL); const sessionBody = JSON.parse(String(values.requests[2]?.init?.body)); expect(sessionBody).toMatchObject({ @@ -151,6 +150,15 @@ describe("namespaced Kubernetes client", () => { }); await expect(conflictingBackend.createWorkspace("command-one", args, PRINCIPAL)).rejects.toThrow("idempotency conflict"); }); + + it("treats an already absent session as a successful idempotent delete", async () => { + const fetch = (async () => Response.json({ reason: "NotFound" }, { status: 404 })) as typeof globalThis.fetch; + const backend = new KubernetesGatewayMutationBackend({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch }), + hostRef: "primary", + }); + expect(await backend.deleteSession("command-delete", "session-gone", PRINCIPAL)).toEqual({ deleted: true }); + }); }); describe("Kubernetes projected identity review", () => { diff --git a/packages/cluster-server/test/kubernetes-projection.test.ts b/packages/cluster-server/test/kubernetes-projection.test.ts index 8aa8c77a..058e0ab7 100644 --- a/packages/cluster-server/test/kubernetes-projection.test.ts +++ b/packages/cluster-server/test/kubernetes-projection.test.ts @@ -37,7 +37,7 @@ const workspace = { displayName: "T4 code", retentionPolicy: "Retain", size: "20Gi", - repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, + repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, }, status: { observedGeneration: 3, @@ -57,7 +57,7 @@ const session = { title: "Cluster task", runtimeProfile: "omp-17.0.5", guiEnabled: true, - ci: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, + ci: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, }, status: { observedGeneration: 5, @@ -137,6 +137,40 @@ describe("Kubernetes infrastructure projection", () => { expect(projection.sessionRefs()).toEqual([]); }); + it("accepts only HTTPS browser origins and exposes GUI infrastructure authorization", () => { + const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development" }); + projection.replace({ + host: { ...host, spec: { allowedOrigins: ["https://t4.tailnet.example", "http://insecure.example", "https://user:secret@t4.example"] } }, + workspaces: [workspace], + sessions: [session], + resourceVersion: "102", + }); + expect(projection.allowedOrigins()).toEqual(["https://t4.tailnet.example"]); + expect(projection.sessionGuiState("session-one", PRINCIPAL)).toBe("Ready"); + projection.applyWatch({ + type: "MODIFIED", + object: { ...session, metadata: { ...session.metadata, resourceVersion: "105" }, spec: { ...session.spec, guiEnabled: false } }, + }); + expect(projection.sessionGuiState("session-one", PRINCIPAL)).toBe("Unavailable"); + expect(projection.sessionGuiState("session-one", "other@example.com")).toBeUndefined(); + }); + + it("removes cached resources immediately if a legacy watch migrates them to another host", () => { + const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development" }); + projection.replace({ host, workspaces: [workspace], sessions: [session], resourceVersion: "102" }); + projection.setSessionAuthority("session-one", authority("omp-session-private")); + projection.applyWatch({ + type: "MODIFIED", + object: { ...session, metadata: { ...session.metadata, resourceVersion: "106" }, spec: { ...session.spec, hostRef: "another-host" } }, + }); + expect(projection.sessionRoute("session-one")).toBeUndefined(); + projection.applyWatch({ + type: "MODIFIED", + object: { ...workspace, metadata: { ...workspace.metadata, resourceVersion: "107" }, spec: { ...workspace.spec, hostRef: "another-host" } }, + }); + expect(projection.workspaceList(PRINCIPAL).workspaces).toEqual([]); + }); + it("fails closed at explicit projection limits", () => { const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", diff --git a/packages/cluster-server/test/pod-host-router.test.ts b/packages/cluster-server/test/pod-host-router.test.ts index a5547455..2a068d21 100644 --- a/packages/cluster-server/test/pod-host-router.test.ts +++ b/packages/cluster-server/test/pod-host-router.test.ts @@ -1,12 +1,30 @@ -import { expect, it } from "vite-plus/test"; +import { expect, it, vi } from "vite-plus/test"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { WebSocketPodHostConnector } from "../src/pod-host-router.ts"; +const welcome = { + v: "omp-app/1", + type: "welcome", + selectedProtocol: "omp-app/1", + hostId: "host-a", + ompVersion: "17.0.5", + ompBuild: "test", + appserverVersion: "0.1.30", + appserverBuild: "cluster-session", + epoch: "pod-epoch", + grantedCapabilities: [], + grantedFeatures: [], + negotiatedLimits: {}, + authentication: "paired", + resumed: false, +} as const; class MemoryWebSocket { readyState = WebSocket.CONNECTING; readonly sent = Promise.withResolvers(); + readonly sentValues: string[] = []; + readonly closes: Array<{ readonly code?: number; readonly reason?: string }> = []; readonly #listeners: Record void>> = {}; addEventListener(type: string, listener: EventListenerOrEventListenerObject): void { const callback = typeof listener === "function" ? listener : event => listener.handleEvent(event as Event); @@ -14,8 +32,9 @@ class MemoryWebSocket { } send(value: string | ArrayBufferLike | Blob | ArrayBufferView): void { this.sent.resolve(String(value)); + this.sentValues.push(String(value)); } - close(): void { this.readyState = WebSocket.CLOSED; } + close(code?: number, reason?: string): void { this.closes.push({ ...(code === undefined ? {} : { code }), ...(reason === undefined ? {} : { reason }) }); this.readyState = WebSocket.CLOSED; } emit(type: string, event: unknown = {}): void { if (type === "open") this.readyState = WebSocket.OPEN; for (const listener of this.#listeners[type] ?? []) listener(event); @@ -37,22 +56,7 @@ it("pod connector reads the current projected identity and presents it in the ex socket.emit("open"); const hello = JSON.parse(await socket.sent.promise); expect(hello.authentication).toEqual({ deviceId: "cluster-server", deviceToken: token }); - socket.emit("message", { data: JSON.stringify({ - v: "omp-app/1", - type: "welcome", - selectedProtocol: "omp-app/1", - hostId: "host-a", - ompVersion: "17.0.5", - ompBuild: "test", - appserverVersion: "0.1.30", - appserverBuild: "cluster-session", - epoch: "pod-epoch", - grantedCapabilities: [], - grantedFeatures: [], - negotiatedLimits: {}, - authentication: "paired", - resumed: false, - }) }); + socket.emit("message", { data: JSON.stringify(welcome) }); const connection = await pending; expect(connection.hostId).toBe("host-a"); connection.close(); @@ -60,3 +64,50 @@ it("pod connector reads the current projected identity and presents it in the ex await rm(directory, { recursive: true, force: true }); } }); + +it("closes and rejects a malformed frame before the authenticated welcome", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-pod-invalid-")); + try { + const path = join(directory, "token"); + await writeFile(path, `header.payload.${"s".repeat(64)}`, { mode: 0o400 }); + const socket = new MemoryWebSocket(); + const connector = new WebSocketPodHostConnector({ identityTokenFile: path, webSocketFactory: () => socket as unknown as WebSocket }); + const pending = connector.connect({ clusterSessionId: "session-one", url: "ws://session-one:8787/v1/ws" }, () => undefined); + socket.emit("open"); + await socket.sent.promise; + socket.emit("message", { data: "{" }); + await expect(pending).rejects.toThrow(); + expect(socket.closes).toContainEqual({ code: 1002, reason: "invalid upstream frame" }); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +it("keeps idle authority sockets alive without forwarding heartbeat pongs", async () => { + vi.useFakeTimers(); + const directory = await mkdtemp(join(tmpdir(), "t4-pod-heartbeat-")); + try { + const path = join(directory, "token"); + await writeFile(path, `header.payload.${"s".repeat(64)}`, { mode: 0o400 }); + const socket = new MemoryWebSocket(); + const frames: unknown[] = []; + const connector = new WebSocketPodHostConnector({ identityTokenFile: path, keepAliveMs: 10, webSocketFactory: () => socket as unknown as WebSocket }); + const pending = connector.connect({ clusterSessionId: "session-one", url: "ws://session-one:8787/v1/ws" }, frame => frames.push(frame)); + socket.emit("open"); + await socket.sent.promise; + socket.emit("message", { data: JSON.stringify(welcome) }); + const connection = await pending; + await vi.advanceTimersByTimeAsync(10); + const heartbeat = JSON.parse(socket.sentValues.at(-1) ?? "{}"); + expect(heartbeat).toMatchObject({ v: "omp-app/1", type: "ping" }); + socket.emit("message", { data: JSON.stringify({ ...heartbeat, type: "pong" }) }); + expect(frames).toEqual([]); + connection.close(); + const sent = socket.sentValues.length; + await vi.advanceTimersByTimeAsync(20); + expect(socket.sentValues).toHaveLength(sent); + } finally { + vi.useRealTimers(); + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/packages/cluster-server/test/woodpecker.test.ts b/packages/cluster-server/test/woodpecker.test.ts index 23c85f7e..a4641578 100644 --- a/packages/cluster-server/test/woodpecker.test.ts +++ b/packages/cluster-server/test/woodpecker.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vite-plus/test"; +import { createHash } from "node:crypto"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -10,6 +11,8 @@ const correlation = { ref: "refs/heads/agent/t4-cluster-operator", commit: "0123456789abcdef0123456789abcdef01234567", }; +const runCorrelation = { ...correlation, commandId: "command-one" }; +const runKey = createHash("sha256").update(correlation.sessionId).update("\u0000").update(runCorrelation.commandId).digest("base64url"); const pipeline = { number: 42, status: "running", @@ -19,7 +22,7 @@ const pipeline = { created: 1_773_964_800, started: 1_773_964_810, finished: 0, - variables: { T4_SESSION_ID: correlation.sessionId }, + variables: { T4_SESSION_ID: correlation.sessionId, T4_IDEMPOTENCY_KEY: runKey }, stages: [ { name: "clone", status: "success" }, { name: "test", status: "running" }, @@ -62,7 +65,7 @@ describe("bounded Woodpecker provider", () => { requests.push({ url: String(input), init }); return Response.json([pipeline]); }) as typeof globalThis.fetch; - const result = await provider(fetch).run(correlation); + const result = await provider(fetch).run(runCorrelation); expect(result).toMatchObject({ triggered: false, pipelineNumber: 42, status: "running" }); expect(requests).toHaveLength(1); expect(requests[0]?.url).toBe("https://ci.example.test/api/repos/owner%2Ft4-code/pipelines?ref=refs%2Fheads%2Fagent%2Ft4-cluster-operator&commit=0123456789abcdef0123456789abcdef01234567&limit=100"); @@ -79,24 +82,53 @@ describe("bounded Woodpecker provider", () => { query++; return Response.json(query === 1 ? [{ ...pipeline, variables: { T4_SESSION_ID: "another-session" } }] : [pipeline]); } - return Response.json(pipeline, { status: 201 }); + return Response.json({ ...pipeline, idempotencyKey: runKey }, { status: 201 }); }) as typeof globalThis.fetch; const woodpecker = provider(fetch); - const first = await woodpecker.run(correlation); + const first = await woodpecker.run(runCorrelation); expect(first.triggered).toBe(true); const post = requests.find(request => request.init?.method === "POST"); expect(post?.url).toBe("https://ci.example.test/api/repos/owner%2Ft4-code/pipelines"); - expect(JSON.parse(String(post?.init?.body))).toEqual({ + const posted = JSON.parse(String(post?.init?.body)); + expect(posted).toMatchObject({ ref: correlation.ref, commit: correlation.commit, event: "manual", variables: { T4_SESSION_ID: correlation.sessionId }, }); - const second = await woodpecker.run(correlation); + expect(posted.variables.T4_IDEMPOTENCY_KEY).toMatch(/^[A-Za-z0-9_-]{43}$/u); + expect(new Headers(post?.init?.headers).get("idempotency-key")).toBe(`t4-${posted.variables.T4_IDEMPOTENCY_KEY}`); + const second = await woodpecker.run(runCorrelation); expect(second).toMatchObject({ triggered: false, pipelineNumber: 42 }); expect(requests.filter(request => request.init?.method === "POST")).toHaveLength(1); }); + it("requires an adapter idempotency receipt before reporting a trigger", async () => { + let posts = 0; + const fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + if (!init?.method || init.method === "GET") return Response.json([]); + posts++; + return Response.json(pipeline, { status: 201 }); + }) as typeof globalThis.fetch; + await expect(provider(fetch).run(runCorrelation)).rejects.toThrow("provider-side idempotency"); + expect(posts).toBe(1); + }); + + it("uses a distinct provider idempotency key for each intentional command", async () => { + const keys: string[] = []; + const fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + if (!init?.method || init.method === "GET") return Response.json([]); + const body = JSON.parse(String(init.body)) as { variables: { T4_IDEMPOTENCY_KEY: string } }; + keys.push(body.variables.T4_IDEMPOTENCY_KEY); + return Response.json({ ...pipeline, idempotencyKey: body.variables.T4_IDEMPOTENCY_KEY }, { status: 201 }); + }) as typeof globalThis.fetch; + const woodpecker = provider(fetch); + await woodpecker.run(runCorrelation); + await woodpecker.run({ ...runCorrelation, commandId: "command-two" }); + expect(keys).toHaveLength(2); + expect(new Set(keys).size).toBe(2); + }); + it("re-reads a bounded projected credential for every provider request", async () => { const directory = await mkdtemp(join(tmpdir(), "t4-woodpecker-token-")); try { diff --git a/packages/host-wire/src/cluster.ts b/packages/host-wire/src/cluster.ts index 75de16ef..1585eb5a 100644 --- a/packages/host-wire/src/cluster.ts +++ b/packages/host-wire/src/cluster.ts @@ -8,8 +8,8 @@ export const CLUSTER_OPERATOR_FEATURE = "cluster.operator" as const; export const CI_TRIGGER_CAPABILITY = "ci.trigger" as const; export const CLUSTER_MAX_WORKSPACES = 256; export const CLUSTER_MAX_CONDITION_MESSAGE_BYTES = 2_048; -export const CLUSTER_MAX_REFERENCE_BYTES = 512; -export const CLUSTER_MAX_REPOSITORY_ID_BYTES = 256; +export const CLUSTER_MAX_REFERENCE_BYTES = 256; +export const CLUSTER_MAX_REPOSITORY_ID_BYTES = 128; export const CLUSTER_CONDITION_STATUSES = ["True", "False", "Unknown"] as const; export type ClusterConditionStatus = (typeof CLUSTER_CONDITION_STATUSES)[number]; @@ -115,7 +115,6 @@ export interface ClusterWorkspaceCreateArguments { readonly displayName: string; readonly retentionPolicy: WorkspaceRetentionPolicy; readonly capacity: string; - readonly storageClass?: string; readonly repository?: ClusterRepositorySelection; } export interface ClusterSessionCiSelection { @@ -160,18 +159,38 @@ function identifier(value: unknown, path: string, max = 256): string { if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/u.test(result)) fail("INVALID_FRAME", "invalid cluster identifier", path); return result; } +function kubernetesName(value: unknown, path: string, max = 253): string { + const result = controlFree(value, path, max); + if (!/^[a-z0-9](?:[-a-z0-9.]*[a-z0-9])?$/u.test(result)) fail("INVALID_FRAME", "invalid Kubernetes resource name", path); + return result; +} +function runtimeProfile(value: unknown, path: string): string { + const result = controlFree(value, path, 64); + if (!/^[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$/u.test(result)) fail("INVALID_FRAME", "invalid runtime profile", path); + return result; +} +function nonEmptyText(value: unknown, path: string, max: number): string { + const result = controlFree(value, path, max); + if (result.length === 0) fail("INVALID_FRAME", "value must not be empty", path); + return result; +} function repositoryId(value: unknown, path: string): string { - const result = controlFree(value, path, CLUSTER_MAX_REPOSITORY_ID_BYTES); - if (!/^[A-Za-z0-9][A-Za-z0-9._/-]{0,255}$/u.test(result) || result.includes("..") || result.includes("://")) + const result = nonEmptyText(value, path, CLUSTER_MAX_REPOSITORY_ID_BYTES); + if (!/^[A-Za-z0-9][A-Za-z0-9._:/-]*$/u.test(result) || result.includes("..") || result.includes("://")) fail("INVALID_FRAME", "invalid repository identifier", path); return result; } function reference(value: unknown, path: string): string { - return controlFree(value, path, CLUSTER_MAX_REFERENCE_BYTES); + return nonEmptyText(value, path, CLUSTER_MAX_REFERENCE_BYTES); +} +function commit(value: unknown, path: string): string { + const result = nonEmptyText(value, path, 64); + if (!/^[0-9a-fA-F]{7,64}$/u.test(result)) fail("INVALID_FRAME", "commit must be 7 to 64 hexadecimal characters", path); + return result; } function quantity(value: unknown, path: string): string { - const result = controlFree(value, path, 64); - if (!/^(?:0|[1-9][0-9]*)(?:[EPTGMK]i?|m)?$/u.test(result)) fail("INVALID_FRAME", "invalid Kubernetes quantity", path); + const result = nonEmptyText(value, path, 32); + if (!/^[1-9][0-9]*(?:Ei|Pi|Ti|Gi|Mi|Ki|E|P|T|G|M|K)$/u.test(result)) fail("INVALID_FRAME", "invalid positive Kubernetes storage quantity", path); return result; } function canonicalTimestamp(value: unknown, path: string): string { @@ -215,11 +234,11 @@ export function decodeWorkspaceInfrastructureProjection( const currentRevision = revision(input.revision, `${path}.revision`); if (input.accessMode !== "ReadWriteMany") fail("INVALID_FRAME", "workspace access mode must be ReadWriteMany", `${path}.accessMode`); return { - id: identifier(input.id, `${path}.id`), - displayName: controlFree(input.displayName, `${path}.displayName`, 256), + id: kubernetesName(input.id, `${path}.id`), + displayName: nonEmptyText(input.displayName, `${path}.displayName`, 128), phase: oneOf(input.phase, `${path}.phase`, WORKSPACE_PHASES), retentionPolicy: oneOf(input.retentionPolicy, `${path}.retentionPolicy`, WORKSPACE_RETENTION_POLICIES), - ...(input.storageClass === undefined ? {} : { storageClass: identifier(input.storageClass, `${path}.storageClass`, 128) }), + ...(input.storageClass === undefined ? {} : { storageClass: kubernetesName(input.storageClass, `${path}.storageClass`, 63) }), ...(input.capacity === undefined ? {} : { capacity: quantity(input.capacity, `${path}.capacity`) }), accessMode: "ReadWriteMany", revision: currentRevision, @@ -300,20 +319,21 @@ function decodeRepository(value: unknown, path: string, requireRefCommit: boolea const input = exact(value, path, ["repositoryId", "ref", "commit"]); if (requireRefCommit && (input.ref === undefined || input.commit === undefined)) fail("INVALID_FRAME", "CI repository requires ref and commit", path); + if (input.commit !== undefined && input.ref === undefined) + fail("INVALID_FRAME", "repository commit requires an explicit ref", path); return { repositoryId: repositoryId(input.repositoryId, `${path}.repositoryId`), ...(input.ref === undefined ? {} : { ref: reference(input.ref, `${path}.ref`) }), - ...(input.commit === undefined ? {} : { commit: reference(input.commit, `${path}.commit`) }), + ...(input.commit === undefined ? {} : { commit: commit(input.commit, `${path}.commit`) }), }; } export function decodeClusterWorkspaceCreateArguments(value: unknown): ClusterWorkspaceCreateArguments { - const input = exact(value, "args", ["displayName", "retentionPolicy", "capacity", "storageClass", "repository"]); + const input = exact(value, "args", ["displayName", "retentionPolicy", "capacity", "repository"]); return { - displayName: controlFree(input.displayName, "args.displayName", 256), + displayName: nonEmptyText(input.displayName, "args.displayName", 128), retentionPolicy: oneOf(input.retentionPolicy, "args.retentionPolicy", WORKSPACE_RETENTION_POLICIES), capacity: quantity(input.capacity, "args.capacity"), - ...(input.storageClass === undefined ? {} : { storageClass: identifier(input.storageClass, "args.storageClass", 128) }), ...(input.repository === undefined ? {} : { repository: decodeRepository(input.repository, "args.repository", false) }), }; } @@ -329,9 +349,9 @@ export function decodeClusterSessionCreateArguments(value: unknown): ClusterSess } if (typeof input.guiEnabled !== "boolean") fail("INVALID_FRAME", "guiEnabled must be boolean", "args.guiEnabled"); return { - workspaceId: identifier(input.workspaceId, "args.workspaceId"), - ...(input.title === undefined ? {} : { title: controlFree(input.title, "args.title", 512) }), - runtimeProfile: identifier(input.runtimeProfile, "args.runtimeProfile", 64), + workspaceId: kubernetesName(input.workspaceId, "args.workspaceId"), + ...(input.title === undefined ? {} : { title: nonEmptyText(input.title, "args.title", 128) }), + runtimeProfile: runtimeProfile(input.runtimeProfile, "args.runtimeProfile"), guiEnabled: input.guiEnabled, ...(ci === undefined ? {} : { ci }), }; @@ -346,7 +366,7 @@ export function decodeCiRunArguments(value: unknown): CiRunArguments { action: "run", repositoryId: repositoryId(input.repositoryId, "args.repositoryId"), ref: reference(input.ref, "args.ref"), - commit: reference(input.commit, "args.commit"), + commit: commit(input.commit, "args.commit"), }; } diff --git a/packages/host-wire/test/cluster-operator.test.ts b/packages/host-wire/test/cluster-operator.test.ts index 9df32c89..e2111861 100644 --- a/packages/host-wire/test/cluster-operator.test.ts +++ b/packages/host-wire/test/cluster-operator.test.ts @@ -220,8 +220,7 @@ describe("cluster operator wire contract", () => { displayName: "T4 code", retentionPolicy: "Retain", capacity: "20Gi", - storageClass: "t4-workspaces-rwx", - repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, + repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, }), ).toMatchObject({ displayName: "T4 code", capacity: "20Gi" }); expect( @@ -230,8 +229,21 @@ describe("cluster operator wire contract", () => { title: "Agent task", runtimeProfile: "omp-17.0.5", guiEnabled: true, - ci: { provider: "woodpecker", repositoryId: "t4-code", ref: "refs/heads/main", commit: "abc" }, + ci: { provider: "woodpecker", repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, }), ).toMatchObject({ workspaceId: "workspace-one", runtimeProfile: "omp-17.0.5" }); + for (const malformed of [ + { displayName: "x".repeat(129), retentionPolicy: "Retain", capacity: "20Gi" }, + { displayName: "T4 code", retentionPolicy: "Retain", capacity: "0Gi" }, + { displayName: "T4 code", retentionPolicy: "Retain", capacity: "20Gi", storageClass: "client-owned" }, + { displayName: "T4 code", retentionPolicy: "Retain", capacity: "20Gi", repository: { repositoryId: "t4-code", commit: "abcdef0" } }, + { displayName: "T4 code", retentionPolicy: "Retain", capacity: "20Gi", repository: { repositoryId: "t4-code", ref: "main", commit: "abc" } }, + ]) expect(() => decodeClusterWorkspaceCreateArguments(malformed)).toThrow(AppWireError); + for (const malformed of [ + { workspaceId: "Workspace:one", title: "Agent", runtimeProfile: "default", guiEnabled: true }, + { workspaceId: "workspace-one", title: "x".repeat(129), runtimeProfile: "default", guiEnabled: true }, + { workspaceId: "workspace-one", title: "Agent", runtimeProfile: "Default:unsafe", guiEnabled: true }, + { workspaceId: "workspace-one", title: "Agent", runtimeProfile: "default", guiEnabled: true, ci: { provider: "woodpecker", repositoryId: "t4-code", ref: "main", commit: "abc" } }, + ]) expect(() => decodeClusterSessionCreateArguments(malformed)).toThrow(AppWireError); }); }); From 539b8f6b0018a13e535344bb43b7298d75c6919b Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 17:26:50 +0000 Subject: [PATCH 24/98] fix(cluster): harden portable operator packaging --- cluster/images/cluster-server/Dockerfile | 7 +- cluster/images/controller/Dockerfile | 4 +- cluster/images/session-runtime/Dockerfile | 29 +- .../crds/t4clusterhosts.cluster.t4.dev.yaml | 6 +- .../crds/t4sessions.cluster.t4.dev.yaml | 3 + .../crds/t4workspaces.cluster.t4.dev.yaml | 3 + .../charts/t4-cluster/templates/_helpers.tpl | 14 +- .../t4-cluster/templates/clusterhost.yaml | 8 +- .../t4-cluster/templates/deployments.yaml | 76 +++-- .../charts/t4-cluster/templates/ingress.yaml | 14 +- .../t4-cluster/templates/networkpolicies.yaml | 27 +- .../t4-cluster/templates/observability.yaml | 16 +- deploy/charts/t4-cluster/templates/rbac.yaml | 50 +-- .../charts/t4-cluster/templates/services.yaml | 8 +- deploy/charts/t4-cluster/values.schema.json | 286 ++++++++++++++++-- deploy/charts/t4-cluster/values.yaml | 13 + docs/CLUSTER_OPERATOR.md | 29 +- .../api/v1alpha1/types_test.go | 12 + .../charttests/chart_contract_test.go | 162 ++++++++-- packages/cluster-operator/cmd/manager/main.go | 1 + .../controllers/reconciler_test.go | 203 ++++++++++++- .../controllers/session_controller.go | 93 +++++- .../controllers/workspace_controller.go | 13 + packages/cluster-server/src/config.ts | 9 +- .../cluster-server/src/session-host-policy.ts | 7 + packages/cluster-server/test/config.test.ts | 5 + .../test/session-host-policy.test.ts | 3 + 27 files changed, 929 insertions(+), 172 deletions(-) diff --git a/cluster/images/cluster-server/Dockerfile b/cluster/images/cluster-server/Dockerfile index ba6e1513..afe8ea68 100644 --- a/cluster/images/cluster-server/Dockerfile +++ b/cluster/images/cluster-server/Dockerfile @@ -7,7 +7,12 @@ RUN corepack enable \ FROM docker.io/oven/bun:1.3.14-debian@sha256:9dba1a1b43ce28c9d7931bfc4eb00feb63b0114720a0277a8f939ae4dfc9db6f USER root -RUN apt-get update \ +RUN rm -f /etc/apt/sources.list.d/* \ + && printf '%s\n' \ + 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/20250721T000000Z bookworm main' \ + 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/20250721T000000Z bookworm-security main' \ + > /etc/apt/sources.list \ + && apt-get -o Acquire::Check-Valid-Until=false update \ && apt-get install -y --no-install-recommends ca-certificates curl tini \ && rm -rf /var/lib/apt/lists/* WORKDIR /opt/t4 diff --git a/cluster/images/controller/Dockerfile b/cluster/images/controller/Dockerfile index f7effc6d..3af46646 100644 --- a/cluster/images/controller/Dockerfile +++ b/cluster/images/controller/Dockerfile @@ -4,8 +4,8 @@ WORKDIR /src/packages/cluster-operator COPY packages/cluster-operator/go.mod ./ RUN --mount=type=cache,target=/go/pkg/mod go mod download COPY packages/cluster-operator/ ./ -ARG TARGETOS=linux -ARG TARGETARCH=amd64 +ARG TARGETOS +ARG TARGETARCH RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ diff --git a/cluster/images/session-runtime/Dockerfile b/cluster/images/session-runtime/Dockerfile index a8c4c910..d778872d 100644 --- a/cluster/images/session-runtime/Dockerfile +++ b/cluster/images/session-runtime/Dockerfile @@ -1,21 +1,21 @@ # syntax=docker/dockerfile:1.7 -ARG BUN_IMAGE=docker.io/oven/bun:1.3.14-debian@sha256:9dba1a1b43ce28c9d7931bfc4eb00feb63b0114720a0277a8f939ae4dfc9db6f - -FROM ${BUN_IMAGE} AS bun-runtime FROM docker.io/library/rust:1.86-slim-bookworm@sha256:57d415bbd61ce11e2d5f73de068103c7bd9f3188dc132c97cef4a8f62989e944 AS omp-build -ARG OMP_TAG=t4code-17.0.5-appserver-10 -ARG OMP_COMMIT=8476f4451ed95c5d5401785d279a93d3c659fac4 -RUN apt-get update \ +RUN rm -f /etc/apt/sources.list.d/* \ + && printf '%s\n' \ + 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/20250721T000000Z bookworm main' \ + 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/20250721T000000Z bookworm-security main' \ + > /etc/apt/sources.list \ + && apt-get -o Acquire::Check-Valid-Until=false update \ && apt-get install -y --no-install-recommends ca-certificates git pkg-config libssl-dev \ && rm -rf /var/lib/apt/lists/* -COPY --from=bun-runtime /usr/local/bin/bun /usr/local/bin/bun +COPY --from=docker.io/oven/bun:1.3.14-debian@sha256:9dba1a1b43ce28c9d7931bfc4eb00feb63b0114720a0277a8f939ae4dfc9db6f /usr/local/bin/bun /usr/local/bin/bun WORKDIR /opt/omp RUN git init \ && git remote add origin https://github.com/lyc-aon/oh-my-pi.git \ - && git fetch --depth=1 origin "refs/tags/${OMP_TAG}" \ - && test "$(git rev-parse FETCH_HEAD)" = "${OMP_COMMIT}" \ - && git checkout --detach "${OMP_COMMIT}" + && git fetch --depth=1 origin "refs/tags/t4code-17.0.5-appserver-10" \ + && test "$(git rev-parse FETCH_HEAD)" = "8476f4451ed95c5d5401785d279a93d3c659fac4" \ + && git checkout --detach "8476f4451ed95c5d5401785d279a93d3c659fac4" RUN bun install --frozen-lockfile --ignore-scripts \ && bun --cwd=packages/natives run build \ && bun --cwd=packages/coding-agent run gen:tool-views \ @@ -28,9 +28,14 @@ COPY . /opt/t4 RUN corepack enable \ && pnpm install --frozen-lockfile --ignore-scripts -FROM ${BUN_IMAGE} +FROM docker.io/oven/bun:1.3.14-debian@sha256:9dba1a1b43ce28c9d7931bfc4eb00feb63b0114720a0277a8f939ae4dfc9db6f USER root -RUN apt-get update \ +RUN rm -f /etc/apt/sources.list.d/* \ + && printf '%s\n' \ + 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/20250721T000000Z bookworm main' \ + 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/20250721T000000Z bookworm-security main' \ + > /etc/apt/sources.list \ + && apt-get -o Acquire::Check-Valid-Until=false update \ && apt-get install -y --no-install-recommends \ bash ca-certificates chromium fluxbox git openssh-client python3 ripgrep tini xvfb \ && rm -rf /var/lib/apt/lists/* \ diff --git a/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml b/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml index ea26aefa..6bbf2336 100644 --- a/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml +++ b/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml @@ -47,8 +47,8 @@ spec: storageClassName: type: string minLength: 1 - maxLength: 63 - pattern: '^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$' + maxLength: 253 + pattern: '^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?(\.[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?)*$' runtimeProfiles: type: array minItems: 1 @@ -58,7 +58,7 @@ spec: type: string minLength: 1 maxLength: 64 - pattern: '^[a-z0-9]([a-z0-9._-]*[a-z0-9])?$' + pattern: '^[a-z0-9]([-._a-z0-9]{0,62}[a-z0-9])?$' projection: type: object properties: diff --git a/deploy/charts/t4-cluster/crds/t4sessions.cluster.t4.dev.yaml b/deploy/charts/t4-cluster/crds/t4sessions.cluster.t4.dev.yaml index 701c6c61..107b0839 100644 --- a/deploy/charts/t4-cluster/crds/t4sessions.cluster.t4.dev.yaml +++ b/deploy/charts/t4-cluster/crds/t4sessions.cluster.t4.dev.yaml @@ -49,6 +49,9 @@ spec: minLength: 1 maxLength: 253 pattern: '^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$' + x-kubernetes-validations: + - rule: self == oldSelf + message: hostRef is immutable workspaceRef: type: string minLength: 1 diff --git a/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml b/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml index cdfa0396..6906f2bf 100644 --- a/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml +++ b/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml @@ -47,6 +47,9 @@ spec: minLength: 1 maxLength: 253 pattern: '^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$' + x-kubernetes-validations: + - rule: self == oldSelf + message: hostRef is immutable displayName: type: string minLength: 1 diff --git a/deploy/charts/t4-cluster/templates/_helpers.tpl b/deploy/charts/t4-cluster/templates/_helpers.tpl index cbfbbe20..289c8721 100644 --- a/deploy/charts/t4-cluster/templates/_helpers.tpl +++ b/deploy/charts/t4-cluster/templates/_helpers.tpl @@ -11,16 +11,16 @@ t4-cluster {{- end -}} {{- define "t4-cluster.labels" -}} -app.kubernetes.io/name: {{ include "t4-cluster.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/part-of: t4-cluster -app.kubernetes.io/managed-by: {{ .Release.Service }} -helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} +app.kubernetes.io/name: {{ include "t4-cluster.name" . | quote }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +app.kubernetes.io/part-of: "t4-cluster" +app.kubernetes.io/managed-by: {{ .Release.Service | quote }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | quote }} {{- end -}} {{- define "t4-cluster.selectorLabels" -}} -app.kubernetes.io/name: {{ include "t4-cluster.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/name: {{ include "t4-cluster.name" . | quote }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} {{- end -}} {{- define "t4-cluster.image" -}} diff --git a/deploy/charts/t4-cluster/templates/clusterhost.yaml b/deploy/charts/t4-cluster/templates/clusterhost.yaml index 88e9091f..b15b4cf0 100644 --- a/deploy/charts/t4-cluster/templates/clusterhost.yaml +++ b/deploy/charts/t4-cluster/templates/clusterhost.yaml @@ -2,11 +2,11 @@ apiVersion: cluster.t4.dev/v1alpha1 kind: T4ClusterHost metadata: - name: {{ .Values.clusterHost.name }} + name: {{ .Values.clusterHost.name | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} spec: - storageClassName: {{ .Values.storage.adminRWXStorageClass }} + storageClassName: {{ .Values.storage.adminRWXStorageClass | quote }} runtimeProfiles: {{- toYaml .Values.clusterHost.runtimeProfiles | nindent 4 }} projection: @@ -18,11 +18,11 @@ spec: ciProvider: {{- if .Values.woodpecker.existingSecret }} secretRef: - name: {{ .Values.woodpecker.existingSecret }} + name: {{ .Values.woodpecker.existingSecret | quote }} {{- else }} serviceAccountAudience: {{ .Values.woodpecker.serviceAccountAudience | quote }} {{- end }} configMapRef: - name: {{ .Values.woodpecker.configMap }} + name: {{ .Values.woodpecker.configMap | quote }} {{- end }} {{- end }} diff --git a/deploy/charts/t4-cluster/templates/deployments.yaml b/deploy/charts/t4-cluster/templates/deployments.yaml index 2be5fe83..bafdf1cd 100644 --- a/deploy/charts/t4-cluster/templates/deployments.yaml +++ b/deploy/charts/t4-cluster/templates/deployments.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "t4-cluster.fullname" . }}-controller + name: {{ printf "%s-controller" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} app.kubernetes.io/component: controller @@ -11,7 +11,7 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxUnavailable: 1 + maxUnavailable: 0 maxSurge: 1 selector: matchLabels: @@ -24,8 +24,8 @@ spec: app.kubernetes.io/component: controller app.kubernetes.io/part-of: t4-cluster spec: - serviceAccountName: {{ include "t4-cluster.fullname" . }}-controller - automountServiceAccountToken: true + serviceAccountName: {{ printf "%s-controller" (include "t4-cluster.fullname" .) | quote }} + automountServiceAccountToken: false terminationGracePeriodSeconds: 30 securityContext: runAsNonRoot: true @@ -45,7 +45,7 @@ spec: - matchExpressions: - key: kubernetes.io/hostname operator: NotIn - values: [k3s-worker-02] + values: ["k3s-worker-02"] podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 @@ -65,18 +65,20 @@ spec: app.kubernetes.io/component: controller containers: - name: controller - image: {{ include "t4-cluster.image" .Values.images.controller }} - imagePullPolicy: {{ .Values.images.controller.pullPolicy }} - args: [--zap-log-level=info] + image: {{ include "t4-cluster.image" .Values.images.controller | quote }} + imagePullPolicy: {{ .Values.images.controller.pullPolicy | quote }} + args: ["--zap-log-level=info"] env: - name: POD_NAMESPACE valueFrom: {fieldRef: {fieldPath: metadata.namespace}} - name: T4_SESSION_RUNTIME_IMAGE - value: {{ include "t4-cluster.image" .Values.images.sessionRuntime }} + value: {{ include "t4-cluster.image" .Values.images.sessionRuntime | quote }} - name: T4_SESSION_SERVICE_ACCOUNT - value: {{ include "t4-cluster.fullname" . }}-session + value: {{ printf "%s-session" (include "t4-cluster.fullname" .) | quote }} - name: T4_CLUSTER_SERVER_SERVICE_ACCOUNT - value: {{ include "t4-cluster.fullname" . }}-server + value: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} + - name: T4_KUBERNETES_API_AUDIENCE + value: {{ .Values.kubernetes.apiAudience | quote }} - name: T4_SESSION_EXCLUDED_NODES value: {{ join "," .Values.session.nodeExclude | quote }} - name: T4_SESSION_REQUEST_CPU @@ -120,15 +122,31 @@ spec: volumeMounts: - name: temporary mountPath: /tmp + - name: kubernetes-api-access + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true volumes: - name: temporary emptyDir: sizeLimit: 128Mi + - name: kubernetes-api-access + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: + audience: {{ .Values.kubernetes.apiAudience | quote }} + expirationSeconds: 3600 + path: token + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt --- apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "t4-cluster.fullname" . }}-server + name: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} app.kubernetes.io/component: server @@ -154,7 +172,7 @@ spec: app.kubernetes.io/component: server app.kubernetes.io/part-of: t4-cluster spec: - serviceAccountName: {{ include "t4-cluster.fullname" . }}-server + serviceAccountName: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} automountServiceAccountToken: false terminationGracePeriodSeconds: {{ .Values.server.terminationGracePeriodSeconds }} securityContext: @@ -175,7 +193,7 @@ spec: - matchExpressions: - key: kubernetes.io/hostname operator: NotIn - values: [k3s-worker-02] + values: ["k3s-worker-02"] podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 @@ -195,8 +213,8 @@ spec: app.kubernetes.io/component: server containers: - name: server - image: {{ include "t4-cluster.image" .Values.images.server }} - imagePullPolicy: {{ .Values.images.server.pullPolicy }} + image: {{ include "t4-cluster.image" .Values.images.server | quote }} + imagePullPolicy: {{ .Values.images.server.pullPolicy | quote }} env: - name: T4_CLUSTER_SERVER_PORT value: "8080" @@ -207,11 +225,13 @@ spec: - name: T4_CLUSTER_TRUSTED_PROXY_CIDRS value: {{ join "," .Values.server.trustedProxyCIDRs | quote }} - name: T4_CLUSTER_HOST_NAME - value: {{ .Values.clusterHost.name }} + value: {{ .Values.clusterHost.name | quote }} - name: T4_CLUSTER_IDENTITY_TOKEN_FILE value: /var/run/secrets/t4-cluster-identity/token - name: T4_CLUSTER_SERVER_SERVICE_ACCOUNT - value: {{ include "t4-cluster.fullname" . }}-server + value: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} + - name: T4_KUBERNETES_API_AUDIENCE + value: {{ .Values.kubernetes.apiAudience | quote }} - name: T4_KUBERNETES_TOKEN_PATH value: /var/run/secrets/kubernetes.io/serviceaccount/token - name: T4_KUBERNETES_CA_PATH @@ -226,8 +246,8 @@ spec: - name: T4_WOODPECKER_TOKEN valueFrom: secretKeyRef: - name: {{ .Values.woodpecker.existingSecret }} - key: {{ .Values.woodpecker.tokenKey }} + name: {{ .Values.woodpecker.existingSecret | quote }} + key: {{ .Values.woodpecker.tokenKey | quote }} {{- else if .Values.woodpecker.serviceAccountAudience }} - name: T4_WOODPECKER_TOKEN_FILE value: /var/run/secrets/t4-ci/token @@ -236,18 +256,18 @@ spec: - name: T4_WOODPECKER_BASE_URL valueFrom: configMapKeyRef: - name: {{ .Values.woodpecker.configMap }} - key: {{ .Values.woodpecker.baseURLKey }} + name: {{ .Values.woodpecker.configMap | quote }} + key: {{ .Values.woodpecker.baseURLKey | quote }} - name: T4_WOODPECKER_WEB_BASE_URL valueFrom: configMapKeyRef: - name: {{ .Values.woodpecker.configMap }} - key: {{ .Values.woodpecker.webBaseURLKey }} + name: {{ .Values.woodpecker.configMap | quote }} + key: {{ .Values.woodpecker.webBaseURLKey | quote }} - name: T4_WOODPECKER_REPOSITORIES valueFrom: configMapKeyRef: - name: {{ .Values.woodpecker.configMap }} - key: {{ .Values.woodpecker.repositoriesKey }} + name: {{ .Values.woodpecker.configMap | quote }} + key: {{ .Values.woodpecker.repositoriesKey | quote }} {{- end }} ports: - name: websocket @@ -299,7 +319,7 @@ spec: defaultMode: 0440 sources: - serviceAccountToken: - audience: t4-cluster-internal + audience: "t4-cluster-internal" expirationSeconds: 600 path: token - name: kubernetes-api-access @@ -307,7 +327,7 @@ spec: defaultMode: 0440 sources: - serviceAccountToken: - audience: https://kubernetes.default.svc + audience: {{ .Values.kubernetes.apiAudience | quote }} expirationSeconds: 3600 path: token - configMap: diff --git a/deploy/charts/t4-cluster/templates/ingress.yaml b/deploy/charts/t4-cluster/templates/ingress.yaml index 78331d55..bc5d6a61 100644 --- a/deploy/charts/t4-cluster/templates/ingress.yaml +++ b/deploy/charts/t4-cluster/templates/ingress.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "t4-cluster.fullname" . }} + name: {{ include "t4-cluster.fullname" . | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} {{- with .Values.ingress.annotations }} @@ -10,13 +10,13 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- with .Values.ingress.className }} - ingressClassName: {{ . }} - {{- end }} - {{- if .Values.ingress.tls.secretName }} + ingressClassName: {{ .Values.ingress.className | quote }} + {{- if .Values.ingress.tls.enabled }} tls: - hosts: [{{ .Values.ingress.host | quote }}] - secretName: {{ .Values.ingress.tls.secretName }} + {{- with .Values.ingress.tls.secretName }} + secretName: {{ . | quote }} + {{- end }} {{- end }} rules: - host: {{ .Values.ingress.host | quote }} @@ -26,7 +26,7 @@ spec: pathType: Prefix backend: service: - name: {{ include "t4-cluster.fullname" . }}-server + name: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} port: name: websocket {{- end }} diff --git a/deploy/charts/t4-cluster/templates/networkpolicies.yaml b/deploy/charts/t4-cluster/templates/networkpolicies.yaml index 37d94e6c..3e0fd176 100644 --- a/deploy/charts/t4-cluster/templates/networkpolicies.yaml +++ b/deploy/charts/t4-cluster/templates/networkpolicies.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: {{ include "t4-cluster.fullname" . }}-default-deny + name: {{ printf "%s-default-deny" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} spec: @@ -14,7 +14,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: {{ include "t4-cluster.fullname" . }}-dns + name: {{ printf "%s-dns" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} spec: @@ -25,11 +25,9 @@ spec: egress: - to: - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: kube-system + {{- toYaml .Values.networkPolicy.dns.namespaceSelector | nindent 12 }} podSelector: - matchLabels: - k8s-app: kube-dns + {{- toYaml .Values.networkPolicy.dns.podSelector | nindent 12 }} ports: - {protocol: UDP, port: 53} - {protocol: TCP, port: 53} @@ -37,7 +35,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: {{ include "t4-cluster.fullname" . }}-controller-api + name: {{ printf "%s-controller-api" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} spec: @@ -58,7 +56,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: {{ include "t4-cluster.fullname" . }}-server-egress + name: {{ printf "%s-server-egress" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} spec: @@ -92,7 +90,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: {{ include "t4-cluster.fullname" . }}-session-host + name: {{ printf "%s-session-host" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} spec: @@ -127,7 +125,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: {{ include "t4-cluster.fullname" . }}-gateway-ingress + name: {{ printf "%s-gateway-ingress" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} spec: @@ -136,7 +134,7 @@ spec: {{- include "t4-cluster.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: server policyTypes: [Ingress] - {{- if or (not (empty .Values.networkPolicy.gatewayIngress.namespaceSelector)) (not (empty .Values.networkPolicy.gatewayIngress.podSelector)) }} + {{- if and (not (empty .Values.networkPolicy.gatewayIngress.namespaceSelector)) (not (empty .Values.networkPolicy.gatewayIngress.podSelector)) }} ingress: - from: - namespaceSelector: @@ -152,15 +150,18 @@ spec: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: {{ include "t4-cluster.fullname" . }}-observability + name: {{ printf "%s-observability" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} spec: podSelector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 6 }} + app.kubernetes.io/part-of: "t4-cluster" matchExpressions: - {key: app.kubernetes.io/component, operator: In, values: [controller, server]} policyTypes: [Ingress] - {{- if or (not (empty .Values.networkPolicy.observability.namespaceSelector)) (not (empty .Values.networkPolicy.observability.podSelector)) }} + {{- if and (not (empty .Values.networkPolicy.observability.namespaceSelector)) (not (empty .Values.networkPolicy.observability.podSelector)) }} ingress: - from: - namespaceSelector: diff --git a/deploy/charts/t4-cluster/templates/observability.yaml b/deploy/charts/t4-cluster/templates/observability.yaml index b34f671b..00622b97 100644 --- a/deploy/charts/t4-cluster/templates/observability.yaml +++ b/deploy/charts/t4-cluster/templates/observability.yaml @@ -2,7 +2,7 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: {{ include "t4-cluster.fullname" . }}-server + name: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} {{- with .Values.observability.serviceMonitor.labels }} @@ -10,20 +10,21 @@ metadata: {{- end }} spec: namespaceSelector: - matchNames: [{{ .Release.Namespace }}] + matchNames: [{{ .Release.Namespace | quote }}] selector: matchLabels: {{- include "t4-cluster.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: server + app.kubernetes.io/part-of: "t4-cluster" endpoints: - port: metrics path: /metrics - interval: {{ .Values.observability.serviceMonitor.interval }} + interval: {{ .Values.observability.serviceMonitor.interval | quote }} --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: {{ include "t4-cluster.fullname" . }}-controller + name: {{ printf "%s-controller" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} {{- with .Values.observability.serviceMonitor.labels }} @@ -31,22 +32,23 @@ metadata: {{- end }} spec: namespaceSelector: - matchNames: [{{ .Release.Namespace }}] + matchNames: [{{ .Release.Namespace | quote }}] selector: matchLabels: {{- include "t4-cluster.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: controller + app.kubernetes.io/part-of: "t4-cluster" endpoints: - port: metrics path: /metrics - interval: {{ .Values.observability.serviceMonitor.interval }} + interval: {{ .Values.observability.serviceMonitor.interval | quote }} {{- end }} {{- if and .Values.enabled .Values.observability.prometheusRule.enabled }} --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: - name: {{ include "t4-cluster.fullname" . }} + name: {{ include "t4-cluster.fullname" . | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} {{- with .Values.observability.prometheusRule.labels }} diff --git a/deploy/charts/t4-cluster/templates/rbac.yaml b/deploy/charts/t4-cluster/templates/rbac.yaml index 3c330c49..e35eca83 100644 --- a/deploy/charts/t4-cluster/templates/rbac.yaml +++ b/deploy/charts/t4-cluster/templates/rbac.yaml @@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ include "t4-cluster.fullname" . }}-controller + name: {{ printf "%s-controller" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} rules: @@ -28,7 +28,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ include "t4-cluster.fullname" . }}-server + name: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} rules: @@ -48,7 +48,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "t4-cluster.fullname" . }}-session-token-reviewer + name: {{ printf "%s-session-token-reviewer" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} rules: @@ -59,7 +59,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "t4-cluster.fullname" . }}-storage-reader + name: {{ printf "%s-storage-reader" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} rules: @@ -70,23 +70,23 @@ rules: apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "t4-cluster.fullname" . }}-controller + name: {{ printf "%s-controller" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} -automountServiceAccountToken: true +automountServiceAccountToken: false --- apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "t4-cluster.fullname" . }}-server + name: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} -automountServiceAccountToken: true +automountServiceAccountToken: false --- apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "t4-cluster.fullname" . }}-session + name: {{ printf "%s-session" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} automountServiceAccountToken: false @@ -94,60 +94,60 @@ automountServiceAccountToken: false apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ include "t4-cluster.fullname" . }}-controller + name: {{ printf "%s-controller" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} subjects: - kind: ServiceAccount - name: {{ include "t4-cluster.fullname" . }}-controller - namespace: {{ .Release.Namespace }} + name: {{ printf "%s-controller" (include "t4-cluster.fullname" .) | quote }} + namespace: {{ .Release.Namespace | quote }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: {{ include "t4-cluster.fullname" . }}-controller + name: {{ printf "%s-controller" (include "t4-cluster.fullname" .) | quote }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: {{ include "t4-cluster.fullname" . }}-storage-reader + name: {{ printf "%s-storage-reader" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} subjects: - kind: ServiceAccount - name: {{ include "t4-cluster.fullname" . }}-controller - namespace: {{ .Release.Namespace }} + name: {{ printf "%s-controller" (include "t4-cluster.fullname" .) | quote }} + namespace: {{ .Release.Namespace | quote }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ include "t4-cluster.fullname" . }}-storage-reader + name: {{ printf "%s-storage-reader" (include "t4-cluster.fullname" .) | quote }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ include "t4-cluster.fullname" . }}-server + name: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} subjects: - kind: ServiceAccount - name: {{ include "t4-cluster.fullname" . }}-server - namespace: {{ .Release.Namespace }} + name: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} + namespace: {{ .Release.Namespace | quote }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: {{ include "t4-cluster.fullname" . }}-server + name: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: {{ include "t4-cluster.fullname" . }}-session-token-reviewer + name: {{ printf "%s-session-token-reviewer" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} subjects: - kind: ServiceAccount - name: {{ include "t4-cluster.fullname" . }}-session - namespace: {{ .Release.Namespace }} + name: {{ printf "%s-session" (include "t4-cluster.fullname" .) | quote }} + namespace: {{ .Release.Namespace | quote }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ include "t4-cluster.fullname" . }}-session-token-reviewer + name: {{ printf "%s-session-token-reviewer" (include "t4-cluster.fullname" .) | quote }} {{- end }} diff --git a/deploy/charts/t4-cluster/templates/services.yaml b/deploy/charts/t4-cluster/templates/services.yaml index 2c11d4c2..936cb399 100644 --- a/deploy/charts/t4-cluster/templates/services.yaml +++ b/deploy/charts/t4-cluster/templates/services.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "t4-cluster.fullname" . }}-server + name: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} app.kubernetes.io/component: server @@ -20,7 +20,7 @@ spec: apiVersion: v1 kind: Service metadata: - name: {{ include "t4-cluster.fullname" . }}-metrics + name: {{ printf "%s-metrics" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} app.kubernetes.io/component: server @@ -38,7 +38,7 @@ spec: apiVersion: v1 kind: Service metadata: - name: {{ include "t4-cluster.fullname" . }}-controller-metrics + name: {{ printf "%s-controller-metrics" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} app.kubernetes.io/component: controller @@ -56,7 +56,7 @@ spec: apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - name: {{ include "t4-cluster.fullname" . }}-server + name: {{ printf "%s-server" (include "t4-cluster.fullname" .) | quote }} labels: {{- include "t4-cluster.labels" . | nindent 4 }} app.kubernetes.io/component: server diff --git a/deploy/charts/t4-cluster/values.schema.json b/deploy/charts/t4-cluster/values.schema.json index 67c62273..abb75780 100644 --- a/deploy/charts/t4-cluster/values.schema.json +++ b/deploy/charts/t4-cluster/values.schema.json @@ -1,16 +1,40 @@ { "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", - "required": ["enabled", "clusterHost", "images", "controller", "server", "session", "storage", "networkPolicy"], + "required": [ + "enabled", + "clusterHost", + "kubernetes", + "images", + "controller", + "server", + "session", + "storage", + "woodpecker", + "ingress", + "networkPolicy", + "observability" + ], "properties": { "enabled": { "type": "boolean", "default": false }, "clusterHost": { "type": "object", "required": ["name", "runtimeProfiles", "allowedOrigins", "projection"], "properties": { - "name": { "type": "string", "minLength": 1, "maxLength": 63 }, - "runtimeProfiles": { "type": "array", "minItems": 1, "maxItems": 16, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 64 } }, - "allowedOrigins": { "type": "array", "maxItems": 32, "uniqueItems": true, "items": { "type": "string", "pattern": "^https://[^/?#]+$", "maxLength": 256 } }, + "name": { "$ref": "#/definitions/dnsSubdomain" }, + "runtimeProfiles": { + "type": "array", + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "items": { "$ref": "#/definitions/runtimeProfile" } + }, + "allowedOrigins": { + "type": "array", + "maxItems": 32, + "uniqueItems": true, + "items": { "type": "string", "pattern": "^https://[^/?#]+$", "maxLength": 256 } + }, "projection": { "type": "object", "required": ["maxWorkspaces", "resyncSeconds"], @@ -21,6 +45,13 @@ } } }, + "kubernetes": { + "type": "object", + "required": ["apiAudience"], + "properties": { + "apiAudience": { "$ref": "#/definitions/audience" } + } + }, "images": { "type": "object", "required": ["controller", "server", "sessionRuntime"], @@ -30,7 +61,14 @@ "sessionRuntime": { "$ref": "#/definitions/image" } } }, - "imagePullSecrets": { "type": "array", "items": { "type": "object", "required": ["name"], "properties": { "name": { "type": "string", "minLength": 1 } } } }, + "imagePullSecrets": { + "type": "array", + "items": { + "type": "object", + "required": ["name"], + "properties": { "name": { "$ref": "#/definitions/dnsSubdomain" } } + } + }, "controller": { "type": "object", "required": ["replicas", "resources"], @@ -45,8 +83,18 @@ "properties": { "replicas": { "type": "integer", "minimum": 2, "maximum": 9 }, "terminationGracePeriodSeconds": { "type": "integer", "minimum": 30, "maximum": 300 }, - "trustedProxyAddresses": { "type": "array", "maxItems": 64, "uniqueItems": true, "items": { "type": "string", "anyOf": [{ "format": "ipv4" }, { "format": "ipv6" }] } }, - "trustedProxyCIDRs": { "type": "array", "maxItems": 64, "uniqueItems": true, "items": { "type": "string", "pattern": "^[0-9A-Fa-f:.]+/(?:[0-9]|[1-9][0-9]|1[01][0-9]|12[0-8])$", "maxLength": 64 } }, + "trustedProxyAddresses": { + "type": "array", + "maxItems": 64, + "uniqueItems": true, + "items": { "type": "string", "anyOf": [{ "format": "ipv4" }, { "format": "ipv6" }] } + }, + "trustedProxyCIDRs": { + "type": "array", + "maxItems": 64, + "uniqueItems": true, + "items": { "$ref": "#/definitions/nonDefaultRouteCIDR" } + }, "resources": { "type": "object" } } }, @@ -54,7 +102,14 @@ "type": "object", "required": ["nodeExclude", "resources", "sharedMemorySize", "temporarySize"], "properties": { - "nodeExclude": { "type": "array", "minItems": 1, "maxItems": 16, "uniqueItems": true, "contains": { "const": "k3s-worker-02" }, "items": { "type": "string", "minLength": 1, "maxLength": 63 } }, + "nodeExclude": { + "type": "array", + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "contains": { "const": "k3s-worker-02" }, + "items": { "$ref": "#/definitions/dnsLabel" } + }, "resources": { "type": "object" }, "sharedMemorySize": { "type": "string", "pattern": "^[1-9][0-9]*(Mi|Gi)$" }, "temporarySize": { "type": "string", "pattern": "^[1-9][0-9]*(Mi|Gi)$" } @@ -63,19 +118,27 @@ "storage": { "type": "object", "required": ["adminRWXStorageClass"], - "properties": { "adminRWXStorageClass": { "type": "string", "maxLength": 63 } } + "properties": { + "adminRWXStorageClass": { + "allOf": [{ "$ref": "#/definitions/optionalDNSSubdomain" }] + } + } }, "woodpecker": { "type": "object", "required": ["existingSecret", "tokenKey", "configMap", "baseURLKey", "webBaseURLKey", "repositoriesKey", "serviceAccountAudience", "tokenExpirationSeconds"], "properties": { - "existingSecret": { "type": "string", "maxLength": 253 }, - "tokenKey": { "type": "string", "minLength": 1, "maxLength": 253 }, - "configMap": { "type": "string", "maxLength": 253 }, - "baseURLKey": { "type": "string", "minLength": 1, "maxLength": 253 }, - "repositoriesKey": { "type": "string", "minLength": 1, "maxLength": 253 }, - "webBaseURLKey": { "type": "string", "minLength": 1, "maxLength": 253 }, - "serviceAccountAudience": { "type": "string", "maxLength": 253, "pattern": "^[A-Za-z0-9]*[A-Za-z0-9._:/-]*$" }, + "existingSecret": { "$ref": "#/definitions/optionalDNSSubdomain" }, + "tokenKey": { "$ref": "#/definitions/configKey" }, + "configMap": { "$ref": "#/definitions/optionalDNSSubdomain" }, + "baseURLKey": { "$ref": "#/definitions/configKey" }, + "repositoriesKey": { "$ref": "#/definitions/configKey" }, + "webBaseURLKey": { "$ref": "#/definitions/configKey" }, + "serviceAccountAudience": { + "type": "string", + "maxLength": 253, + "pattern": "^$|^[A-Za-z0-9][A-Za-z0-9._:/-]*$" + }, "tokenExpirationSeconds": { "type": "integer", "minimum": 600, "maximum": 3600 } }, "oneOf": [ @@ -84,11 +147,168 @@ { "properties": { "existingSecret": { "maxLength": 0 }, "configMap": { "minLength": 1 }, "serviceAccountAudience": { "minLength": 1 } } } ] }, - "ingress": { "type": "object" }, - "networkPolicy": { "type": "object" }, - "observability": { "type": "object" } + "ingress": { + "type": "object", + "required": ["enabled", "className", "host", "annotations", "tls"], + "properties": { + "enabled": { "type": "boolean" }, + "className": { "$ref": "#/definitions/optionalDNSLabel" }, + "host": { "$ref": "#/definitions/optionalDNSSubdomain" }, + "annotations": { "type": "object" }, + "tls": { + "type": "object", + "required": ["enabled", "secretName"], + "properties": { + "enabled": { "type": "boolean" }, + "secretName": { "$ref": "#/definitions/optionalDNSSubdomain" } + } + } + } + }, + "networkPolicy": { + "type": "object", + "required": ["enabled", "kubernetesApiCIDRs", "modelRouteCIDRs", "ciProviderCIDRs", "dns", "gatewayIngress", "observability"], + "properties": { + "enabled": { "type": "boolean" }, + "kubernetesApiCIDRs": { "$ref": "#/definitions/cidrList" }, + "modelRouteCIDRs": { "$ref": "#/definitions/cidrList" }, + "ciProviderCIDRs": { "$ref": "#/definitions/cidrList" }, + "dns": { + "type": "object", + "required": ["namespaceSelector", "podSelector"], + "properties": { + "namespaceSelector": { "allOf": [{ "$ref": "#/definitions/labelSelector" }], "minProperties": 1 }, + "podSelector": { "allOf": [{ "$ref": "#/definitions/labelSelector" }], "minProperties": 1 } + } + }, + "gatewayIngress": { "$ref": "#/definitions/pairedSelectors" }, + "observability": { "$ref": "#/definitions/pairedSelectors" } + } + }, + "observability": { + "type": "object", + "required": ["serviceMonitor", "prometheusRule"], + "properties": { + "serviceMonitor": { + "type": "object", + "required": ["enabled", "interval", "labels"], + "properties": { + "enabled": { "type": "boolean" }, + "interval": { "type": "string", "pattern": "^[1-9][0-9]*(ms|s|m|h)$" }, + "labels": { "type": "object" } + } + }, + "prometheusRule": { + "type": "object", + "required": ["enabled", "labels"], + "properties": { + "enabled": { "type": "boolean" }, + "labels": { "type": "object" } + } + } + } + } }, "definitions": { + "dnsLabel": { + "type": "string", + "minLength": 1, + "maxLength": 63, + "pattern": "^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$" + }, + "optionalDNSLabel": { + "type": "string", + "maxLength": 63, + "pattern": "^$|^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$" + }, + "dnsSubdomain": { + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?)*$" + }, + "optionalDNSSubdomain": { + "type": "string", + "maxLength": 253, + "pattern": "^$|^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?)*$" + }, + "runtimeProfile": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9]([-._a-z0-9]{0,62}[a-z0-9])?$" + }, + "audience": { + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:/-]*$" + }, + "configKey": { + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^[-._A-Za-z0-9]+$" + }, + "nonDefaultRouteCIDR": { + "type": "string", + "maxLength": 64, + "pattern": "^[0-9A-Fa-f:.]+/([1-9]|[1-9][0-9]|1[01][0-9]|12[0-8])$" + }, + "cidrList": { + "type": "array", + "maxItems": 64, + "uniqueItems": true, + "items": { "$ref": "#/definitions/nonDefaultRouteCIDR" } + }, + "labelSelector": { + "type": "object", + "maxProperties": 2, + "properties": { + "matchLabels": { + "type": "object", + "minProperties": 1, + "maxProperties": 32, + "additionalProperties": { "type": "string", "maxLength": 63 } + }, + "matchExpressions": { + "type": "array", + "minItems": 1, + "maxItems": 32, + "items": { + "type": "object", + "required": ["key", "operator"], + "properties": { + "key": { "type": "string", "minLength": 1, "maxLength": 253 }, + "operator": { "type": "string", "enum": ["In", "NotIn", "Exists", "DoesNotExist"] }, + "values": { "type": "array", "maxItems": 32, "items": { "type": "string", "maxLength": 63 } } + } + } + } + } + }, + "pairedSelectors": { + "type": "object", + "required": ["namespaceSelector", "podSelector"], + "properties": { + "namespaceSelector": { "$ref": "#/definitions/labelSelector" }, + "podSelector": { "$ref": "#/definitions/labelSelector" } + }, + "oneOf": [ + { + "properties": { + "namespaceSelector": { "maxProperties": 0 }, + "podSelector": { "maxProperties": 0 } + } + }, + { + "properties": { + "namespaceSelector": { "minProperties": 1 }, + "podSelector": { "minProperties": 1 } + } + } + ] + }, "image": { "type": "object", "required": ["repository", "digest", "pullPolicy"], @@ -120,6 +340,34 @@ } } } + }, + { + "if": { "properties": { "ingress": { "properties": { "enabled": { "const": true } } } } }, + "then": { + "properties": { + "ingress": { + "properties": { + "className": { "minLength": 1 }, + "host": { "minLength": 1 }, + "tls": { "properties": { "enabled": { "const": true } } } + }, + "oneOf": [ + { + "properties": { + "className": { "const": "tailscale" }, + "tls": { "properties": { "secretName": { "maxLength": 0 } } } + } + }, + { + "properties": { + "className": { "not": { "const": "tailscale" } }, + "tls": { "properties": { "secretName": { "minLength": 1 } } } + } + } + ] + } + } + } } ] } diff --git a/deploy/charts/t4-cluster/values.yaml b/deploy/charts/t4-cluster/values.yaml index 00cfcb75..354a2cbc 100644 --- a/deploy/charts/t4-cluster/values.yaml +++ b/deploy/charts/t4-cluster/values.yaml @@ -9,6 +9,10 @@ clusterHost: maxWorkspaces: 256 resyncSeconds: 60 +kubernetes: + # Audience accepted by this cluster's API server for projected API credentials. + apiAudience: https://kubernetes.default.svc + images: controller: repository: ghcr.io/lycaonllc/t4-cluster-operator @@ -83,6 +87,8 @@ ingress: host: "" annotations: {} tls: + enabled: true + # Leave empty for the Tailscale ingress class; its operator provisions TLS. secretName: "" networkPolicy: @@ -91,6 +97,13 @@ networkPolicy: kubernetesApiCIDRs: [] modelRouteCIDRs: [] ciProviderCIDRs: [] + dns: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns gatewayIngress: # Empty selectors fail closed. Set both to the ingress controller identity. namespaceSelector: {} diff --git a/docs/CLUSTER_OPERATOR.md b/docs/CLUSTER_OPERATOR.md index a0f680db..ab2b3cb4 100644 --- a/docs/CLUSTER_OPERATOR.md +++ b/docs/CLUSTER_OPERATOR.md @@ -47,10 +47,20 @@ images: repository: registry.example/t4-session-runtime digest: sha256:0000000000000000000000000000000000000000000000000000000000000000 pullPolicy: IfNotPresent +kubernetes: + # Set this to an audience accepted by this cluster's API server. + apiAudience: https://kubernetes.default.svc networkPolicy: kubernetesApiCIDRs: [192.0.2.10/32] modelRouteCIDRs: [198.51.100.20/32] ciProviderCIDRs: [203.0.113.30/32] + dns: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns gatewayIngress: namespaceSelector: matchLabels: @@ -69,7 +79,7 @@ session: nodeExclude: [k3s-worker-02] ``` -The sample CIDRs and labels are documentation values, not usable defaults. Keep the chart backend-neutral and set destinations to the actual cluster endpoints. Empty destination lists and selectors deny those flows. +The sample destination CIDRs and gateway/observability labels are documentation values, not usable defaults. Keep the chart backend-neutral and set destinations and integration sources to the actual cluster endpoints. Empty destination lists and paired integration selectors deny those flows. Install or enable with immutable digests: @@ -79,7 +89,11 @@ helm upgrade --install t4-cluster deploy/charts/t4-cluster --namespace t4-system The controller always has two replicas and uses a Kubernetes Lease named from `t4-cluster-operator.cluster.t4.dev`; one replica reconciles at a time. The server defaults to three stateless replicas and supports a minimum of two. Its Deployment uses `maxUnavailable: 0`, a `minAvailable: 2` PDB, topology spread, anti-affinity, readiness draining, and an explicit `k3s-worker-02` exclusion. Session pods also exclude that node by default. Additional cluster-specific exclusions belong in deployment values, not this portable chart. -The chart creates dedicated server and session ServiceAccounts instead of a shared credential Secret. Each server pod receives a 10-minute projected token with the fixed `t4-cluster-internal` audience and presents it only in the existing `omp-app/1` upstream authentication field when dialing a session host. Session pods use `automountServiceAccountToken: false`; their only Kubernetes identity mount is an explicit short-lived API-audience token plus the cluster CA and namespace. The session ServiceAccount may only create `authentication.k8s.io/tokenreviews`, and the host accepts a connection only when TokenReview confirms the expected server ServiceAccount username and audience. +The chart creates dedicated controller, server, and session ServiceAccounts instead of a shared credential Secret. Controller and server pods disable automatic token mounting and receive explicit short-lived Kubernetes API projections. Each server pod also receives a separate 10-minute projected token with the fixed `t4-cluster-internal` audience and presents it only in the existing `omp-app/1` upstream authentication field when dialing a session host. Session pods use `automountServiceAccountToken: false`; their only Kubernetes identity mount is an explicit short-lived API-audience token plus the cluster CA and namespace. The session ServiceAccount may only create `authentication.k8s.io/tokenreviews`, and the host accepts a connection only when TokenReview confirms the expected server ServiceAccount username and audience. + +The Kubernetes API audience is configurable because managed clusters can reject the conventional `https://kubernetes.default.svc` audience. The same value is used for the controller API token, server API token, and session-host TokenReview credential. The internal server identity audience remains the fixed `t4-cluster-internal` boundary. DNS selectors default to the conventional `kube-system`/`k8s-app: kube-dns` pair and can be replaced together for clusters using a different DNS deployment. + +When chart-managed ingress is enabled, a host, ingress class, and TLS stanza are mandatory. For `ingressClassName: tailscale`, the chart renders the TLS host without a Secret reference so the Tailscale operator can provision and manage the certificate. Other ingress classes must reference an administrator-managed TLS Secret. ## API configuration @@ -97,7 +111,7 @@ The optional initial prompt is referenced by Secret name and key `prompt`; the S ## Images and runtime -`cluster/images/controller/Dockerfile`, `cluster/images/cluster-server/Dockerfile`, and `cluster/images/session-runtime/Dockerfile` use digest-pinned build bases. Published chart values also require image digests. +`cluster/images/controller/Dockerfile`, `cluster/images/cluster-server/Dockerfile`, and `cluster/images/session-runtime/Dockerfile` use digest-pinned multi-platform build bases. BuildKit selects the requested target platform; the Dockerfiles do not hardcode or label an architecture that was not built. Debian packages are resolved through a dated snapshot. Published chart values still require per-image immutable digests. The session runtime verifies and builds the exact OMP tag `t4code-17.0.5-appserver-10` at commit `8476f4451ed95c5d5401785d279a93d3c659fac4`. It preserves `t4-omp-authority/1`, starts the existing T4 session-host entrypoint, and provides Xvfb, a minimal window manager, and Chromium without privileged mode or host display access. The shared claim is mounted at `/workspace`; authority and browser state live in controller-selected per-session subdirectories. `/dev/shm` is an explicit memory-backed volume. Browser Preview remains the existing GUI stream and control surface. @@ -105,7 +119,14 @@ Session pods do not receive an automatically mounted ServiceAccount token. The e ## Upgrade and rollback -CRD changes must remain structural and additive. Helm installs files under `crds/` on first install but does not upgrade or delete them. Before `helm upgrade`, an administrator should review and server-side apply the newer CRDs, then upgrade the chart with the new immutable image digests. Wait for the server Deployment to retain at least two ready replicas and for the controller Lease holder to reconcile the new generation. +CRD changes must remain structural and additive. Helm installs files under `crds/` on first install but does not upgrade or delete them. Before every `helm upgrade`, an administrator must review and apply each newer CRD independently, wait for API discovery to serve the updated schemas, and only then upgrade workloads with the new immutable image digests: + +```sh +kubectl apply --server-side -f deploy/charts/t4-cluster/crds/ +kubectl wait --for=condition=Established crd/t4clusterhosts.cluster.t4.dev crd/t4workspaces.cluster.t4.dev crd/t4sessions.cluster.t4.dev +``` + +Do not rely on `helm upgrade` to change CRDs. If a CRD pre-upgrade apply fails validation, stop the upgrade and keep the currently deployed controller/server images; do not force-replace the CRD or remove stored versions. ```sh helm upgrade t4-cluster deploy/charts/t4-cluster --namespace t4-system --values operator-values.yaml diff --git a/packages/cluster-operator/api/v1alpha1/types_test.go b/packages/cluster-operator/api/v1alpha1/types_test.go index 3b9f2fbb..fde778fb 100644 --- a/packages/cluster-operator/api/v1alpha1/types_test.go +++ b/packages/cluster-operator/api/v1alpha1/types_test.go @@ -126,6 +126,18 @@ func TestCRDSchemasAreStructuralBoundedAndValidated(t *testing.T) { t.Fatalf("%s status lacks observedGeneration", crd.Name) } assertBoundedSchema(t, crd.Name+".spec", root.Properties["spec"]) + if crd.Name != "t4clusterhosts.cluster.t4.dev" { + hostRef := root.Properties["spec"].Properties["hostRef"] + immutable := false + for _, validation := range hostRef.XValidations { + if validation.Rule == "self == oldSelf" { + immutable = true + } + } + if !immutable { + t.Fatalf("%s hostRef is mutable", crd.Name) + } + } } } diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go index ad171bc4..116cadf6 100644 --- a/packages/cluster-operator/charttests/chart_contract_test.go +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -49,13 +49,13 @@ func TestEnabledChartRendersHARestrictedWorkloads(t *testing.T) { "coordination.k8s.io", "resources:", ) - server := documentContainingKind(t, output, "Deployment", "name: release-name-t4-cluster-server") + server := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-server\"") assertContains(t, server, "automountServiceAccountToken: false", "name: T4_CLUSTER_TRUSTED_PROXY_CIDRS", - "value: 192.0.2.0/24", + "value: \"192.0.2.0/24\"", "name: kubernetes-api-access", - "audience: https://kubernetes.default.svc", + "audience: \"https://kubernetes.default.svc\"", "expirationSeconds: 3600", ) if strings.Contains(output, "privileged: true") || strings.Contains(output, "hostNetwork: true") || strings.Contains(output, "hostPID: true") { @@ -66,10 +66,111 @@ func TestEnabledChartRendersHARestrictedWorkloads(t *testing.T) { } } +func TestEachDeploymentUsesZeroUnavailableAndConfiguredAPIAudience(t *testing.T) { + output := helmTemplate(t, append(enabledValues(), "--set-string", "kubernetes.apiAudience=kubernetes.custom.example")...) + controller := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-controller\"") + server := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-server\"") + for name, deployment := range map[string]string{"controller": controller, "server": server} { + assertCount(t, deployment, "maxUnavailable: 0", 1) + assertContains(t, deployment, + "automountServiceAccountToken: false", + "name: T4_KUBERNETES_API_AUDIENCE", + "value: \"kubernetes.custom.example\"", + "audience: \"kubernetes.custom.example\"", + ) + if strings.Contains(deployment, "maxUnavailable: 1") { + t.Fatalf("%s Deployment permits an unavailable replica", name) + } + } + assertContains(t, server, "audience: \"t4-cluster-internal\"") +} + +func TestValuesSchemaRejectsUnsafeNamesProfilesCIDRsAndHalfSelectors(t *testing.T) { + for name, values := range map[string][]string{ + "cluster host name": {"--set-string", "clusterHost.name=Bad_Name"}, + "storage class name": {"--set-string", "storage.adminRWXStorageClass=Bad_Name"}, + "runtime profile": {"--set-string", "clusterHost.runtimeProfiles[0]=-bad"}, + "Woodpecker Secret name": {"--set-string", "woodpecker.existingSecret=Bad_Name", "--set-string", "woodpecker.configMap=woodpecker-config"}, + "Woodpecker ConfigMap name": {"--set-string", "woodpecker.existingSecret=woodpecker-token", "--set-string", "woodpecker.configMap=Bad_Name"}, + "Woodpecker key": {"--set-string", "woodpecker.existingSecret=woodpecker-token", "--set-string", "woodpecker.configMap=woodpecker-config", "--set-string", "woodpecker.tokenKey=bad/key"}, + "Woodpecker audience": {"--set-string", "woodpecker.serviceAccountAudience=/bad", "--set-string", "woodpecker.configMap=woodpecker-config"}, + "IPv4 default route": {"--set-string", "server.trustedProxyCIDRs[0]=0.0.0.0/0"}, + "IPv6 default route": {"--set-string", "server.trustedProxyCIDRs[0]=::/0"}, + "gateway half selector": {"--set-string", "networkPolicy.gatewayIngress.namespaceSelector.matchLabels.scope=gateway"}, + "observability half selector": {"--set-string", "networkPolicy.observability.podSelector.matchLabels.scope=metrics"}, + } { + t.Run(name, func(t *testing.T) { + helmTemplateMustFail(t, append(enabledValues(), values...)...) + }) + } +} + +func TestNumericDNSReferencesStayQuoted(t *testing.T) { + output := helmTemplate(t, append(enabledValues(), + "--set-string", "clusterHost.name=123", + "--set-string", "storage.adminRWXStorageClass=456", + )...) + host := documentContainingKind(t, output, "T4ClusterHost", "name: \"123\"") + assertContains(t, host, "storageClassName: \"456\"") +} + +func TestDNSAndSourceSelectorsAreConfigurableAndReleaseScoped(t *testing.T) { + defaults := helmTemplate(t, enabledValues()...) + defaultDNS := documentContainingKind(t, defaults, "NetworkPolicy", "name: \"release-name-t4-cluster-dns\"") + assertContains(t, defaultDNS, "kubernetes.io/metadata.name: kube-system", "k8s-app: kube-dns") + output := helmTemplate(t, append(enabledValues(), + "--set-string", "networkPolicy.dns.namespaceSelector.matchLabels.scope=custom-dns-namespace", + "--set-string", "networkPolicy.dns.podSelector.matchLabels.scope=custom-dns-pod", + "--set-string", "networkPolicy.gatewayIngress.namespaceSelector.matchLabels.scope=gateway-namespace", + "--set-string", "networkPolicy.gatewayIngress.podSelector.matchLabels.scope=gateway-pod", + "--set-string", "networkPolicy.observability.namespaceSelector.matchLabels.scope=metrics-namespace", + "--set-string", "networkPolicy.observability.podSelector.matchLabels.scope=metrics-pod", + )...) + dns := documentContainingKind(t, output, "NetworkPolicy", "name: \"release-name-t4-cluster-dns\"") + assertContains(t, dns, "scope: custom-dns-namespace", "scope: custom-dns-pod") + gateway := documentContainingKind(t, output, "NetworkPolicy", "name: \"release-name-t4-cluster-gateway-ingress\"") + assertContains(t, gateway, "scope: gateway-namespace", "scope: gateway-pod") + metrics := documentContainingKind(t, output, "NetworkPolicy", "name: \"release-name-t4-cluster-observability\"") + assertContains(t, metrics, + "app.kubernetes.io/instance: \"release-name\"", + "app.kubernetes.io/part-of: \"t4-cluster\"", + "scope: metrics-namespace", + "scope: metrics-pod", + ) +} + +func TestIngressRequiresTLSAndSupportsTailscaleManagedCertificates(t *testing.T) { + output := helmTemplate(t, append(enabledValues(), + "--set", "ingress.enabled=true", + "--set-string", "ingress.className=tailscale", + "--set-string", "ingress.host=operator.example.ts.net", + )...) + ingress := documentContainingKind(t, output, "Ingress", "name: \"release-name-t4-cluster\"") + assertContains(t, ingress, + "ingressClassName: \"tailscale\"", + "tls:", + "hosts: [\"operator.example.ts.net\"]", + ) + if strings.Contains(ingress, "secretName:") { + t.Fatal("Tailscale-managed ingress invented a TLS Secret reference") + } + helmTemplateMustFail(t, append(enabledValues(), + "--set", "ingress.enabled=true", + "--set-string", "ingress.className=nginx", + "--set-string", "ingress.host=operator.example.test", + )...) + helmTemplateMustFail(t, append(enabledValues(), + "--set", "ingress.enabled=true", + "--set-string", "ingress.className=tailscale", + "--set-string", "ingress.host=operator.example.ts.net", + "--set", "ingress.tls.enabled=false", + )...) +} + func TestRBACSeparatesControllerMutationFromServerProjection(t *testing.T) { output := helmTemplate(t, enabledValues()...) - controllerRole := documentContaining(t, output, "name: release-name-t4-cluster-controller") - serverRole := documentContaining(t, output, "name: release-name-t4-cluster-server") + controllerRole := documentContaining(t, output, "name: \"release-name-t4-cluster-controller\"") + serverRole := documentContaining(t, output, "name: \"release-name-t4-cluster-server\"") assertContains(t, controllerRole, "persistentvolumeclaims", "pods", "services", "t4sessions/status", "leases") assertContains(t, serverRole, "t4clusterhosts", "t4workspaces", "t4sessions", "create", "list", "watch") if strings.Contains(serverRole, "secrets") || strings.Contains(serverRole, "persistentvolumeclaims") || strings.Contains(serverRole, "t4sessions/status") { @@ -81,23 +182,23 @@ func TestChartUsesOnlyProjectedServiceAccountIdentityForInternalPeers(t *testing output := helmTemplate(t, enabledValues()...) assertCount(t, output, "kind: ServiceAccount", 3) assertCount(t, output, "kind: Secret", 0) - server := documentContainingKind(t, output, "Deployment", "name: release-name-t4-cluster-server") + server := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-server\"") assertContains(t, server, - "serviceAccountName: release-name-t4-cluster-server", + "serviceAccountName: \"release-name-t4-cluster-server\"", "name: T4_CLUSTER_IDENTITY_TOKEN_FILE", "/var/run/secrets/t4-cluster-identity/token", "serviceAccountToken:", - "audience: t4-cluster-internal", + "audience: \"t4-cluster-internal\"", "expirationSeconds: 600", ) - controller := documentContainingKind(t, output, "Deployment", "name: release-name-t4-cluster-controller") + controller := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-controller\"") assertContains(t, controller, "name: T4_SESSION_SERVICE_ACCOUNT", - "value: release-name-t4-cluster-session", + "value: \"release-name-t4-cluster-session\"", "name: T4_CLUSTER_SERVER_SERVICE_ACCOUNT", - "value: release-name-t4-cluster-server", + "value: \"release-name-t4-cluster-server\"", ) - sessionRole := documentContainingKind(t, output, "ClusterRole", "name: release-name-t4-cluster-session-token-reviewer") + sessionRole := documentContainingKind(t, output, "ClusterRole", "name: \"release-name-t4-cluster-session-token-reviewer\"") assertContains(t, sessionRole, "apiGroups: [authentication.k8s.io]", "resources: [tokenreviews]", @@ -115,14 +216,14 @@ func TestNetworkPoliciesDefaultDenyAndAllowOnlyDeclaredFlows(t *testing.T) { "--set", "networkPolicy.ciProviderCIDRs[0]=203.0.113.8/32", )...) assertContains(t, output, - "name: release-name-t4-cluster-default-deny", + "name: \"release-name-t4-cluster-default-deny\"", "192.0.2.10/32", "198.51.100.4/32", "203.0.113.8/32", "port: 53", "port: 8787", ) - sessionPolicy := documentContainingKind(t, output, "NetworkPolicy", "name: release-name-t4-cluster-session-host") + sessionPolicy := documentContainingKind(t, output, "NetworkPolicy", "name: \"release-name-t4-cluster-session-host\"") assertContains(t, sessionPolicy, "192.0.2.10/32", "port: 443", "port: 6443") if strings.Contains(output, "0.0.0.0/0") { t.Fatal("network policy contains broad Internet egress") @@ -135,15 +236,15 @@ func TestWoodpeckerCanUseRotatingProjectedServiceAccountIdentity(t *testing.T) { "--set", "woodpecker.serviceAccountAudience=woodpecker-ci-trigger", ) output := helmTemplate(t, values...) - server := documentContainingKind(t, output, "Deployment", "name: release-name-t4-cluster-server") + server := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-server\"") assertContains(t, server, "name: T4_WOODPECKER_TOKEN_FILE", "/var/run/secrets/t4-ci/token", - "audience: woodpecker-ci-trigger", + "audience: \"woodpecker-ci-trigger\"", "expirationSeconds: 600", ) - host := documentContainingKind(t, output, "T4ClusterHost", "name: t4-cluster") - assertContains(t, host, "serviceAccountAudience: woodpecker-ci-trigger", "name: woodpecker-config") + host := documentContainingKind(t, output, "T4ClusterHost", "name: \"t4-cluster\"") + assertContains(t, host, "serviceAccountAudience: \"woodpecker-ci-trigger\"", "name: \"woodpecker-config\"") } func TestCRDsRemainExplicitAcrossUpgradeAndUninstall(t *testing.T) { @@ -159,7 +260,7 @@ func TestCRDsRemainExplicitAcrossUpgradeAndUninstall(t *testing.T) { if err != nil { t.Fatal(err) } - for _, required := range []string{"helm upgrade", "helm rollback", "helm uninstall", "Retain", "Delete", "CRDs are not removed"} { + for _, required := range []string{"helm upgrade", "helm rollback", "helm uninstall", "kubectl apply --server-side -f deploy/charts/t4-cluster/crds/", "condition=Established", "Do not rely on `helm upgrade` to change CRDs", "Retain", "Delete", "CRDs are not removed"} { if !strings.Contains(string(docs), required) { t.Fatalf("operator guide lacks upgrade/uninstall contract %q", required) } @@ -190,6 +291,19 @@ func TestImageContractsArePinnedAndAuthorityCompatible(t *testing.T) { t.Fatalf("%s image synthesizes an uncommitted dependency lock", name) } } + if strings.Contains(session, "ARG BUN_IMAGE") || strings.Contains(session, "ARG OMP_TAG") || strings.Contains(session, "ARG OMP_COMMIT") { + t.Fatal("session runtime permits overriding a labeled runtime pin") + } + assertContains(t, session, + "refs/tags/t4code-17.0.5-appserver-10", + "git checkout --detach \"8476f4451ed95c5d5401785d279a93d3c659fac4\"", + "snapshot.debian.org/archive/debian/20250721T000000Z", + ) + assertContains(t, server, "snapshot.debian.org/archive/debian/20250721T000000Z") + assertContains(t, controller, "ARG TARGETOS\n", "ARG TARGETARCH\n") + if strings.Contains(controller, "TARGETARCH=amd64") || strings.Contains(controller, "org.opencontainers.image.architecture") { + t.Fatal("controller image hardcodes or claims a single/unbuilt architecture") + } assertContains(t, server, "packages/cluster-server/src/main.ts") entrypoint := mustRead(t, filepath.Join(root, "cluster", "images", "session-runtime", "session-entrypoint.sh")) assertContains(t, entrypoint, @@ -211,6 +325,16 @@ func helmTemplate(t *testing.T, extra ...string) string { } return string(output) } +func helmTemplateMustFail(t *testing.T, extra ...string) { + t.Helper() + args := []string{"template", "release-name", filepath.Join(repoRoot(t), "deploy", "charts", "t4-cluster"), "--namespace", "t4-system"} + args = append(args, extra...) + command := exec.Command("helm", args...) + if output, err := command.CombinedOutput(); err == nil { + t.Fatalf("helm unexpectedly accepted invalid values: %s", output) + } +} + func enabledValues() []string { return []string{ diff --git a/packages/cluster-operator/cmd/manager/main.go b/packages/cluster-operator/cmd/manager/main.go index 23a5bede..047c1e28 100644 --- a/packages/cluster-operator/cmd/manager/main.go +++ b/packages/cluster-operator/cmd/manager/main.go @@ -81,6 +81,7 @@ func main() { RuntimeImage: os.Getenv("T4_SESSION_RUNTIME_IMAGE"), SessionServiceAccountName: sessionServiceAccount, ServerServiceAccountName: serverServiceAccount, + KubernetesAPIAudience: envOr("T4_KUBERNETES_API_AUDIENCE", controllers.DefaultKubernetesAPIAudience), ExcludedNodeNames: excludedNodes, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 1445df54..6da8ff6d 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -209,7 +209,7 @@ func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) c := fake.NewClientBuilder().WithScheme(scheme). WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). WithObjects(testHost(), workspace, pvc, session).Build() - r := &controllers.SessionReconciler{Client: c, Scheme: scheme, RuntimeImage: "registry.example/t4/session@sha256:0123456789abcdef"} + r := &controllers.SessionReconciler{Client: c, Scheme: scheme, RuntimeImage: "registry.example/t4/session@sha256:0123456789abcdef", KubernetesAPIAudience: "kubernetes.custom.example"} reconcileMany(t, 2, func() error { _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) return err @@ -262,6 +262,15 @@ func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) if serverIdentity != controllers.DefaultServerServiceAccount { t.Fatalf("expected server ServiceAccount = %q", serverIdentity) } + apiAudience := "" + for _, env := range pod.Spec.Containers[0].Env { + if env.Name == "T4_KUBERNETES_API_AUDIENCE" { + apiAudience = env.Value + } + } + if apiAudience != r.KubernetesAPIAudience { + t.Fatalf("reviewer API audience environment = %q", apiAudience) + } if !hasMount(pod.Spec.Containers[0].VolumeMounts, "kubernetes-api-access", "/var/run/secrets/kubernetes.io/serviceaccount") { t.Fatal("explicit Kubernetes reviewer projection is not mounted") } @@ -275,7 +284,7 @@ func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) t.Fatalf("Kubernetes reviewer projection = %#v", projection) } serviceToken := projection.Sources[0].ServiceAccountToken - if serviceToken == nil || serviceToken.Audience != controllers.KubernetesAPIAudience || serviceToken.ExpirationSeconds == nil || *serviceToken.ExpirationSeconds != controllers.SessionReviewerTokenExpirationSeconds || serviceToken.Path != "token" { + if serviceToken == nil || serviceToken.Audience != r.KubernetesAPIAudience || serviceToken.ExpirationSeconds == nil || *serviceToken.ExpirationSeconds != controllers.SessionReviewerTokenExpirationSeconds || serviceToken.Path != "token" { t.Fatalf("reviewer token projection = %#v", serviceToken) } clusterCA := projection.Sources[1].ConfigMap @@ -368,13 +377,199 @@ func TestSessionRecreatesPodWhenImmutableDesiredStateChanges(t *testing.T) { } } +func TestSessionRecreatesExternallyExposedOwnedService(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := &controllers.SessionReconciler{Client: c, Scheme: scheme, RuntimeImage: "registry.example/session@sha256:deadbeef"} + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + serviceKey := types.NamespacedName{Namespace: "team", Name: controllers.SessionServiceName(session)} + var service corev1.Service + if err := c.Get(context.Background(), serviceKey, &service); err != nil { + t.Fatal(err) + } + service.Spec.Type = corev1.ServiceTypeNodePort + service.Spec.ExternalIPs = []string{"192.0.2.8"} + service.Spec.Ports[0].NodePort = 32080 + if err := c.Update(context.Background(), &service); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if err := c.Get(context.Background(), serviceKey, &service); !apierrors.IsNotFound(err) { + t.Fatalf("externally exposed Service was not deleted for safe recreation: %v", err) + } + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + if err := c.Get(context.Background(), serviceKey, &service); err != nil { + t.Fatal(err) + } + if service.Spec.Type != corev1.ServiceTypeClusterIP || len(service.Spec.ExternalIPs) != 0 || service.Spec.Ports[0].NodePort != 0 { + t.Fatalf("recreated Service retains external exposure: %#v", service.Spec) + } +} + +func TestSessionRestoresRequiredPodSelectorLabelsBeforeAvailability(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := &controllers.SessionReconciler{Client: c, Scheme: scheme, RuntimeImage: "registry.example/session@sha256:deadbeef"} + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + podKey := types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)} + var pod corev1.Pod + if err := c.Get(context.Background(), podKey, &pod); err != nil { + t.Fatal(err) + } + delete(pod.Labels, "cluster.t4.dev/session") + if err := c.Update(context.Background(), &pod); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if err := c.Get(context.Background(), podKey, &pod); err != nil { + t.Fatal(err) + } + if pod.Labels["cluster.t4.dev/session"] != controllers.SessionPodName(session) { + t.Fatalf("required selector labels were not restored: %#v", pod.Labels) + } + var got clusterv1alpha1.T4Session + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + if got.Status.Phase == clusterv1alpha1.InfrastructureRunning { + t.Fatal("session became available in the same reconcile that repaired endpoint labels") + } +} + +func TestWorkspaceDeletionRefusesForeignDeterministicPVC(t *testing.T) { + for _, policy := range []clusterv1alpha1.RetentionPolicy{clusterv1alpha1.RetentionPolicyRetain, clusterv1alpha1.RetentionPolicyDelete} { + for _, mismatch := range []string{"uid-annotation", "controller-owner"} { + t.Run(string(policy)+"/"+mismatch, func(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(policy) + workspace.UID = "workspace-uid" + workspace.Finalizers = []string{clusterv1alpha1.WorkspaceFinalizer} + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: controllers.WorkspacePVCName(workspace), Namespace: workspace.Namespace, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, + }} + if mismatch == "uid-annotation" { + pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] = "foreign-workspace-uid" + } else { + pvc.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: "foreign", UID: "foreign-workspace-uid", Controller: ptr(true), + }} + } + expectedOwnerCount := len(pvc.OwnerReferences) + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc).Build() + if err := c.Delete(context.Background(), workspace); err != nil { + t.Fatal(err) + } + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var untouched corev1.PersistentVolumeClaim + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pvc), &untouched); err != nil { + t.Fatalf("foreign deterministic PVC was deleted: %v", err) + } + if untouched.Annotations[clusterv1alpha1.RetainedPVCAnnotation] != "" || len(untouched.OwnerReferences) != expectedOwnerCount { + t.Fatalf("foreign deterministic PVC was mutated: %#v", untouched.ObjectMeta) + } + var waiting clusterv1alpha1.T4Workspace + if err := c.Get(context.Background(), client.ObjectKeyFromObject(workspace), &waiting); err != nil { + t.Fatalf("workspace finalizer was removed on conflict: %v", err) + } + condition := findCondition(waiting.Status.Conditions, "Ready") + if !contains(waiting.Finalizers, clusterv1alpha1.WorkspaceFinalizer) || condition == nil || condition.Reason != "CleanupOwnershipConflict" { + t.Fatalf("workspace cleanup conflict not retained: %#v", waiting) + } + }) + } + } +} + +func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { + for _, foreignKind := range []string{"Pod", "Service"} { + t.Run(foreignKind, func(t *testing.T) { + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + session.Finalizers = []string{clusterv1alpha1.SessionFinalizer} + controller := true + ownerReferences := []metav1.OwnerReference{{ + APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Session", Name: session.Name, UID: session.UID, Controller: &controller, + }} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace, OwnerReferences: ownerReferences}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace, OwnerReferences: ownerReferences}} + if foreignKind == "Pod" { + pod.OwnerReferences = nil + } else { + service.OwnerReferences = nil + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() + if err := c.Delete(context.Background(), session); err != nil { + t.Fatal(err) + } + r := &controllers.SessionReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, c, 1, 1) + var waiting clusterv1alpha1.T4Session + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &waiting); err != nil { + t.Fatalf("session finalizer was removed on cleanup conflict: %v", err) + } + condition := findCondition(waiting.Status.Conditions, "Available") + if !contains(waiting.Finalizers, clusterv1alpha1.SessionFinalizer) || condition == nil || condition.Reason != "CleanupOwnershipConflict" { + t.Fatalf("session cleanup conflict not retained: %#v", waiting) + } + }) + } +} + + func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { scheme := testScheme(t) session := testSession() session.UID = "session-uid" session.Finalizers = []string{clusterv1alpha1.SessionFinalizer} - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: "team"}} - service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: "team"}} + controller := true + ownerReferences := []metav1.OwnerReference{{ + APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Session", Name: session.Name, UID: session.UID, Controller: &controller, + }} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: "team", OwnerReferences: ownerReferences}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: "team", OwnerReferences: ownerReferences}} c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() if err := c.Delete(context.Background(), session); err != nil { t.Fatal(err) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index feba2a8d..2567f41c 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -27,7 +27,7 @@ import ( const ( DefaultSessionServiceAccount = "t4-cluster-session" DefaultServerServiceAccount = "t4-cluster-server" - KubernetesAPIAudience = "https://kubernetes.default.svc" + DefaultKubernetesAPIAudience = "https://kubernetes.default.svc" SessionReviewerTokenExpirationSeconds int64 = 3600 ) @@ -37,6 +37,7 @@ type SessionReconciler struct { RuntimeImage string SessionServiceAccountName string ServerServiceAccountName string + KubernetesAPIAudience string ExcludedNodeNames []string Resources corev1.ResourceRequirements SharedMemorySize apiresource.Quantity @@ -122,6 +123,14 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{}, err } else if !metav1.IsControlledBy(&service, &session) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "Available", "ServiceOwnershipConflict", "deterministic session Service is not controlled by this session") + } else if !serviceExposureIsInternal(&service) { + if err := r.Delete(ctx, &service); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err := r.updateSessionPending(ctx, &session, podName, serviceName, "ServiceExposureChanged", "session Service is being recreated with ClusterIP-only exposure"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Second}, nil } else if !reflect.DeepEqual(service.Spec.Selector, desiredService.Spec.Selector) || !reflect.DeepEqual(service.Spec.Ports, desiredService.Spec.Ports) || !reflect.DeepEqual(service.Labels, desiredService.Labels) { service.Spec.Selector = desiredService.Spec.Selector service.Spec.Ports = desiredService.Spec.Ports @@ -148,6 +157,20 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{}, err } else if !metav1.IsControlledBy(&pod, &session) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "Available", "PodOwnershipConflict", "deterministic session Pod is not controlled by this session") + } else if !labelsContain(pod.Labels, desiredPod.Labels) { + if pod.Labels == nil { + pod.Labels = map[string]string{} + } + for key, value := range desiredPod.Labels { + pod.Labels[key] = value + } + if err := r.Update(ctx, &pod); err != nil { + return ctrl.Result{}, err + } + if err := r.updateSessionPending(ctx, &session, podName, serviceName, "PodLabelsChanged", "session Pod selector labels are being restored"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Second}, nil } else if pod.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] != desiredPod.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] { if err := r.Delete(ctx, &pod); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err @@ -216,6 +239,10 @@ func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcNa if serverServiceAccount == "" { serverServiceAccount = DefaultServerServiceAccount } + kubernetesAPIAudience := r.KubernetesAPIAudience + if kubernetesAPIAudience == "" { + kubernetesAPIAudience = DefaultKubernetesAPIAudience + } excluded := r.ExcludedNodeNames if len(excluded) == 0 { excluded = []string{"k3s-worker-02"} @@ -229,6 +256,7 @@ func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcNa {Name: "T4_KUBERNETES_TOKEN_PATH", Value: "/var/run/secrets/kubernetes.io/serviceaccount/token"}, {Name: "T4_KUBERNETES_CA_PATH", Value: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"}, {Name: "T4_KUBERNETES_NAMESPACE_PATH", Value: "/var/run/secrets/kubernetes.io/serviceaccount/namespace"}, + {Name: "T4_KUBERNETES_API_AUDIENCE", Value: kubernetesAPIAudience}, {Name: "T4_SESSION_NAME", Value: session.Name}, {Name: "T4_WORKSPACE_ROOT", Value: "/workspace"}, {Name: "T4_SESSION_STATE_ROOT", Value: "/workspace/.t4/sessions/" + stateID}, @@ -261,7 +289,7 @@ func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcNa {Name: "kubernetes-api-access", VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ DefaultMode: ptr(int32(0440)), Sources: []corev1.VolumeProjection{ - {ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Audience: KubernetesAPIAudience, ExpirationSeconds: ptr(SessionReviewerTokenExpirationSeconds), Path: "token"}}, + {ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Audience: kubernetesAPIAudience, ExpirationSeconds: ptr(SessionReviewerTokenExpirationSeconds), Path: "token"}}, {ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "kube-root-ca.crt"}, Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}}}, {DownwardAPI: &corev1.DownwardAPIProjection{Items: []corev1.DownwardAPIVolumeFile{{Path: "namespace", FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}}, }, @@ -325,24 +353,42 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste return ctrl.Result{}, err } } - remaining := false - for _, object := range []client.Object{ + objects := []client.Object{ &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: SessionPodName(session), Namespace: session.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, - } { + } + existing := make([]client.Object, 0, len(objects)) + for _, object := range objects { err := r.Get(ctx, client.ObjectKeyFromObject(object), object) - if err == nil { - remaining = true - if object.GetDeletionTimestamp().IsZero() { - if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return ctrl.Result{}, err + } + if !metav1.IsControlledBy(object, session) { + before := session.Status + if session.Status.Conditions != nil { + before.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "CleanupOwnershipConflict", fmt.Sprintf("deterministic %T is not controlled by this session", object), session.Generation)) + if !reflect.DeepEqual(before, session.Status) { + if err := r.Status().Update(ctx, session); err != nil { return ctrl.Result{}, err } } - } else if !apierrors.IsNotFound(err) { - return ctrl.Result{}, err + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } + existing = append(existing, object) } - if remaining { + for _, object := range existing { + if object.GetDeletionTimestamp().IsZero() { + if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + } + if len(existing) > 0 { return ctrl.Result{RequeueAfter: time.Second}, nil } controllerutil.RemoveFinalizer(session, clusterv1alpha1.SessionFinalizer) @@ -363,6 +409,29 @@ func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *c return r.Status().Update(ctx, session) } +func serviceExposureIsInternal(service *corev1.Service) bool { + if service.Spec.Type != corev1.ServiceTypeClusterIP || service.Spec.ClusterIP == corev1.ClusterIPNone || service.Spec.ExternalName != "" || + len(service.Spec.ExternalIPs) != 0 || service.Spec.LoadBalancerIP != "" || len(service.Spec.LoadBalancerSourceRanges) != 0 || + service.Spec.LoadBalancerClass != nil || service.Spec.HealthCheckNodePort != 0 || service.Spec.AllocateLoadBalancerNodePorts != nil { + return false + } + for _, port := range service.Spec.Ports { + if port.NodePort != 0 { + return false + } + } + return true +} + +func labelsContain(actual, required map[string]string) bool { + for key, value := range required { + if actual[key] != value { + return false + } + } + return true +} + func (r *SessionReconciler) SetupWithManager(manager ctrl.Manager) error { return ctrl.NewControllerManagedBy(manager). For(&clusterv1alpha1.T4Session{}). diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index f284982b..d85c317c 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -203,6 +203,19 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl pvcKey := types.NamespacedName{Namespace: workspace.Namespace, Name: WorkspacePVCName(workspace)} var pvc corev1.PersistentVolumeClaim err := r.Get(ctx, pvcKey, &pvc) + if err == nil && !workspaceOwnsPVC(workspace, &pvc) { + before := workspace.Status + if workspace.Status.Conditions != nil { + before.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) + } + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "CleanupOwnershipConflict", "deterministic workspace PVC does not belong to this workspace", workspace.Generation)) + if !reflect.DeepEqual(before, workspace.Status) { + if err := r.Status().Update(ctx, workspace); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyRetain { if err == nil { before := pvc.DeepCopy() diff --git a/packages/cluster-server/src/config.ts b/packages/cluster-server/src/config.ts index c5f34c53..f7bfddbc 100644 --- a/packages/cluster-server/src/config.ts +++ b/packages/cluster-server/src/config.ts @@ -3,6 +3,7 @@ import { isAbsolute } from "node:path"; import { isIP } from "node:net"; const DNS_LABEL = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; +const AUDIENCE = /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/u; export interface ClusterServerConfig { readonly namespace: string; readonly podName: string; @@ -13,6 +14,7 @@ export interface ClusterServerConfig { readonly kubernetesBaseUrl: string; readonly kubernetesTokenPath: string; readonly kubernetesCaPath: string; + readonly kubernetesApiAudience: string; readonly identityTokenPath: string; readonly serverServiceAccountName: string; readonly trustedProxyAddresses: readonly string[]; @@ -34,6 +36,10 @@ function dns(value: string, name: string): string { if (!DNS_LABEL.test(value)) throw new Error(`${name} is invalid`); return value; } +function audience(value: string, name: string): string { + if (value.length > 253 || !AUDIENCE.test(value)) throw new Error(`${name} is invalid`); + return value; +} function port(value: string | undefined, fallback: number, name: string): number { const result = Number(value ?? fallback); if (!Number.isSafeInteger(result) || result < 1 || result > 65_535) throw new Error(`${name} is invalid`); @@ -81,7 +87,7 @@ function canonicalCidr(value: string): string { const family = isIP(address); const prefix = Number(pieces[1]); const width = family === 4 ? 32 : family === 6 ? 128 : 0; - if (prefix > width) throw new Error("T4_CLUSTER_TRUSTED_PROXY_CIDRS is invalid"); + if (prefix < 1 || prefix > width) throw new Error("T4_CLUSTER_TRUSTED_PROXY_CIDRS is invalid"); const canonicalAddress = family === 4 ? address.split(".").map(Number).join(".") : new URL(`http://[${address}]/`).hostname.slice(1, -1); @@ -132,6 +138,7 @@ export function clusterServerConfigFromEnv(env: Readonly; features: Set; } @@ -96,6 +97,7 @@ export interface SessionHostConfig { readonly kubernetesTokenPath: string; readonly kubernetesCaPath: string; readonly kubernetesNamespacePath: string; + readonly kubernetesApiAudience: string; readonly serverServiceAccountName: string; readonly sessionName: string; readonly ompExecutable: string; @@ -115,6 +117,10 @@ function absolutePath(value: string, name: string): string { if (!isAbsolute(value)) throw new Error(`${name} must be absolute`); return value; } +function audience(value: string): string { + if (value.length > 253 || !AUDIENCE.test(value)) throw new Error("T4_KUBERNETES_API_AUDIENCE is invalid"); + return value; +} export function sessionHostConfigFromEnv(env: Readonly>): SessionHostConfig { const serviceHost = required(env, "KUBERNETES_SERVICE_HOST"); const servicePort = Number(env.KUBERNETES_SERVICE_PORT_HTTPS ?? env.KUBERNETES_SERVICE_PORT ?? "443"); @@ -133,6 +139,7 @@ export function sessionHostConfigFromEnv(env: Readonly { serverServiceAccountName: "release-t4-cluster-server", kubernetesTokenPath: "/var/run/secrets/kubernetes.io/serviceaccount/token", kubernetesCaPath: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + kubernetesApiAudience: "https://kubernetes.default.svc", }); expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_IDENTITY_TOKEN_FILE: "relative/token" })).toThrow("absolute"); + expect(clusterServerConfigFromEnv({ ...BASE_ENV, T4_KUBERNETES_API_AUDIENCE: "kubernetes.custom.example" }).kubernetesApiAudience).toBe("kubernetes.custom.example"); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_KUBERNETES_API_AUDIENCE: "/invalid" })).toThrow("T4_KUBERNETES_API_AUDIENCE"); }); it("reads only a bounded regular projected identity file", async () => { @@ -77,6 +80,8 @@ describe("trusted cluster gateway proxy sources", () => { it("rejects CIDRs with host bits or non-canonical notation", () => { expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: "10.42.1.7/16" })).toThrow(); expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: "fd7a:115c:a1e0:0::/48" })).toThrow(); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: "0.0.0.0/0" })).toThrow(); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: "::/0" })).toThrow(); }); it("bounds the trusted CIDR list", () => { diff --git a/packages/cluster-server/test/session-host-policy.test.ts b/packages/cluster-server/test/session-host-policy.test.ts index 6ad253bc..42a4e652 100644 --- a/packages/cluster-server/test/session-host-policy.test.ts +++ b/packages/cluster-server/test/session-host-policy.test.ts @@ -87,6 +87,7 @@ describe("one-session pod host authority", () => { expect(sessionHostConfigFromEnv({ KUBERNETES_SERVICE_HOST: "10.96.0.1", KUBERNETES_SERVICE_PORT_HTTPS: "443", + T4_KUBERNETES_API_AUDIENCE: "kubernetes.custom.example", T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "release-t4-cluster-server", T4_SESSION_NAME: "session-one", T4_OMP_EXECUTABLE: "/opt/t4/bin/omp", @@ -97,6 +98,7 @@ describe("one-session pod host authority", () => { kubernetesTokenPath: "/var/run/secrets/kubernetes.io/serviceaccount/token", kubernetesCaPath: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", kubernetesNamespacePath: "/var/run/secrets/kubernetes.io/serviceaccount/namespace", + kubernetesApiAudience: "kubernetes.custom.example", serverServiceAccountName: "release-t4-cluster-server", sessionName: "session-one", ompExecutable: "/opt/t4/bin/omp", @@ -104,5 +106,6 @@ describe("one-session pod host authority", () => { port: 8787, }); expect(() => sessionHostConfigFromEnv({ KUBERNETES_SERVICE_HOST: "10.96.0.1", T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "server", T4_SESSION_NAME: "bad/name" })).toThrow("T4_SESSION_NAME"); + expect(() => sessionHostConfigFromEnv({ KUBERNETES_SERVICE_HOST: "10.96.0.1", T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "server", T4_SESSION_NAME: "session", T4_SESSION_STATE_ROOT: "/workspace/.t4/sessions/session", T4_KUBERNETES_API_AUDIENCE: "/invalid" })).toThrow("T4_KUBERNETES_API_AUDIENCE"); }); }); From 8e1ba92cd7df1d78bbd697567f904290bdc62dc3 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 17:37:01 +0000 Subject: [PATCH 25/98] fix(ci): bind cluster proof and image evidence --- .dockerignore | 2 + .woodpecker.yml | 159 ++++++--- .../cluster-ci/assemble-image-manifest.mjs | 242 +++++++++++-- scripts/cluster-ci/assemble-proof.mjs | 212 +++++++---- scripts/cluster-ci/build-image.sh | 20 +- scripts/cluster-ci/capture-image-evidence.sh | 30 +- .../cluster-ci/capture-redacted-frames.mjs | 44 ++- .../cluster-ci/cluster-ci-contract.test.mjs | 330 +++++++++++++++--- scripts/cluster-ci/cluster-proof.schema.json | 42 ++- .../cluster-ci/collect-readonly-cluster.mjs | 41 ++- .../cluster-ci/normalize-registry-auth.mjs | 4 +- scripts/cluster-ci/promote-images.sh | 43 +++ scripts/cluster-ci/proof-contract.mjs | 63 +++- scripts/cluster-ci/publish-artifact.sh | 7 +- scripts/cluster-ci/readonly-cluster-proof.mjs | 252 +++++++++---- 15 files changed, 1162 insertions(+), 329 deletions(-) create mode 100755 scripts/cluster-ci/promote-images.sh diff --git a/.dockerignore b/.dockerignore index 30750596..1c4b766d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,8 @@ .git .gitignore .cluster-ci +.cluster-ci/registry-auth +**/.cluster-ci/registry-auth artifacts **/artifacts node_modules diff --git a/.woodpecker.yml b/.woodpecker.yml index 4a2c52c5..051cf93b 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -6,14 +6,14 @@ when: clone: git: - image: mirror.gcr.io/woodpeckerci/plugin-git:2.6.5 + image: mirror.gcr.io/woodpeckerci/plugin-git:2.6.5@sha256:2b34ecbbfd2f318405bd5d2765a750f6b02dae5dfb8707e16c943b9faeef5fc5 settings: depth: 50 partial: false steps: dependencies: - image: mirror.gcr.io/library/node:24.13.1-bookworm + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 commands: - corepack enable - pnpm install --frozen-lockfile @@ -33,7 +33,7 @@ steps: upstream-core: depends_on: [cluster-ci-contracts] - image: mirror.gcr.io/library/node:24.13.1-bookworm + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 commands: - corepack enable - pnpm check @@ -56,7 +56,7 @@ steps: upstream-tooling: depends_on: [upstream-core] - image: mirror.gcr.io/library/node:24.13.1-bookworm + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 commands: - corepack enable - pnpm test:tooling @@ -75,7 +75,7 @@ steps: legacy-authority-source: depends_on: [upstream-tooling] - image: mirror.gcr.io/alpine/git:2.49.1 + image: mirror.gcr.io/alpine/git:2.49.1@sha256:c0280cf9572316299b08544065d3bf35db65043d5e3963982ec50647d2746e26 commands: - git init .continuity/omp - git -C .continuity/omp remote add origin https://github.com/lyc-aon/oh-my-pi.git @@ -97,7 +97,7 @@ steps: legacy-authority-build: depends_on: [legacy-authority-source] - image: mirror.gcr.io/oven/bun:1.3.14 + image: mirror.gcr.io/oven/bun:1.3.14@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 commands: - cd .continuity/omp && bun install --frozen-lockfile - cd .continuity/omp && bun run build:native @@ -116,7 +116,7 @@ steps: legacy-bridge-continuity: depends_on: [legacy-authority-build] - image: mirror.gcr.io/library/node:24.13.1-bookworm + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 environment: T4_OMP_SOURCE_DIR: /woodpecker/src/.continuity/omp commands: @@ -137,7 +137,7 @@ steps: android-debug: depends_on: [legacy-bridge-continuity] - image: cimg/android:2026.03.1-node + image: cimg/android:2026.03.1-node@sha256:b62bbea69ba6bb28fe165ff913c23bde5220d0305fed7247121fb2919ecffd0e commands: - corepack enable - pnpm --filter @t4-code/mobile check:android:debug @@ -156,7 +156,7 @@ steps: cluster-ci-contracts: depends_on: [dependencies] - image: mirror.gcr.io/library/node:24.13.1-bookworm + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 commands: - corepack enable - pnpm test:cluster:ci @@ -175,7 +175,7 @@ steps: cluster-server-tests: depends_on: [cluster-ci-contracts] - image: mirror.gcr.io/library/node:24.13.1-bookworm + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 environment: T4_PROOF_OUTPUT_DIR: artifacts/cluster-proof/scenarios commands: @@ -197,7 +197,7 @@ steps: cluster-wire-tests: depends_on: [cluster-server-tests] - image: mirror.gcr.io/library/node:24.13.1-bookworm + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 commands: - corepack enable - pnpm --filter @t4-code/host-wire test @@ -217,7 +217,7 @@ steps: cluster-operator-tests: depends_on: [cluster-wire-tests] - image: mirror.gcr.io/library/golang:1.25.1-bookworm + image: mirror.gcr.io/library/golang:1.25.1-bookworm@sha256:c423747fbd96fd8f0b1102d947f51f9b266060217478e5f9bf86f145969562ee directory: packages/cluster-operator commands: - go test ./api/... ./controllers/... ./cmd/... @@ -238,7 +238,7 @@ steps: cluster-chart-tests: depends_on: [cluster-operator-tests] - image: mirror.gcr.io/alpine/helm:3.19.0 + image: mirror.gcr.io/alpine/helm:3.19.0@sha256:aef9b56f64e866207d9591d0abd8f6d767b36aadd12edf68f8a719716d9d29c9 directory: packages/cluster-operator/charttests commands: - helm lint ../../../deploy/charts/t4-cluster @@ -265,6 +265,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-dev-verifier @@ -280,10 +281,10 @@ steps: build-controller: depends_on: [harbor-auth] - image: mirror.gcr.io/moby/buildkit:v0.30.0-rootless + image: mirror.gcr.io/moby/buildkit:v0.30.0-rootless@sha256:d76eb1caecac5733ef7553c1e90a1b21f1bb218cd1142d3553de0747b4a14ba9 environment: BUILDKIT_ADDR: tcp://woodpecker-buildkitd.linkedin-ci.svc.cluster.local:1234 - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/build-image.sh controller t4-cluster-operator cluster/images/controller/Dockerfile @@ -291,6 +292,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -306,9 +308,9 @@ steps: sbom-controller: depends_on: [build-session-runtime] - image: anchore/syft:1.33.0 + image: anchore/syft:v1.33.0@sha256:f94e5d9fce1f2278491a8e3a63bd5f6ddb81fdfdbb8bf7a1637565c1d5344357 environment: - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/capture-image-evidence.sh sbom controller t4-cluster-operator @@ -316,6 +318,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -331,9 +334,9 @@ steps: vulnerability-controller: depends_on: [sbom-controller] - image: aquasec/trivy:0.66.0 + image: aquasec/trivy:0.66.0@sha256:086971aaf400beebd94e8300fd8ea623774419597169156cec56eec5b00dfb1e environment: - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/capture-image-evidence.sh vulnerability controller t4-cluster-operator @@ -341,6 +344,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -356,9 +360,9 @@ steps: provenance-controller: depends_on: [vulnerability-controller] - image: gcr.io/projectsigstore/cosign:v2.6.0 + image: gcr.io/projectsigstore/cosign:v2.6.0@sha256:71b96f6405dd3c3f081ebf73099f18edac7a716699398660890a86c42bf5edfd environment: - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/capture-image-evidence.sh provenance controller t4-cluster-operator @@ -366,6 +370,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -381,10 +386,10 @@ steps: build-cluster-server: depends_on: [build-controller] - image: mirror.gcr.io/moby/buildkit:v0.30.0-rootless + image: mirror.gcr.io/moby/buildkit:v0.30.0-rootless@sha256:d76eb1caecac5733ef7553c1e90a1b21f1bb218cd1142d3553de0747b4a14ba9 environment: BUILDKIT_ADDR: tcp://woodpecker-buildkitd.linkedin-ci.svc.cluster.local:1234 - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/build-image.sh cluster-server t4-cluster-server cluster/images/cluster-server/Dockerfile @@ -392,6 +397,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -407,9 +413,9 @@ steps: sbom-cluster-server: depends_on: [provenance-controller] - image: anchore/syft:1.33.0 + image: anchore/syft:v1.33.0@sha256:f94e5d9fce1f2278491a8e3a63bd5f6ddb81fdfdbb8bf7a1637565c1d5344357 environment: - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/capture-image-evidence.sh sbom cluster-server t4-cluster-server @@ -417,6 +423,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -432,9 +439,9 @@ steps: vulnerability-cluster-server: depends_on: [sbom-cluster-server] - image: aquasec/trivy:0.66.0 + image: aquasec/trivy:0.66.0@sha256:086971aaf400beebd94e8300fd8ea623774419597169156cec56eec5b00dfb1e environment: - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/capture-image-evidence.sh vulnerability cluster-server t4-cluster-server @@ -442,6 +449,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -457,9 +465,9 @@ steps: provenance-cluster-server: depends_on: [vulnerability-cluster-server] - image: gcr.io/projectsigstore/cosign:v2.6.0 + image: gcr.io/projectsigstore/cosign:v2.6.0@sha256:71b96f6405dd3c3f081ebf73099f18edac7a716699398660890a86c42bf5edfd environment: - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/capture-image-evidence.sh provenance cluster-server t4-cluster-server @@ -467,6 +475,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -482,10 +491,10 @@ steps: build-session-runtime: depends_on: [build-cluster-server] - image: mirror.gcr.io/moby/buildkit:v0.30.0-rootless + image: mirror.gcr.io/moby/buildkit:v0.30.0-rootless@sha256:d76eb1caecac5733ef7553c1e90a1b21f1bb218cd1142d3553de0747b4a14ba9 environment: BUILDKIT_ADDR: tcp://woodpecker-buildkitd.linkedin-ci.svc.cluster.local:1234 - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/build-image.sh session-runtime t4-session-runtime cluster/images/session-runtime/Dockerfile @@ -493,6 +502,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -508,9 +518,9 @@ steps: sbom-session-runtime: depends_on: [provenance-cluster-server] - image: anchore/syft:1.33.0 + image: anchore/syft:v1.33.0@sha256:f94e5d9fce1f2278491a8e3a63bd5f6ddb81fdfdbb8bf7a1637565c1d5344357 environment: - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/capture-image-evidence.sh sbom session-runtime t4-session-runtime @@ -518,6 +528,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -533,9 +544,9 @@ steps: vulnerability-session-runtime: depends_on: [sbom-session-runtime] - image: aquasec/trivy:0.66.0 + image: aquasec/trivy:0.66.0@sha256:086971aaf400beebd94e8300fd8ea623774419597169156cec56eec5b00dfb1e environment: - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/capture-image-evidence.sh vulnerability session-runtime t4-session-runtime @@ -543,6 +554,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -558,9 +570,9 @@ steps: provenance-session-runtime: depends_on: [vulnerability-session-runtime] - image: gcr.io/projectsigstore/cosign:v2.6.0 + image: gcr.io/projectsigstore/cosign:v2.6.0@sha256:71b96f6405dd3c3f081ebf73099f18edac7a716699398660890a86c42bf5edfd environment: - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/capture-image-evidence.sh provenance session-runtime t4-session-runtime @@ -568,6 +580,33 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 128Mi + ephemeral-storage: 128Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 1Gi + + promote-images: + depends_on: [image-publication-manifest] + image: ghcr.io/oras-project/oras:v1.3.0@sha256:6ce045ce069a89934d6666b8b49f9c4c0145201bd6de6dbe2aee267814c55468 + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + commands: + - scripts/cluster-ci/promote-images.sh + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -583,7 +622,10 @@ steps: image-publication-manifest: depends_on: [provenance-session-runtime] - image: mirror.gcr.io/library/node:24.13.1-bookworm + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot commands: - node scripts/cluster-ci/assemble-image-manifest.mjs - node scripts/cluster-ci/validate-proof.mjs artifacts/cluster-proof/image-publication.json --images-only @@ -591,6 +633,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -605,10 +648,10 @@ steps: ephemeral-storage: 1Gi publish-image-evidence: - depends_on: [image-publication-manifest] - image: ghcr.io/oras-project/oras:v1.3.0 + depends_on: [promote-images] + image: ghcr.io/oras-project/oras:v1.3.0@sha256:6ce045ce069a89934d6666b8b49f9c4c0145201bd6de6dbe2aee267814c55468 environment: - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/publish-artifact.sh images @@ -616,6 +659,7 @@ steps: - event: push branch: main - event: manual + branch: [main, agent/t4-cluster-operator] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -631,13 +675,16 @@ steps: cleanup-image-registry-auth: depends_on: [publish-image-evidence] - image: mirror.gcr.io/library/busybox:1.37.0-musl + image: mirror.gcr.io/library/busybox:1.37.0-musl@sha256:222ad6d973c0d198014546a65cd02c5fdedcc172123c5b4c2bf0af636550bd94 commands: - rm -rf .cluster-ci/registry-auth when: - event: push branch: main + status: [success, failure] - event: manual + branch: [main, agent/t4-cluster-operator] + status: [success, failure] backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted @@ -660,6 +707,7 @@ steps: - node scripts/cluster-ci/collect-readonly-cluster.mjs when: - event: manual + branch: [main, agent/t4-cluster-operator] evaluate: 'T4_CLUSTER_PROOF_ENABLED == "true"' backend_options: kubernetes: @@ -674,18 +722,14 @@ steps: memory: 1Gi ephemeral-storage: 1Gi - live-ui-proof: + live-frame-proof: depends_on: [live-cluster-observations] - image: mcr.microsoft.com/playwright:v1.61.1-noble - environment: - T4_CLUSTER_NAMESPACE: t4-development - + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 commands: - - corepack enable - - pnpm test:cluster:e2e - node scripts/cluster-ci/capture-redacted-frames.mjs when: - event: manual + branch: [main, agent/t4-cluster-operator] evaluate: 'T4_CLUSTER_PROOF_ENABLED == "true"' backend_options: kubernetes: @@ -701,13 +745,14 @@ steps: ephemeral-storage: 6Gi live-proof-assembly: - depends_on: [live-ui-proof] - image: mirror.gcr.io/library/node:24.13.1-bookworm + depends_on: [live-frame-proof] + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 commands: - node scripts/cluster-ci/assemble-proof.mjs - node scripts/cluster-ci/validate-proof.mjs artifacts/cluster-proof/manifest.json when: - event: manual + branch: [main, agent/t4-cluster-operator] evaluate: 'T4_CLUSTER_PROOF_ENABLED == "true"' backend_options: kubernetes: @@ -729,6 +774,7 @@ steps: - sh scripts/cluster-ci/load-registry-auth.sh when: - event: manual + branch: [main, agent/t4-cluster-operator] evaluate: 'T4_CLUSTER_PROOF_ENABLED == "true"' backend_options: kubernetes: @@ -745,14 +791,15 @@ steps: publish-live-proof: depends_on: [harbor-auth-live-proof] - image: ghcr.io/oras-project/oras:v1.3.0 + image: ghcr.io/oras-project/oras:v1.3.0@sha256:6ce045ce069a89934d6666b8b49f9c4c0145201bd6de6dbe2aee267814c55468 environment: - HARBOR_REGISTRY: harbor.linkedin-registry.svc.cluster.local + HARBOR_REGISTRY: harbor.tailb18de3.ts.net HARBOR_PROJECT: linkedin-bot commands: - scripts/cluster-ci/publish-artifact.sh proof when: - event: manual + branch: [main, agent/t4-cluster-operator] evaluate: 'T4_CLUSTER_PROOF_ENABLED == "true"' backend_options: kubernetes: @@ -769,11 +816,13 @@ steps: cleanup-live-registry-auth: depends_on: [publish-live-proof] - image: mirror.gcr.io/library/busybox:1.37.0-musl + image: mirror.gcr.io/library/busybox:1.37.0-musl@sha256:222ad6d973c0d198014546a65cd02c5fdedcc172123c5b4c2bf0af636550bd94 commands: - rm -rf .cluster-ci/registry-auth when: - event: manual + branch: [main, agent/t4-cluster-operator] + status: [success, failure] evaluate: 'T4_CLUSTER_PROOF_ENABLED == "true"' backend_options: kubernetes: @@ -791,7 +840,7 @@ steps: verify: depends_on: - cluster-chart-tests - image: mirror.gcr.io/library/alpine:3.22.1 + image: mirror.gcr.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 commands: - test -f package.json when: diff --git a/scripts/cluster-ci/assemble-image-manifest.mjs b/scripts/cluster-ci/assemble-image-manifest.mjs index 4adcda87..5bf247d1 100644 --- a/scripts/cluster-ci/assemble-image-manifest.mjs +++ b/scripts/cluster-ci/assemble-image-manifest.mjs @@ -11,6 +11,10 @@ import { const repoRoot = resolve(import.meta.dirname, "../.."); const artifactDirectory = resolve(repoRoot, "artifacts/cluster-proof/images"); const outputPath = resolve(repoRoot, "artifacts/cluster-proof/image-publication.json"); +const CANONICAL_SOURCE_REPOSITORY = "z-peterson/t4-code"; +const CANONICAL_SOURCE_URL = "https://github.com/z-peterson/t4-code"; +const HARBOR_REGISTRY = "harbor.tailb18de3.ts.net"; +const QUARANTINE_PREFIX = "quarantine"; const suffixes = { controller: "t4-cluster-operator", "cluster-server": "t4-cluster-server", @@ -25,9 +29,19 @@ function requiredEnvironment(name, environment = process.env) { function woodpeckerIdentity(environment = process.env) { const url = requiredEnvironment("CI_PIPELINE_URL", environment); - const match = new URL(url).pathname.match(/\/repos\/([1-9][0-9]*)\/pipeline\/([1-9][0-9]*)\/?$/u); + const parsedUrl = new URL(url); + const match = parsedUrl.pathname.match(/\/repos\/([1-9][0-9]*)\/pipeline\/([1-9][0-9]*)\/?$/u); const pipelineNumber = Number(requiredEnvironment("CI_PIPELINE_NUMBER", environment)); - if (!match || !Number.isSafeInteger(pipelineNumber) || pipelineNumber <= 0) { + if ( + parsedUrl.origin !== "https://woodpecker-ci-dev.tailb18de3.ts.net" || + parsedUrl.username || + parsedUrl.password || + parsedUrl.search || + parsedUrl.hash || + !match || + !Number.isSafeInteger(pipelineNumber) || + pipelineNumber <= 0 + ) { throw new Error("Woodpecker pipeline URL/number identity is invalid"); } return { @@ -48,14 +62,85 @@ async function json(path, label) { return value; } -function vulnerabilityCounts(report) { - if (!report || typeof report !== "object" || !Array.isArray(report.Results)) { - throw new Error("Trivy report is malformed"); +function exactImagePurl(locator, repository, digest) { + if (typeof locator !== "string" || locator.length > 2048) return false; + let decoded; + try { + decoded = decodeURIComponent(locator); + } catch { + return false; + } + const queryIndex = decoded.indexOf("?"); + const identity = queryIndex === -1 ? decoded : decoded.slice(0, queryIndex); + if (!identity.startsWith("pkg:oci/") || !identity.endsWith(`@${digest}`)) return false; + const parameters = new URLSearchParams(queryIndex === -1 ? "" : decoded.slice(queryIndex + 1)); + return parameters.get("repository_url") === repository; +} + +export function verifySpdx(sbom, { repository, digest, reference }) { + if ( + !sbom || + typeof sbom !== "object" || + sbom.spdxVersion !== "SPDX-2.3" || + sbom.dataLicense !== "CC0-1.0" || + sbom.SPDXID !== "SPDXRef-DOCUMENT" || + sbom.name !== reference || + typeof sbom.documentNamespace !== "string" || + !sbom.documentNamespace.includes(digest.slice("sha256:".length)) || + !Array.isArray(sbom.documentDescribes) || + sbom.documentDescribes.length !== 1 || + !Array.isArray(sbom.packages) || + sbom.packages.length < 1 || + sbom.packages.length > 100_000 + ) { + throw new Error("SPDX document identity is not bound to the scanned image"); + } + const imagePackage = sbom.packages.find(({ SPDXID }) => SPDXID === sbom.documentDescribes[0]); + if ( + !imagePackage || + typeof imagePackage.name !== "string" || + imagePackage.name !== repository.slice(repository.lastIndexOf("/") + 1) || + !Array.isArray(imagePackage.externalRefs) || + !imagePackage.externalRefs.some( + (externalRef) => + externalRef?.referenceCategory === "PACKAGE-MANAGER" && + externalRef?.referenceType === "purl" && + exactImagePurl(externalRef.referenceLocator, repository, digest), + ) + ) { + throw new Error("SPDX described package/external reference does not bind the image repository and digest"); + } +} + +export function vulnerabilityCounts(report, { repository, digest, reference }) { + if ( + !report || + typeof report !== "object" || + report.ArtifactName !== reference || + report.ArtifactType !== "container_image" || + !report.Metadata || + !Array.isArray(report.Metadata.RepoDigests) || + !report.Metadata.RepoDigests.includes(`${repository}@${digest}`) || + !(report.Metadata.ImageID === digest || report.Metadata.ImageID?.endsWith(`@${digest}`)) || + !Array.isArray(report.Results) || + report.Results.length < 1 || + report.Results.length > 4096 + ) { + throw new Error("Trivy report artifact/results identity is malformed or unbound"); } const counts = { critical: 0, high: 0 }; for (const result of report.Results) { - if (!Array.isArray(result.Vulnerabilities)) continue; - for (const vulnerability of result.Vulnerabilities) { + if ( + typeof result?.Target !== "string" || + result.Target.length < 1 || + result.Target.length > 2048 || + typeof result.Class !== "string" || + typeof result.Type !== "string" || + (result.Vulnerabilities !== undefined && !Array.isArray(result.Vulnerabilities)) + ) { + throw new Error("Trivy result entry is malformed"); + } + for (const vulnerability of result.Vulnerabilities ?? []) { if (vulnerability?.Severity === "CRITICAL") counts.critical += 1; else if (vulnerability?.Severity === "HIGH") counts.high += 1; } @@ -66,39 +151,136 @@ function vulnerabilityCounts(report) { return counts; } -function verifyProvenance(jsonLines, digest) { - const expected = digest.slice("sha256:".length); +function boundedStrings(value, depth = 0, output = []) { + if (depth > 12 || output.length > 4096) throw new Error("provenance exceeded its structural bound"); + if (typeof value === "string") { + if (value.length > 4096) throw new Error("provenance string exceeded its bound"); + output.push(value); + } else if (Array.isArray(value)) { + value.forEach((item) => boundedStrings(item, depth + 1, output)); + } else if (value && typeof value === "object") { + Object.values(value).forEach((item) => boundedStrings(item, depth + 1, output)); + } + return output; +} + +function trustedSourceMaterial(material, commit) { + if (!material || typeof material !== "object" || typeof material.uri !== "string") return false; + let source; + try { + source = new URL(material.uri.replace(/^git\+/u, "")); + } catch { + return false; + } + return ( + source.protocol === "https:" && + source.hostname === "github.com" && + source.pathname === "/z-peterson/t4-code.git" && + source.hash === `#${commit}` && + material.digest?.sha1 === commit + ); +} + +export function verifyProvenance(jsonLines, { repository, digest, commit }) { + const expectedDigest = digest.slice("sha256:".length); const lines = jsonLines.split("\n").filter(Boolean); if (lines.length < 1 || lines.length > 32) throw new Error("provenance attestation count is invalid"); const statements = []; for (const line of lines) { - const envelope = JSON.parse(line); + let envelope; + let statement; + try { + envelope = JSON.parse(line); + statement = JSON.parse(Buffer.from(envelope.payload, "base64").toString("utf8")); + } catch (error) { + throw new Error("provenance attestation is not valid DSSE JSON", { cause: error }); + } if (envelope?.payloadType !== "application/vnd.in-toto+json" || typeof envelope.payload !== "string") { throw new Error("provenance attestation is not an in-toto envelope"); } - statements.push(JSON.parse(Buffer.from(envelope.payload, "base64").toString("utf8"))); + statements.push(statement); } - const provenance = statements.find( - (statement) => - typeof statement?.predicateType === "string" && - statement.predicateType.includes("slsa.dev/provenance") && - statement.subject?.some(({ digest: subjectDigest }) => subjectDigest?.sha256 === expected), - ); - if (!provenance) throw new Error("BuildKit provenance does not bind the published image digest"); + const provenance = statements.find((statement) => { + const predicate = statement?.predicate; + const sourceMaterial = predicate?.materials?.find((material) => trustedSourceMaterial(material, commit)); + const baseMaterial = predicate?.materials?.some( + (material) => + typeof material?.uri === "string" && + material.uri.startsWith("pkg:docker/") && + /^[0-9a-f]{64}$/u.test(material.digest?.sha256 ?? ""), + ); + const invocationStrings = boundedStrings(predicate?.invocation?.parameters ?? {}); + return ( + statement?._type === "https://in-toto.io/Statement/v0.1" && + typeof statement.predicateType === "string" && + predicate?.builder?.id === "https://mobyproject.org/buildkit@v1" && + statement.predicateType.startsWith("https://slsa.dev/provenance/") && + predicate?.buildType === "https://mobyproject.org/buildkit@v1" && + statement.subject?.some( + (subject) => subject?.name === repository && subject?.digest?.sha256 === expectedDigest, + ) && + sourceMaterial && + baseMaterial && + invocationStrings.includes(commit) && + invocationStrings.includes(CANONICAL_SOURCE_URL) + ); + }); + if (!provenance) { + throw new Error("BuildKit provenance does not bind subject, trusted source repository, CI commit, and materials"); + } +} + +export function provenanceVerificationMode(environment = process.env) { + const identity = environment.T4_COSIGN_CERTIFICATE_IDENTITY?.trim() ?? ""; + const issuer = environment.T4_COSIGN_CERTIFICATE_OIDC_ISSUER?.trim() ?? ""; + if (Boolean(identity) !== Boolean(issuer)) { + throw new Error("cosign certificate identity and OIDC issuer must be configured together"); + } + return identity ? { mode: "cosign-keyless", signatureVerified: true } : { mode: "buildkit-content", signatureVerified: false }; +} + +function validateProvenanceVerification(value, expected) { + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + Object.keys(value).sort().join(",") !== "mode,signatureVerified" || + value.mode !== expected.mode || + value.signatureVerified !== expected.signatureVerified + ) { + throw new Error("provenance signer verification record is missing or not truthful"); + } + return value; } -async function imageEntry(component, commit, registry, project) { +async function imageEntry(component, commit, registry, project, expectedVerification) { const digest = (await readFile(resolve(artifactDirectory, `${component}.digest`), "utf8")).trim(); const repository = `${registry}/${project}/${suffixes[component]}`; + const evidenceRepository = `${registry}/${project}/${QUARANTINE_PREFIX}/${suffixes[component]}`; + const evidenceReference = `${evidenceRepository}@${digest}`; const sbomPath = resolve(artifactDirectory, `${component}.spdx.json`); const provenancePath = resolve(artifactDirectory, `${component}.provenance.jsonl`); + const provenanceVerificationPath = resolve(artifactDirectory, `${component}.provenance-verification.json`); const vulnerabilityPath = resolve(artifactDirectory, `${component}.trivy.json`); - const sbom = await json(sbomPath, `${component} SBOM`); - if (typeof sbom.spdxVersion !== "string" || !sbom.spdxVersion.startsWith("SPDX-")) { - throw new Error(`${component} SBOM is not SPDX JSON`); - } - verifyProvenance(await readFile(provenancePath, "utf8"), digest); - const counts = vulnerabilityCounts(await json(vulnerabilityPath, `${component} vulnerability report`)); + verifySpdx(await json(sbomPath, `${component} SBOM`), { + repository: evidenceRepository, + digest, + reference: evidenceReference, + }); + verifyProvenance(await readFile(provenancePath, "utf8"), { + repository: evidenceRepository, + digest, + commit, + }); + const verification = validateProvenanceVerification( + await json(provenanceVerificationPath, `${component} provenance verification`), + expectedVerification, + ); + const counts = vulnerabilityCounts(await json(vulnerabilityPath, `${component} vulnerability report`), { + repository: evidenceRepository, + digest, + reference: evidenceReference, + }); return { component, repository, @@ -106,7 +288,7 @@ async function imageEntry(component, commit, registry, project) { digest, reference: `${repository}@${digest}`, sbom: await createFileEvidence(sbomPath, { artifactRoot: repoRoot }), - provenance: await createFileEvidence(provenancePath, { artifactRoot: repoRoot }), + provenance: { ...(await createFileEvidence(provenancePath, { artifactRoot: repoRoot })), ...verification }, vulnerability: { ...(await createFileEvidence(vulnerabilityPath, { artifactRoot: repoRoot })), scanner: "trivy", @@ -117,17 +299,21 @@ async function imageEntry(component, commit, registry, project) { export async function assembleImagePublicationManifest(environment = process.env) { const commit = requiredEnvironment("CI_COMMIT_SHA", environment); + const repository = requiredEnvironment("CI_REPO", environment); const registry = requiredEnvironment("HARBOR_REGISTRY", environment).replace(/\/$/u, ""); const project = requiredEnvironment("HARBOR_PROJECT", environment).replace(/^\/+|\/+$/gu, ""); + if (repository !== CANONICAL_SOURCE_REPOSITORY) throw new Error("CI_REPO is not the canonical source repository"); + if (registry !== HARBOR_REGISTRY) throw new Error("HARBOR_REGISTRY must be the exact HTTPS tailnet Harbor host"); + const provenanceVerification = provenanceVerificationMode(environment); const manifest = { schemaVersion: "t4-cluster-images/1", source: { - repository: requiredEnvironment("CI_REPO", environment), + repository, commit, woodpecker: woodpeckerIdentity(environment), }, images: await Promise.all( - IMAGE_COMPONENTS.map((component) => imageEntry(component, commit, registry, project)), + IMAGE_COMPONENTS.map((component) => imageEntry(component, commit, registry, project, provenanceVerification)), ), }; return validateImagePublicationManifest(manifest); diff --git a/scripts/cluster-ci/assemble-proof.mjs b/scripts/cluster-ci/assemble-proof.mjs index 493969d7..fafa812e 100644 --- a/scripts/cluster-ci/assemble-proof.mjs +++ b/scripts/cluster-ci/assemble-proof.mjs @@ -16,6 +16,48 @@ const proofRoot = resolve(repoRoot, "artifacts/cluster-proof"); const MAX_HTTP_BYTES = 2 * 1024 * 1024; const MAX_LOCAL_ARTIFACTS = 32; const SCENARIO_ASSERTION = /^[a-z0-9][a-z0-9._-]{0,127}$/u; +const CONTRACT_SCENARIOS = new Set([ + "wire-reconnect-idempotency", + "gui-auth-isolation", + "desktop-viewport", + "mobile-viewport", +]); +const WOODPECKER_ORIGIN = "https://woodpecker-ci-dev.tailb18de3.ts.net"; +const REMOTE_OBSERVATIONS = Object.freeze({ + prometheus: { + origin: "https://interview-responder-prometheus.tailb18de3.ts.net", + environment: "T4_PROMETHEUS_EVIDENCE_URL", + }, + loki: { + origin: "https://interview-responder-loki.tailb18de3.ts.net", + environment: "T4_LOKI_EVIDENCE_URL", + }, + grafana: { + origin: "https://grafana.tailb18de3.ts.net", + environment: "T4_GRAFANA_EVIDENCE_URL", + }, +}); + +export function trustedProofUrl(value, label, allowedHosts) { + let url; + try { + url = new URL(value); + } catch { + throw new Error(`${label} must be a valid URL`); + } + if ( + url.protocol !== "https:" || + !allowedHosts.includes(url.hostname) || + url.port || + url.username || + url.password || + url.search || + url.hash + ) { + throw new Error(`${label} must be credential-free HTTPS on its exact tailnet/internal allowlist`); + } + return url.href; +} function requiredEnvironment(name) { const value = process.env[name]?.trim(); @@ -24,22 +66,33 @@ function requiredEnvironment(name) { } function woodpeckerIdentity() { - const url = requiredEnvironment("CI_PIPELINE_URL"); - const match = new URL(url).pathname.match(/\/repos\/([1-9][0-9]*)\/pipeline\/([1-9][0-9]*)\/?$/u); + const value = requiredEnvironment("CI_PIPELINE_URL"); + const url = new URL(value); + const match = url.pathname.match(/\/repos\/([1-9][0-9]*)\/pipeline\/([1-9][0-9]*)\/?$/u); const pipelineNumber = Number(requiredEnvironment("CI_PIPELINE_NUMBER")); - if (!match || !Number.isSafeInteger(pipelineNumber) || pipelineNumber <= 0) { + if ( + url.origin !== WOODPECKER_ORIGIN || + url.username || + url.password || + url.search || + url.hash || + !match || + !Number.isSafeInteger(pipelineNumber) || + pipelineNumber <= 0 + ) { throw new Error("Woodpecker pipeline URL/number identity is invalid"); } return { repositoryId: Number(match[1]), pipelineId: Number(match[2]), pipelineNumber, - url, + url: value, }; } -async function boundedFetchJson(url, label) { - const response = await fetch(url, { +async function boundedFetchJson(url, label, allowedHosts) { + const trustedUrl = trustedProofUrl(url, label, allowedHosts); + const response = await fetch(trustedUrl, { headers: { Accept: "application/json", "User-Agent": "t4-cluster-proof/1" }, redirect: "error", signal: AbortSignal.timeout(20_000), @@ -100,6 +153,7 @@ async function scenarioEntries() { entries.push({ id, status: "passed", + evidenceType: CONTRACT_SCENARIOS.has(id) ? "contract" : "live", observedAt: utcTimestamp(record.observedAt, `scenario ${id}.observedAt`), assertions: record.assertions, evidence: [await createFileEvidence(path, { artifactRoot: repoRoot })], @@ -124,7 +178,7 @@ function safeSummary(value, label, depth = 0) { const entries = Object.entries(value); if (entries.length > 128) throw new Error(`${label} exceeded its field bound`); for (const [key, item] of entries) { - if (/authorization|cookie|credential|password|prompt|secret|token|transcript/iu.test(key)) { + if (/api.?key|private.?key|authorization|auth|bearer|cookie|credential|password|prompt|secret|token|transcript/iu.test(key)) { throw new Error(`${label} contained sensitive field ${key}`); } safeSummary(item, `${label}.${key}`, depth + 1); @@ -132,75 +186,75 @@ function safeSummary(value, label, depth = 0) { } async function observationEntries() { - const metadata = await boundedFetchJson( - requiredEnvironment("T4_OBSERVABILITY_MANIFEST_URL"), - "observability manifest", + const directory = resolve(proofRoot, "observations"); + await mkdir(directory, { recursive: true }); + const commit = requiredEnvironment("CI_COMMIT_SHA"); + const entries = []; + + const woodpecker = woodpeckerIdentity(); + const woodpeckerObservedAt = new Date().toISOString(); + const woodpeckerPath = resolve(directory, "woodpecker.json"); + await writeFile( + woodpeckerPath, + `${JSON.stringify({ schemaVersion: "t4-woodpecker-observation/1", observedAt: woodpeckerObservedAt, ...woodpecker }, null, 2)}\n`, + { mode: 0o600 }, ); - exactKeys(metadata, ["schemaVersion", "sourceCommit", "observations"], "observability manifest"); + entries.push({ + system: "woodpecker", + observedAt: woodpeckerObservedAt, + url: WOODPECKER_ORIGIN, + ids: [String(woodpecker.repositoryId), String(woodpecker.pipelineId), String(woodpecker.pipelineNumber)], + evidence: await createFileEvidence(woodpeckerPath, { artifactRoot: repoRoot }), + }); + + const kubernetesPath = resolve(directory, "kubernetes.json"); + const kubernetes = JSON.parse(await readFile(kubernetesPath, "utf8")); if ( - metadata.schemaVersion !== "t4-cluster-observations/1" || - metadata.sourceCommit !== requiredEnvironment("CI_COMMIT_SHA") || - !Array.isArray(metadata.observations) || - metadata.observations.length !== OBSERVATION_SYSTEMS.length + kubernetes?.schemaVersion !== "t4-cluster-readonly-snapshot/1" || + !Array.isArray(kubernetes.deployments) || + kubernetes.deployments.length < 2 ) { - throw new Error("observability manifest identity or coverage is invalid"); - } - const bySystem = new Map(); - for (const observation of metadata.observations) { - exactKeys(observation, ["system", "observedAt", "url", "ids", "evidenceUrl"], "observation metadata"); - if (!OBSERVATION_SYSTEMS.includes(observation.system) || bySystem.has(observation.system)) { - throw new Error("observability manifest contains an unknown or duplicate system"); - } - bySystem.set(observation.system, observation); + throw new Error("Kubernetes observation does not contain exact read-only cluster evidence"); } + entries.push({ + system: "kubernetes", + observedAt: utcTimestamp(kubernetes.observedAt, "kubernetes observedAt"), + url: null, + ids: kubernetes.deployments.map(({ name }) => name), + evidence: await createFileEvidence(kubernetesPath, { artifactRoot: repoRoot }), + }); - const directory = resolve(proofRoot, "observations"); - await mkdir(directory, { recursive: true }); - const entries = []; - for (const system of OBSERVATION_SYSTEMS) { - const metadataEntry = bySystem.get(system); - if (!metadataEntry) throw new Error(`observability manifest is missing ${system}`); - let path; - if (system === "kubernetes") { - path = resolve(directory, "kubernetes.json"); - } else if (system === "woodpecker") { - path = resolve(directory, "woodpecker.json"); - await writeFile( - path, - `${JSON.stringify( - { - schemaVersion: "t4-woodpecker-observation/1", - observedAt: new Date().toISOString(), - ...woodpeckerIdentity(), - }, - null, - 2, - )}\n`, - { mode: 0o600 }, - ); - } else { - const payload = await boundedFetchJson(metadataEntry.evidenceUrl, `${system} evidence`); - exactKeys(payload, ["schemaVersion", "system", "observedAt", "redacted", "summary"], `${system} evidence`); - if ( - payload.schemaVersion !== "t4-cluster-observation/1" || - payload.system !== system || - payload.redacted !== true - ) { - throw new Error(`${system} evidence is not an exact redacted observation`); - } - utcTimestamp(payload.observedAt, `${system} evidence observedAt`); - safeSummary(payload.summary, `${system} evidence summary`); - path = resolve(directory, `${system}.json`); - await writeFile(path, `${JSON.stringify(payload, null, 2)}\n`, { mode: 0o600 }); + for (const system of ["prometheus", "loki", "grafana"]) { + const configuration = REMOTE_OBSERVATIONS[system]; + const evidenceUrl = requiredEnvironment(configuration.environment); + const payload = await boundedFetchJson(evidenceUrl, `${system} evidence`, [new URL(configuration.origin).hostname]); + exactKeys(payload, ["schemaVersion", "sourceCommit", "system", "observedAt", "redacted", "ids", "summary"], `${system} evidence`); + if ( + payload.schemaVersion !== "t4-cluster-observation/1" || + payload.sourceCommit !== commit || + payload.system !== system || + payload.redacted !== true || + !Array.isArray(payload.ids) || + payload.ids.length < 1 || + payload.ids.length > 16 + ) { + throw new Error(`${system} evidence is not an exact source-bound redacted observation`); } + utcTimestamp(payload.observedAt, `${system} evidence observedAt`); + safeSummary(payload.summary, `${system} evidence summary`); + const path = resolve(directory, `${system}.json`); + await writeFile(path, `${JSON.stringify(payload, null, 2)}\n`, { mode: 0o600 }); entries.push({ system, - observedAt: utcTimestamp(metadataEntry.observedAt, `${system}.observedAt`), - url: metadataEntry.url, - ids: metadataEntry.ids, + observedAt: payload.observedAt, + url: configuration.origin, + ids: payload.ids, evidence: await createFileEvidence(path, { artifactRoot: repoRoot }), }); } + if (entries.length !== OBSERVATION_SYSTEMS.length) { + throw new Error("observation coverage does not match the truthfully available systems"); + } return entries; } @@ -227,6 +281,7 @@ async function localArtifacts(kind, extensions) { results.push({ ...(await createFileEvidence(path, { artifactRoot: repoRoot })), redacted: true, + evidenceType: kind === "frames" ? "live" : "contract", ...(viewport ? { viewport } : {}), }); } @@ -244,17 +299,25 @@ async function verifyLiveImageDigests(images) { ) { throw new Error("Kubernetes observation has no bounded live image identity"); } + const identities = { + controller: { component: "controller", container: "controller" }, + "cluster-server": { component: "server", container: "server" }, + "session-runtime": { name: "t4-session-runtime", container: "session-runtime" }, + }; for (const image of images) { - const suffix = image.repository.slice(image.repository.lastIndexOf("/") + 1); - const match = snapshot.images.find( - ({ image: declaredImage, imageID }) => - typeof declaredImage === "string" && - (declaredImage.includes(`/${suffix}:`) || declaredImage.includes(`/${suffix}@`)) && - typeof imageID === "string" && - imageID.endsWith(`@${image.digest}`), + const identity = identities[image.component]; + const matches = snapshot.images.filter((observed) => + observed?.labels?.partOf === "t4-cluster" && + (identity.component ? observed.labels.component === identity.component : observed.labels.name === identity.name) && + observed.container === identity.container && + observed.phase === "Running" && + observed.ready === true && + observed.image === image.reference && + typeof observed.imageID === "string" && + observed.imageID.endsWith(`@${image.digest}`), ); - if (!match) { - throw new Error(`live Kubernetes pods do not run published ${image.component} digest ${image.digest}`); + if (matches.length < 1) { + throw new Error(`live Kubernetes pods do not run the exact ready ${image.component} reference ${image.reference}`); } } } @@ -281,7 +344,6 @@ export async function assembleProofManifest() { artifacts: { frames: await localArtifacts("frames", [".json"]), screenshots: await localArtifacts("screenshots", [".png", ".webp"]), - videos: await localArtifacts("videos", [".webm"]), }, }; return validateProofManifest(manifest); diff --git a/scripts/cluster-ci/build-image.sh b/scripts/cluster-ci/build-image.sh index cbd6f687..b8c7ce4b 100755 --- a/scripts/cluster-ci/build-image.sh +++ b/scripts/cluster-ci/build-image.sh @@ -30,6 +30,15 @@ esac : "${BUILDKIT_ADDR:?BUILDKIT_ADDR is required}" : "${HARBOR_REGISTRY:?HARBOR_REGISTRY is required}" : "${HARBOR_PROJECT:?HARBOR_PROJECT is required}" +if [ "$HARBOR_REGISTRY" != "harbor.tailb18de3.ts.net" ]; then + echo "HARBOR_REGISTRY must be the exact HTTPS tailnet Harbor host" >&2 + exit 64 +fi +: "${CI_REPO:?CI_REPO is required}" +if [ "$CI_REPO" != "z-peterson/t4-code" ]; then + echo "CI_REPO must be the canonical source repository" >&2 + exit 64 +fi auth_dir=${T4_REGISTRY_AUTH_DIR:-${CI_WORKSPACE:-$PWD}/.cluster-ci/registry-auth} test -r "$auth_dir/config.json" export DOCKER_CONFIG="$auth_dir" @@ -40,16 +49,19 @@ mkdir -p "$artifact_dir" metadata="$artifact_dir/$component.buildkit.json" digest_file="$artifact_dir/$component.digest" -repository="$HARBOR_REGISTRY/$HARBOR_PROJECT/$repository_suffix" +repository="$HARBOR_REGISTRY/$HARBOR_PROJECT/quarantine/$repository_suffix" reference="$repository:$CI_COMMIT_SHA" +source_context="https://github.com/z-peterson/t4-code.git#$CI_COMMIT_SHA" buildctl --addr "$BUILDKIT_ADDR" build \ --frontend dockerfile.v0 \ - --local context=. \ - --local dockerfile=. \ + --opt "context=$source_context" \ --opt "filename=$dockerfile" \ - --opt platform=linux/amd64 \ + --opt platform=linux/amd64,linux/arm64 \ --opt "build-arg:SOURCE_COMMIT=$CI_COMMIT_SHA" \ + --opt "build-arg:SOURCE_REPOSITORY=https://github.com/z-peterson/t4-code" \ + --opt "label:org.opencontainers.image.source=https://github.com/z-peterson/t4-code" \ + --opt "label:org.opencontainers.image.revision=$CI_COMMIT_SHA" \ --output "type=image,name=$reference,push=true,compression=zstd,force-compression=true,oci-mediatypes=true" \ --attest type=sbom \ --attest type=provenance,mode=max \ diff --git a/scripts/cluster-ci/capture-image-evidence.sh b/scripts/cluster-ci/capture-image-evidence.sh index 947220f6..1f6c2031 100755 --- a/scripts/cluster-ci/capture-image-evidence.sh +++ b/scripts/cluster-ci/capture-image-evidence.sh @@ -24,6 +24,10 @@ esac : "${CI_COMMIT_SHA:?CI_COMMIT_SHA is required}" : "${HARBOR_REGISTRY:?HARBOR_REGISTRY is required}" : "${HARBOR_PROJECT:?HARBOR_PROJECT is required}" +if [ "$HARBOR_REGISTRY" != "harbor.tailb18de3.ts.net" ]; then + echo "HARBOR_REGISTRY must be the exact HTTPS tailnet Harbor host" >&2 + exit 64 +fi auth_dir=${T4_REGISTRY_AUTH_DIR:-${CI_WORKSPACE:-$PWD}/.cluster-ci/registry-auth} test -r "$auth_dir/config.json" export DOCKER_CONFIG="$auth_dir" @@ -37,17 +41,14 @@ case "$digest" in exit 65 ;; esac -reference="$HARBOR_REGISTRY/$HARBOR_PROJECT/$repository_suffix@$digest" +reference="$HARBOR_REGISTRY/$HARBOR_PROJECT/quarantine/$repository_suffix@$digest" case "$mode" in sbom) - export SYFT_REGISTRY_INSECURE_SKIP_TLS_VERIFY=true - export SYFT_REGISTRY_INSECURE_USE_HTTP=true syft "registry:$reference" -o "spdx-json=$artifact_dir/$component.spdx.json" test -s "$artifact_dir/$component.spdx.json" ;; vulnerability) - export TRIVY_INSECURE=true trivy image \ --format json \ --output "$artifact_dir/$component.trivy.json" \ @@ -58,9 +59,24 @@ case "$mode" in test -s "$artifact_dir/$component.trivy.json" ;; provenance) - export COSIGN_DOCKER_MEDIA_TYPES=1 - export COSIGN_ALLOW_INSECURE_REGISTRY=1 - cosign download attestation "$reference" > "$artifact_dir/$component.provenance.jsonl" + identity=${T4_COSIGN_CERTIFICATE_IDENTITY:-} + issuer=${T4_COSIGN_CERTIFICATE_OIDC_ISSUER:-} + if [ -n "$identity" ] && [ -n "$issuer" ]; then + cosign verify-attestation \ + --type slsaprovenance \ + --certificate-identity "$identity" \ + --certificate-oidc-issuer "$issuer" \ + "$reference" > "$artifact_dir/$component.provenance.jsonl" + printf '%s\n' '{"mode":"cosign-keyless","signatureVerified":true}' > "$artifact_dir/$component.provenance-verification.json" + elif [ -z "$identity" ] && [ -z "$issuer" ]; then + cosign download attestation "$reference" > "$artifact_dir/$component.provenance.jsonl" + printf '%s\n' '{"mode":"buildkit-content","signatureVerified":false}' > "$artifact_dir/$component.provenance-verification.json" + else + echo "cosign certificate identity and OIDC issuer must be configured together" >&2 + exit 64 + fi + unset identity issuer test -s "$artifact_dir/$component.provenance.jsonl" + test -s "$artifact_dir/$component.provenance-verification.json" ;; esac diff --git a/scripts/cluster-ci/capture-redacted-frames.mjs b/scripts/cluster-ci/capture-redacted-frames.mjs index 8f3a35e5..c96264f7 100644 --- a/scripts/cluster-ci/capture-redacted-frames.mjs +++ b/scripts/cluster-ci/capture-redacted-frames.mjs @@ -11,6 +11,16 @@ const outputPath = resolve(repoRoot, "artifacts/cluster-proof/frames/live-operat const MAX_INBOUND_BYTES = 1024 * 1024; const MAX_FRAMES = 32; const SAFE_AGENT_ID_KEYS = ["agentId", "rootAgentId", "activeAgentId", "parentAgentId"]; +const CLUSTER_HOST = "t4-dev.tailb18de3.ts.net"; +const EXPECTED_OMP_VERSION = "17.0.5"; +const EXPECTED_OMP_BUILD = "8476f4451ed95c5d5401785d279a93d3c659fac4"; +const REQUIRED_CAPABILITIES = Object.freeze([ + "sessions.read", + "ci.trigger", + "preview.read", + "preview.control", + "preview.input", +]); function requiredEnvironment(name) { const value = process.env[name]?.trim(); @@ -18,13 +28,22 @@ function requiredEnvironment(name) { return value; } -function clusterWebSocketUrl(baseUrl) { +export function clusterWebSocketUrl(baseUrl) { const url = new URL(baseUrl); - if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash) { - throw new Error("T4_CLUSTER_BASE_URL must be a credential-free HTTPS origin"); + if ( + url.protocol !== "https:" || + url.hostname !== CLUSTER_HOST || + url.port || + url.pathname !== "/" || + url.username || + url.password || + url.search || + url.hash + ) { + throw new Error(`T4_CLUSTER_BASE_URL must be the credential-free HTTPS origin https://${CLUSTER_HOST}/`); } url.protocol = "wss:"; - url.pathname = "/omp-app/v1"; + url.pathname = "/v1/ws"; return url; } @@ -106,8 +125,6 @@ function projectListResult(kind, hostId, value) { async function capture() { const baseUrl = requiredEnvironment("T4_CLUSTER_BASE_URL"); - const deviceId = requiredEnvironment("T4_CLUSTER_DEVICE_ID"); - const deviceToken = requiredEnvironment("T4_CLUSTER_AUTH_TOKEN"); const origin = new URL(baseUrl).origin; const pipelineNumber = requiredEnvironment("CI_PIPELINE_NUMBER"); const sessionRequest = `proof-session-list-${pipelineNumber}`; @@ -140,9 +157,9 @@ async function capture() { type: "hello", protocol: { min: "omp-app/1", max: "omp-app/1" }, client: { name: "t4-cluster-proof", version: "1", build: "1", platform: "woodpecker" }, - requestedFeatures: ["cluster.operator", "resume"], + requestedFeatures: ["cluster.operator", "resume", "preview.control"], + capabilities: REQUIRED_CAPABILITIES, savedCursors: [], - authentication: { deviceId, deviceToken }, }), ); }); @@ -161,12 +178,15 @@ async function capture() { hostId = boundedId(frame.hostId, "welcome.hostId"); if ( frame.selectedProtocol !== "omp-app/1" || + frame.ompVersion !== EXPECTED_OMP_VERSION || + frame.ompBuild !== EXPECTED_OMP_BUILD || !Array.isArray(frame.grantedFeatures) || !frame.grantedFeatures.includes("cluster.operator") || + !frame.grantedFeatures.includes("preview.control") || !Array.isArray(frame.grantedCapabilities) || - !frame.grantedCapabilities.includes("sessions.read") + REQUIRED_CAPABILITIES.some((capability) => !frame.grantedCapabilities.includes(capability)) ) { - return finish(rejectComplete, new Error("cluster operator/read capability was not negotiated")); + return finish(rejectComplete, new Error("cluster operator sessions.read/CI/preview capability contract was not negotiated")); } frames.push({ v: "omp-app/1", @@ -174,8 +194,10 @@ async function capture() { hostId, epoch: boundedId(frame.epoch, "welcome.epoch"), selectedProtocol: frame.selectedProtocol, + ompVersion: frame.ompVersion, + ompBuild: frame.ompBuild, clusterOperator: true, - sessionsRead: true, + grantedCapabilities: REQUIRED_CAPABILITIES, }); for (const [requestId, command] of [ [sessionRequest, "session.list"], diff --git a/scripts/cluster-ci/cluster-ci-contract.test.mjs b/scripts/cluster-ci/cluster-ci-contract.test.mjs index 630f3758..212d9cc8 100644 --- a/scripts/cluster-ci/cluster-ci-contract.test.mjs +++ b/scripts/cluster-ci/cluster-ci-contract.test.mjs @@ -20,12 +20,26 @@ import { validateDefaultOffRender, } from "./readonly-cluster-proof.mjs"; import { HARBOR_REGISTRY_ALIASES, normalizeRegistryAuth } from "./normalize-registry-auth.mjs"; +import { provenanceVerificationMode, verifyProvenance, verifySpdx, vulnerabilityCounts } from "./assemble-image-manifest.mjs"; +import { trustedProofUrl } from "./assemble-proof.mjs"; +import { clusterWebSocketUrl } from "./capture-redacted-frames.mjs"; const COMMIT = "0123456789abcdef0123456789abcdef01234567"; const DIGEST = `sha256:${"a".repeat(64)}`; const FILE_SHA = "b".repeat(64); const OBSERVED_AT = "2026-07-20T12:34:56.000Z"; const repoRoot = resolve(import.meta.dirname, "../.."); +const OBSERVATION_HOSTS = { + woodpecker: "woodpecker-ci-dev.tailb18de3.ts.net", + prometheus: "interview-responder-prometheus.tailb18de3.ts.net", + loki: "interview-responder-loki.tailb18de3.ts.net", + grafana: "grafana.tailb18de3.ts.net", +}; +const CONTRACT_SCENARIOS = new Set(["wire-reconnect-idempotency", "gui-auth-isolation", "desktop-viewport", "mobile-viewport"]); +const CLUSTER_VALIDATION = { + now: Date.parse(OBSERVED_AT), + ciMapping: { repositoryId: "71", ref: "refs/heads/agent/t4-cluster-operator", commit: COMMIT }, +}; function fileEvidence(path) { return { path: `artifacts/cluster-proof/${path}`, sha256: FILE_SHA }; @@ -46,7 +60,7 @@ function validProof() { repositoryId: 71, pipelineId: 401, pipelineNumber: 99, - url: "https://woodpecker.example.test/repos/71/pipeline/99", + url: "https://woodpecker-ci-dev.tailb18de3.ts.net/repos/71/pipeline/99", }, }, images: IMAGE_COMPONENTS.map((component) => { @@ -58,7 +72,7 @@ function validProof() { digest: DIGEST, reference: `${repository}@${DIGEST}`, sbom: fileEvidence(`images/${component}.spdx.json`), - provenance: fileEvidence(`images/${component}.provenance.json`), + provenance: { ...fileEvidence(`images/${component}.provenance.json`), mode: "buildkit-content", signatureVerified: false }, vulnerability: { ...fileEvidence(`images/${component}.trivy.json`), scanner: "trivy", @@ -70,6 +84,7 @@ function validProof() { scenarios: PROOF_SCENARIOS.map((id) => ({ id, status: "passed", + evidenceType: CONTRACT_SCENARIOS.has(id) ? "contract" : "live", observedAt: OBSERVED_AT, assertions: [`${id}.observable-contract`], evidence: [fileEvidence(`scenarios/${id}.json`)], @@ -77,40 +92,51 @@ function validProof() { observations: OBSERVATION_SYSTEMS.map((system, index) => ({ system, observedAt: OBSERVED_AT, - url: `https://${system}.example.test/evidence/${index + 1}`, + url: system === "kubernetes" ? null : `https://${OBSERVATION_HOSTS[system]}/`, ids: [`${system}-${index + 1}`], evidence: fileEvidence(`observations/${system}.json`), })), artifacts: { frames: [ - { ...fileEvidence("frames/omp-app.json"), redacted: true }, + { ...fileEvidence("frames/omp-app.json"), redacted: true, evidenceType: "live" }, ], screenshots: [ - { ...fileEvidence("screenshots/desktop.png"), redacted: true, viewport: "desktop" }, - { ...fileEvidence("screenshots/mobile.png"), redacted: true, viewport: "mobile" }, - ], - videos: [ - { ...fileEvidence("videos/desktop.webm"), redacted: true, viewport: "desktop" }, - { ...fileEvidence("videos/mobile.webm"), redacted: true, viewport: "mobile" }, + { ...fileEvidence("screenshots/desktop.png"), redacted: true, evidenceType: "contract", viewport: "desktop" }, + { ...fileEvidence("screenshots/mobile.png"), redacted: true, evidenceType: "contract", viewport: "mobile" }, ], }, }; } function liveClusterResponses() { + const workloadImage = `harbor.tailb18de3.ts.net/linkedin-bot/workload@${DIGEST}`; + const labels = (component) => ({ + "app.kubernetes.io/name": "t4-cluster", + "app.kubernetes.io/part-of": "t4-cluster", + "app.kubernetes.io/component": component, + }); + const workloadPod = (name, component, container) => ({ + metadata: { name, labels: labels(component) }, + spec: { nodeName: "k3s-worker-01" }, + status: { + phase: "Running", + conditions: [{ type: "Ready", status: "True" }], + containerStatuses: [{ name: container, image: workloadImage, imageID: `containerd://${workloadImage}`, ready: true }], + }, + }); return { deployments: { items: [ { - metadata: { name: "t4-cluster-controller" }, + metadata: { name: "t4-cluster-controller", generation: 4, labels: labels("controller") }, spec: { replicas: 2, - strategy: { type: "RollingUpdate", rollingUpdate: { maxUnavailable: 0 } }, + strategy: { type: "RollingUpdate", rollingUpdate: { maxUnavailable: 1 } }, }, status: { observedGeneration: 4, availableReplicas: 2 }, }, { - metadata: { name: "t4-cluster-server" }, + metadata: { name: "t4-cluster-server", generation: 6, labels: labels("server") }, spec: { replicas: 3, strategy: { type: "RollingUpdate", rollingUpdate: { maxUnavailable: 0 } }, @@ -119,16 +145,13 @@ function liveClusterResponses() { }, ], }, - leases: { - items: [ - { - metadata: { name: "t4-cluster-controller" }, - spec: { holderIdentity: "controller-7cbbc8-x7v2k", renewTime: OBSERVED_AT }, - }, - ], + lease: { + metadata: { name: "t4-cluster-operator.cluster.t4.dev" }, + spec: { holderIdentity: "t4-cluster-controller-7cbbc8-x7v2k", renewTime: OBSERVED_AT }, }, customresourcedefinitions: { items: ["t4clusterhosts", "t4workspaces", "t4sessions"].map((plural) => ({ + metadata: { name: `${plural}.cluster.t4.dev` }, spec: { group: "cluster.t4.dev", scope: "Namespaced", @@ -138,34 +161,39 @@ function liveClusterResponses() { })), }, t4clusterhosts: { - items: [{ metadata: { name: "development" }, status: { observedGeneration: 2 } }], + items: [{ metadata: { name: "development", generation: 2 }, status: { observedGeneration: 2 } }], }, t4workspaces: { items: [ { - metadata: { name: "proof-workspace" }, + metadata: { name: "proof-workspace", generation: 3 }, spec: { retentionPolicy: "Retain" }, - status: { - observedGeneration: 3, - pvcRef: { name: "proof-workspace" }, - phase: "Ready", - }, + status: { observedGeneration: 3, pvcName: "proof-workspace-data", phase: "Ready" }, }, ], }, t4sessions: { items: [ { - metadata: { name: "proof-session" }, - spec: { workspaceRef: "proof-workspace" }, - status: { observedGeneration: 5, phase: "Running" }, + metadata: { name: "proof-session", generation: 5 }, + spec: { + workspaceRef: "proof-workspace", + ci: { repositoryId: "71", ref: "refs/heads/agent/t4-cluster-operator", commit: COMMIT }, + }, + status: { observedGeneration: 5, phase: "Running", podName: "t4-session-proof-session" }, }, ], }, persistentvolumeclaims: { items: [ { - metadata: { name: "proof-workspace" }, + metadata: { + name: "proof-workspace-data", + labels: { + "app.kubernetes.io/part-of": "t4-cluster", + "cluster.t4.dev/workspace": "proof-workspace", + }, + }, spec: { accessModes: ["ReadWriteMany"], storageClassName: "t4-workspaces-rwx" }, status: { phase: "Bound", capacity: { storage: "20Gi" } }, }, @@ -173,18 +201,32 @@ function liveClusterResponses() { }, pods: { items: [ + workloadPod("t4-cluster-controller-a", "controller", "controller"), + workloadPod("t4-cluster-server-a", "server", "server"), + workloadPod("t4-cluster-server-b", "server", "server"), { - metadata: { name: "t4-session-proof-session", labels: { "cluster.t4.dev/session": "proof-session" } }, + metadata: { + name: "t4-session-proof-session", + labels: { + "app.kubernetes.io/name": "t4-session-runtime", + "app.kubernetes.io/part-of": "t4-cluster", + "cluster.t4.dev/session": "t4-session-proof-session", + }, + }, spec: { nodeName: "k3s-worker-01" }, - status: { phase: "Running" }, + status: { + phase: "Running", + conditions: [{ type: "Ready", status: "True" }], + containerStatuses: [{ name: "session-runtime", image: workloadImage, imageID: `containerd://${workloadImage}`, ready: true }], + }, }, ], }, services: { items: [ { - metadata: { name: "t4-cluster-server" }, - spec: { ports: [{ name: "omp-app", port: 8080 }, { name: "admin", port: 9090 }] }, + metadata: { name: "t4-cluster-server", labels: labels("server") }, + spec: { ports: [{ name: "websocket", port: 8080 }, { name: "admin", port: 9090 }] }, }, ], }, @@ -237,6 +279,11 @@ test("Woodpecker keeps upstream gates and serializes bounded cluster publication assert.deepEqual(steps["build-controller"].depends_on, ["harbor-auth"]); assert.deepEqual(steps["live-cluster-observations"].depends_on, ["cleanup-image-registry-auth"]); assert.deepEqual(steps["publish-live-proof"].depends_on, ["harbor-auth-live-proof"]); + assert.deepEqual(steps["image-publication-manifest"].depends_on, ["provenance-session-runtime"]); + assert.deepEqual(steps["promote-images"].depends_on, ["image-publication-manifest"]); + assert.deepEqual(steps["publish-image-evidence"].depends_on, ["promote-images"]); + assert.equal(steps["image-publication-manifest"].environment.HARBOR_REGISTRY, "harbor.tailb18de3.ts.net"); + assert.match(pipeline.clone.git.image, /@sha256:[0-9a-f]{64}$/u); const orderedBuilds = ["build-controller", "build-cluster-server", "build-session-runtime"]; for (const [index, name] of orderedBuilds.entries()) { @@ -249,14 +296,56 @@ test("Woodpecker keeps upstream gates and serializes bounded cluster publication assert.match(steps["build-cluster-server"].commands[0], /t4-cluster-server/u); assert.match(steps["build-session-runtime"].commands[0], /t4-session-runtime/u); - for (const script of ["build-image.sh", "capture-image-evidence.sh", "publish-artifact.sh"]) { + for (const [name, step] of Object.entries(steps)) { + assert.match(step.image, /@sha256:[0-9a-f]{64}$/u, `${name} image must be immutable`); + if (step.environment?.HARBOR_REGISTRY) { + assert.equal(step.environment.HARBOR_REGISTRY, "harbor.tailb18de3.ts.net"); + } + for (const condition of step.when ?? []) { + if (condition.event === "manual") { + assert.deepEqual(condition.branch, ["main", "agent/t4-cluster-operator"]); + } + } + } + assert.deepEqual(steps["cleanup-image-registry-auth"].when[0].status, ["success", "failure"]); + assert.deepEqual(steps["cleanup-live-registry-auth"].when[0].status, ["success", "failure"]); + assert.deepEqual(steps["live-frame-proof"].commands, ["node scripts/cluster-ci/capture-redacted-frames.mjs"]); + assert.deepEqual(steps["live-proof-assembly"].depends_on, ["live-frame-proof"]); + const frameCaptureSource = await readFile(resolve(repoRoot, "scripts/cluster-ci/capture-redacted-frames.mjs"), "utf8"); + assert.doesNotMatch(frameCaptureSource, /client.?secret|deviceId|access.?token/iu); + for (const capability of ["sessions.read", "ci.trigger", "preview.read", "preview.control", "preview.input"]) { + assert.match(frameCaptureSource, new RegExp(capability.replace(".", "\\."), "u")); + } + assert.match(frameCaptureSource, /\/v1\/ws/u); + + const buildSource = await readFile(resolve(repoRoot, "scripts/cluster-ci/build-image.sh"), "utf8"); + assert.match(buildSource, /platform=linux\/amd64,linux\/arm64/u); + assert.match(buildSource, /quarantine/u); + const provenanceSource = await readFile(resolve(repoRoot, "scripts/cluster-ci/capture-image-evidence.sh"), "utf8"); + assert.match(provenanceSource, /cosign verify-attestation/u); + assert.match(provenanceSource, /cosign download attestation/u); + assert.match(provenanceSource, /must be configured together/u); + assert.doesNotMatch(provenanceSource, /INSECURE|plain-http/iu); + const dockerignore = await readFile(resolve(repoRoot, ".dockerignore"), "utf8"); + assert.match(dockerignore, /^\.cluster-ci\/registry-auth$/mu); + assert.deepEqual(provenanceVerificationMode({}), { mode: "buildkit-content", signatureVerified: false }); + assert.deepEqual( + provenanceVerificationMode({ T4_COSIGN_CERTIFICATE_IDENTITY: "builder", T4_COSIGN_CERTIFICATE_OIDC_ISSUER: "https://issuer.example" }), + { mode: "cosign-keyless", signatureVerified: true }, + ); + assert.throws( + () => provenanceVerificationMode({ T4_COSIGN_CERTIFICATE_IDENTITY: "builder" }), + /configured together/u, + ); + + for (const script of ["build-image.sh", "capture-image-evidence.sh", "promote-images.sh", "publish-artifact.sh"]) { const source = await readFile(resolve(repoRoot, "scripts/cluster-ci", script), "utf8"); assert.doesNotMatch(source, /HARBOR_(?:USERNAME|PASSWORD)/u); assert.match(source, /DOCKER_CONFIG/u); } }); -test("registry auth is restricted to the fixed internal and tailnet Harbor aliases", () => { +test("registry auth is restricted to the exact HTTPS tailnet Harbor aliases", () => { const auth = "dXNlcjpwYXNz"; const normalized = normalizeRegistryAuth({ auths: { @@ -279,7 +368,7 @@ test("proof schema is strict and enumerates every bounded evidence domain", asyn assert.deepEqual(schema.$defs.observation.properties.system.enum, OBSERVATION_SYSTEMS); assert.deepEqual(schema.$defs.image.properties.component.enum, IMAGE_COMPONENTS); assert.equal(schema.$defs.artifacts.properties.frames.maxItems, 32); - assert.equal(schema.$defs.artifacts.properties.videos.maxItems, 8); + assert.equal("videos" in schema.$defs.artifacts.properties, false); }); test("proof validation accepts exact run/image/scenario identity and rejects fabricated gaps", () => { @@ -302,6 +391,18 @@ test("proof validation accepts exact run/image/scenario identity and rejects fab unredacted.artifacts.frames[0].redacted = false; assert.throws(() => validateProofManifest(unredacted), /redacted/u); + const fabricatedLiveScreenshot = structuredClone(proof); + fabricatedLiveScreenshot.artifacts.screenshots[0].evidenceType = "live"; + assert.throws(() => validateProofManifest(fabricatedLiveScreenshot), /contract evidence/u); + + const fabricatedLiveScenario = structuredClone(proof); + fabricatedLiveScenario.scenarios.find(({ id }) => id === "desktop-viewport").evidenceType = "live"; + assert.throws(() => validateProofManifest(fabricatedLiveScenario), /must be contract/u); + + const unauthenticatedSignerClaim = structuredClone(proof); + unauthenticatedSignerClaim.images[0].provenance.mode = "cosign-keyless"; + assert.throws(() => validateProofManifest(unauthenticatedSignerClaim), /signature verification/u); + const extra = structuredClone(proof); extra.source.token = "must-not-survive"; assert.throws(() => validateProofManifest(extra), /unexpected field/u); @@ -327,6 +428,10 @@ test("frame redaction strips authority-sensitive content and bounds retained sta cursor: 11, revision: 7, authorization: "Bearer secret", + auth: "private-auth", + bearer: "private-bearer", + apiKey: "private-api-key", + privateKey: "private-key", token: "secret-token", prompt: "private prompt", transcript: [{ text: "private output" }], @@ -338,6 +443,10 @@ test("frame redaction strips authority-sensitive content and bounds retained sta cursor: 11, revision: 7, authorization: "[REDACTED]", + auth: "[REDACTED]", + bearer: "[REDACTED]", + apiKey: "[REDACTED]", + privateKey: "[REDACTED]", token: "[REDACTED]", prompt: "[REDACTED]", transcript: "[REDACTED]", @@ -360,24 +469,153 @@ test("read-only collection executes only bounded Kubernetes GETs", async () => { }, }); assert.equal(calls.length, 9); - assert.equal(validateClusterSnapshot(snapshot), snapshot); + assert.equal(validateClusterSnapshot(snapshot, CLUSTER_VALIDATION), snapshot); }); -test("live snapshot validation enforces HA, Lease, CRD, RWX, placement, and service ports", () => { +test("live snapshot validation rejects stale or loosely matched cluster truth", () => { const responses = liveClusterResponses(); - assert.equal(validateClusterSnapshot(responses), responses); + assert.equal(validateClusterSnapshot(responses, CLUSTER_VALIDATION), responses); const noLeader = structuredClone(responses); - noLeader.leases.items[0].spec.holderIdentity = ""; - assert.throws(() => validateClusterSnapshot(noLeader), /leader Lease/u); + noLeader.lease.spec.holderIdentity = ""; + assert.throws(() => validateClusterSnapshot(noLeader, CLUSTER_VALIDATION), /leader Lease/u); + + const staleLeader = structuredClone(responses); + staleLeader.lease.spec.renewTime = "2026-07-20T12:33:00.000Z"; + assert.throws(() => validateClusterSnapshot(staleLeader, CLUSTER_VALIDATION), /freshly held/u); + + const wrongLeaseName = structuredClone(responses); + wrongLeaseName.lease.metadata.name = "t4-cluster-controller"; + assert.throws(() => validateClusterSnapshot(wrongLeaseName, CLUSTER_VALIDATION), /t4-cluster-operator\.cluster\.t4\.dev/u); + + const staleGeneration = structuredClone(responses); + staleGeneration.t4sessions.items[0].status.observedGeneration = 4; + assert.throws(() => validateClusterSnapshot(staleGeneration, CLUSTER_VALIDATION), /metadata\.generation exactly/u); + + const wrongControllerRollout = structuredClone(responses); + wrongControllerRollout.deployments.items[0].spec.strategy.rollingUpdate.maxUnavailable = 0; + assert.throws(() => validateClusterSnapshot(wrongControllerRollout, CLUSTER_VALIDATION), /exact HA rollout/u); + + const legacyPvcField = structuredClone(responses); + legacyPvcField.t4workspaces.items[0].status.pvcRef = { name: "proof-workspace-data" }; + delete legacyPvcField.t4workspaces.items[0].status.pvcName; + assert.throws(() => validateClusterSnapshot(legacyPvcField, CLUSTER_VALIDATION), /status\.pvcName/u); const wrongStorage = structuredClone(responses); wrongStorage.persistentvolumeclaims.items[0].spec.accessModes = ["ReadWriteOnce"]; - assert.throws(() => validateClusterSnapshot(wrongStorage), /ReadWriteMany/u); + assert.throws(() => validateClusterSnapshot(wrongStorage, CLUSTER_VALIDATION), /ReadWriteMany/u); + + const notReady = structuredClone(responses); + notReady.pods.items[3].status.conditions[0].status = "False"; + assert.throws(() => validateClusterSnapshot(notReady, CLUSTER_VALIDATION), /Running and Ready/u); + + const mutableContainer = structuredClone(responses); + mutableContainer.pods.items[3].status.containerStatuses[0].imageID = "containerd://mutable:latest"; + assert.throws(() => validateClusterSnapshot(mutableContainer, CLUSTER_VALIDATION), /current digest/u); + + const mismatchedContainerDigest = structuredClone(responses); + mismatchedContainerDigest.pods.items[3].status.containerStatuses[0].imageID = `containerd://harbor.tailb18de3.ts.net/linkedin-bot/workload@sha256:${"c".repeat(64)}`; + assert.throws(() => validateClusterSnapshot(mismatchedContainerDigest, CLUSTER_VALIDATION), /declared current digest/u); + + const wrongCiCommit = structuredClone(responses); + wrongCiCommit.t4sessions.items[0].spec.ci.commit = "f".repeat(40); + assert.throws(() => validateClusterSnapshot(wrongCiCommit, CLUSTER_VALIDATION), /CI mapping/u); const forbiddenPlacement = structuredClone(responses); - forbiddenPlacement.pods.items[0].spec.nodeName = "k3s-worker-03"; - assert.throws(() => validateClusterSnapshot(forbiddenPlacement), /durable session placement/u); + forbiddenPlacement.pods.items[3].spec.nodeName = "k3s-worker-03"; + assert.throws(() => validateClusterSnapshot(forbiddenPlacement, CLUSTER_VALIDATION), /durable session placement/u); + + const wrongSessionLabel = structuredClone(responses); + wrongSessionLabel.pods.items[3].metadata.labels["cluster.t4.dev/session"] = "proof-session"; + assert.throws(() => validateClusterSnapshot(wrongSessionLabel, CLUSTER_VALIDATION), /label cluster\.t4\.dev\/session/u); + + const wrongPortName = structuredClone(responses); + wrongPortName.services.items[0].spec.ports[0].name = "omp-app"; + assert.throws(() => validateClusterSnapshot(wrongPortName, CLUSTER_VALIDATION), /websocket\/admin ports/u); +}); + +test("SPDX, Trivy, and BuildKit content bind every image identity field", () => { + const repository = "harbor.tailb18de3.ts.net/linkedin-bot/quarantine/t4-cluster-operator"; + const reference = `${repository}@${DIGEST}`; + const spdx = { + spdxVersion: "SPDX-2.3", + dataLicense: "CC0-1.0", + SPDXID: "SPDXRef-DOCUMENT", + name: reference, + documentNamespace: `https://anchore.com/syft/image/${DIGEST.slice(7)}`, + documentDescribes: ["SPDXRef-Image"], + packages: [{ + SPDXID: "SPDXRef-Image", + name: "t4-cluster-operator", + externalRefs: [{ + referenceCategory: "PACKAGE-MANAGER", + referenceType: "purl", + referenceLocator: `pkg:oci/t4-cluster-operator@${DIGEST}?repository_url=${encodeURIComponent(repository)}`, + }], + }], + }; + assert.doesNotThrow(() => verifySpdx(spdx, { repository, digest: DIGEST, reference })); + assert.throws(() => verifySpdx({ spdxVersion: "SPDX-2.3" }, { repository, digest: DIGEST, reference }), /SPDX document identity/u); + const wrongSpdxDigest = structuredClone(spdx); + wrongSpdxDigest.packages[0].externalRefs[0].referenceLocator = "pkg:oci/t4-cluster-operator@sha256:deadbeef"; + assert.throws(() => verifySpdx(wrongSpdxDigest, { repository, digest: DIGEST, reference }), /external reference/u); + + const trivy = { + ArtifactName: reference, + ArtifactType: "container_image", + Metadata: { ImageID: DIGEST, RepoDigests: [reference] }, + Results: [{ Target: "debian", Class: "os-pkgs", Type: "debian", Vulnerabilities: [] }], + }; + assert.deepEqual(vulnerabilityCounts(trivy, { repository, digest: DIGEST, reference }), { critical: 0, high: 0 }); + assert.throws( + () => vulnerabilityCounts({ ...trivy, Results: [] }, { repository, digest: DIGEST, reference }), + /results identity/u, + ); + assert.throws( + () => vulnerabilityCounts({ ...trivy, ArtifactName: "wrong" }, { repository, digest: DIGEST, reference }), + /artifact\/results identity/u, + ); + + const statement = { + _type: "https://in-toto.io/Statement/v0.1", + predicateType: "https://slsa.dev/provenance/v0.2", + subject: [{ name: repository, digest: { sha256: DIGEST.slice(7) } }], + predicate: { + builder: { id: "https://mobyproject.org/buildkit@v1" }, + buildType: "https://mobyproject.org/buildkit@v1", + invocation: { parameters: { source: "https://github.com/z-peterson/t4-code", commit: COMMIT } }, + materials: [ + { uri: `https://github.com/z-peterson/t4-code.git#${COMMIT}`, digest: { sha1: COMMIT } }, + { uri: `pkg:docker/node@${DIGEST}`, digest: { sha256: DIGEST.slice(7) } }, + ], + }, + }; + const envelope = (payload) => `${JSON.stringify({ + payloadType: "application/vnd.in-toto+json", + payload: Buffer.from(JSON.stringify(payload)).toString("base64"), + })}\n`; + assert.doesNotThrow(() => verifyProvenance(envelope(statement), { repository, digest: DIGEST, commit: COMMIT })); + const wrongSource = structuredClone(statement); + wrongSource.predicate.materials[0].uri = `https://github.com/attacker/t4-code.git#${COMMIT}`; + assert.throws(() => verifyProvenance(envelope(wrongSource), { repository, digest: DIGEST, commit: COMMIT }), /trusted source/u); + assert.throws(() => verifyProvenance(envelope(statement), { repository, digest: DIGEST, commit: "f".repeat(40) }), /CI commit/u); +}); + +test("proof fetch and frame URLs reject every host or credential outside the exact allowlist", () => { + assert.equal( + trustedProofUrl("https://interview-responder-prometheus.tailb18de3.ts.net/evidence.json", "prometheus", ["interview-responder-prometheus.tailb18de3.ts.net"]), + "https://interview-responder-prometheus.tailb18de3.ts.net/evidence.json", + ); + for (const value of [ + "https://attacker.example/evidence.json", + "http://interview-responder-prometheus.tailb18de3.ts.net/evidence.json", + "https://user:password@interview-responder-prometheus.tailb18de3.ts.net/evidence.json", + "https://interview-responder-prometheus.tailb18de3.ts.net/evidence.json?token=value", + ]) { + assert.throws(() => trustedProofUrl(value, "prometheus", ["interview-responder-prometheus.tailb18de3.ts.net"]), /allowlist/u); + } + assert.equal(clusterWebSocketUrl("https://t4-dev.tailb18de3.ts.net/").href, "wss://t4-dev.tailb18de3.ts.net/v1/ws"); + assert.throws(() => clusterWebSocketUrl("https://other.tailb18de3.ts.net/"), /credential-free HTTPS origin/u); }); test("default-off proof evaluates rendered resources rather than chart source text", () => { diff --git a/scripts/cluster-ci/cluster-proof.schema.json b/scripts/cluster-ci/cluster-proof.schema.json index 7e68e96e..d628c647 100644 --- a/scripts/cluster-ci/cluster-proof.schema.json +++ b/scripts/cluster-ci/cluster-proof.schema.json @@ -22,8 +22,8 @@ }, "observations": { "type": "array", - "minItems": 7, - "maxItems": 7, + "minItems": 5, + "maxItems": 5, "items": { "$ref": "#/$defs/observation" } }, "artifacts": { "$ref": "#/$defs/artifacts" } @@ -79,7 +79,7 @@ "maxLength": 340 }, "sbom": { "$ref": "#/$defs/fileEvidence" }, - "provenance": { "$ref": "#/$defs/fileEvidence" }, + "provenance": { "$ref": "#/$defs/provenanceEvidence" }, "vulnerability": { "type": "object", "additionalProperties": false, @@ -103,10 +103,28 @@ "sha256": { "$ref": "#/$defs/sha256" } } }, + "provenanceEvidence": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256", "mode", "signatureVerified"], + "properties": { + "path": { "$ref": "#/$defs/artifactPath" }, + "sha256": { "$ref": "#/$defs/sha256" }, + "mode": { "enum": ["buildkit-content", "cosign-keyless"] }, + "signatureVerified": { "type": "boolean" } + }, + "allOf": [ + { + "if": { "properties": { "mode": { "const": "cosign-keyless" } } }, + "then": { "properties": { "signatureVerified": { "const": true } } }, + "else": { "properties": { "signatureVerified": { "const": false } } } + } + ] + }, "scenario": { "type": "object", "additionalProperties": false, - "required": ["id", "status", "observedAt", "assertions", "evidence"], + "required": ["id", "status", "evidenceType", "observedAt", "assertions", "evidence"], "properties": { "id": { "enum": [ @@ -122,6 +140,7 @@ ] }, "status": { "const": "passed" }, + "evidenceType": { "enum": ["contract", "live"] }, "observedAt": { "$ref": "#/$defs/timestamp" }, "assertions": { "type": "array", @@ -143,9 +162,9 @@ "additionalProperties": false, "required": ["system", "observedAt", "url", "ids", "evidence"], "properties": { - "system": { "enum": ["woodpecker", "flux", "kubernetes", "tailscale", "prometheus", "loki", "grafana"] }, + "system": { "enum": ["woodpecker", "kubernetes", "prometheus", "loki", "grafana"] }, "observedAt": { "$ref": "#/$defs/timestamp" }, - "url": { "$ref": "#/$defs/httpsUrl" }, + "url": { "oneOf": [{ "$ref": "#/$defs/httpsUrl" }, { "type": "null" }] }, "ids": { "type": "array", "minItems": 1, @@ -159,18 +178,19 @@ "visualEvidence": { "type": "object", "additionalProperties": false, - "required": ["path", "sha256", "redacted"], + "required": ["path", "sha256", "redacted", "evidenceType"], "properties": { "path": { "$ref": "#/$defs/artifactPath" }, "sha256": { "$ref": "#/$defs/sha256" }, "redacted": { "const": true }, + "evidenceType": { "enum": ["contract", "live"] }, "viewport": { "enum": ["desktop", "mobile"] } } }, "artifacts": { "type": "object", "additionalProperties": false, - "required": ["frames", "screenshots", "videos"], + "required": ["frames", "screenshots"], "properties": { "frames": { "type": "array", @@ -183,12 +203,6 @@ "minItems": 2, "maxItems": 32, "items": { "$ref": "#/$defs/visualEvidence" } - }, - "videos": { - "type": "array", - "minItems": 2, - "maxItems": 8, - "items": { "$ref": "#/$defs/visualEvidence" } } } } diff --git a/scripts/cluster-ci/collect-readonly-cluster.mjs b/scripts/cluster-ci/collect-readonly-cluster.mjs index 63095430..462e9dfd 100644 --- a/scripts/cluster-ci/collect-readonly-cluster.mjs +++ b/scripts/cluster-ci/collect-readonly-cluster.mjs @@ -18,6 +18,34 @@ const proofRoot = resolve(repoRoot, "artifacts/cluster-proof"); const scenarioRoot = resolve(proofRoot, "scenarios"); const observationRoot = resolve(proofRoot, "observations"); +function requiredEnvironment(name) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +function currentCiMapping() { + const pipelineUrl = new URL(requiredEnvironment("CI_PIPELINE_URL")); + const match = pipelineUrl.pathname.match(/^\/repos\/([1-9][0-9]*)\/pipeline\/[1-9][0-9]*\/?$/u); + if ( + pipelineUrl.origin !== "https://woodpecker-ci-dev.tailb18de3.ts.net" || + pipelineUrl.username || + pipelineUrl.password || + pipelineUrl.search || + pipelineUrl.hash || + !match + ) { + throw new Error("CI_PIPELINE_URL does not identify the exact credential-free Woodpecker repository"); + } + const repository = requiredEnvironment("CI_REPO"); + if (repository !== "z-peterson/t4-code") throw new Error("CI_REPO is not the canonical source repository"); + return { + repositoryId: match[1], + ref: requiredEnvironment("CI_COMMIT_REF"), + commit: requiredEnvironment("CI_COMMIT_SHA"), + }; +} + async function atomicJson(path, value) { const temporaryPath = `${path}.tmp`; await writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); @@ -54,16 +82,17 @@ async function defaultOffEvidence(namespace) { return result; } -export async function collectClusterEvidence({ namespace }) { +export async function collectClusterEvidence({ namespace, ciMapping = currentCiMapping() }) { await mkdir(scenarioRoot, { recursive: true }); await mkdir(observationRoot, { recursive: true }); const snapshot = await collectReadOnlyClusterSnapshot({ namespace }); - const summary = summarizeClusterSnapshot(snapshot); + const summary = summarizeClusterSnapshot(snapshot, { ciMapping }); await atomicJson(resolve(observationRoot, "kubernetes.json"), summary); await scenario("ha-manifest", summary.observedAt, [ "controller.replicas-2", + "controller.rolling-update.max-unavailable-1", "server.replicas-3", - "rolling-update.max-unavailable-0", + "server.rolling-update.max-unavailable-0", ]); await scenario("leader-election", summary.observedAt, [ "lease.active-holder", @@ -76,6 +105,12 @@ export async function collectClusterEvidence({ namespace }) { "storage.bound-read-write-many", "placement.session-worker-exclusions", ]); + await scenario("ci-mapping", summary.observedAt, [ + "ci.repository-id-exact", + "ci.ref-exact", + "ci.commit-exact", + "ci.session-running-ready", + ]); const off = await defaultOffEvidence(namespace); await scenario("feature-off", new Date().toISOString(), [ off.clusterOperatorEnabled ? "feature-off.invalid" : "feature-off.no-workloads", diff --git a/scripts/cluster-ci/normalize-registry-auth.mjs b/scripts/cluster-ci/normalize-registry-auth.mjs index caae1b02..3dce5db8 100644 --- a/scripts/cluster-ci/normalize-registry-auth.mjs +++ b/scripts/cluster-ci/normalize-registry-auth.mjs @@ -4,9 +4,7 @@ import { fileURLToPath } from "node:url"; export const HARBOR_REGISTRY_ALIASES = Object.freeze([ "harbor.tailb18de3.ts.net", - "harbor.tailb18de3.ts.net:80", - "harbor.linkedin-registry.svc.cluster.local", - "harbor.linkedin-registry.svc.cluster.local:80", + "https://harbor.tailb18de3.ts.net", ]); function isObject(value) { diff --git a/scripts/cluster-ci/promote-images.sh b/scripts/cluster-ci/promote-images.sh new file mode 100755 index 00000000..e96b85c1 --- /dev/null +++ b/scripts/cluster-ci/promote-images.sh @@ -0,0 +1,43 @@ +#!/bin/sh +set -eu + +umask 077 + +: "${CI_COMMIT_SHA:?CI_COMMIT_SHA is required}" +: "${HARBOR_REGISTRY:?HARBOR_REGISTRY is required}" +: "${HARBOR_PROJECT:?HARBOR_PROJECT is required}" +if [ "$HARBOR_REGISTRY" != "harbor.tailb18de3.ts.net" ]; then + echo "HARBOR_REGISTRY must be the exact HTTPS tailnet Harbor host" >&2 + exit 64 +fi +case "$CI_COMMIT_SHA" in + [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) ;; + *) echo "CI_COMMIT_SHA must be an exact lowercase 40-character SHA" >&2; exit 64 ;; +esac + +auth_dir=${T4_REGISTRY_AUTH_DIR:-${CI_WORKSPACE:-$PWD}/.cluster-ci/registry-auth} +test -r "$auth_dir/config.json" +export DOCKER_CONFIG="$auth_dir" +artifact_dir=artifacts/cluster-proof/images + +for entry in \ + controller:t4-cluster-operator \ + cluster-server:t4-cluster-server \ + session-runtime:t4-session-runtime +do + component=${entry%%:*} + repository_suffix=${entry#*:} + digest=$(cat "$artifact_dir/$component.digest") + case "$digest" in + sha256:????????????????????????????????????????????????????????????????) ;; + *) echo "$component image digest artifact is malformed" >&2; exit 65 ;; + esac + source="$HARBOR_REGISTRY/$HARBOR_PROJECT/quarantine/$repository_suffix@$digest" + destination="$HARBOR_REGISTRY/$HARBOR_PROJECT/$repository_suffix:$CI_COMMIT_SHA" + oras copy --recursive "$source" "$destination" + resolved=$(oras resolve "$destination") + if [ "$resolved" != "$digest" ]; then + echo "$component promoted reference did not resolve to the gated digest" >&2 + exit 65 + fi +done diff --git a/scripts/cluster-ci/proof-contract.mjs b/scripts/cluster-ci/proof-contract.mjs index 5b99013a..55bc9af3 100644 --- a/scripts/cluster-ci/proof-contract.mjs +++ b/scripts/cluster-ci/proof-contract.mjs @@ -16,13 +16,17 @@ export const PROOF_SCENARIOS = Object.freeze([ ]); export const OBSERVATION_SYSTEMS = Object.freeze([ "woodpecker", - "flux", "kubernetes", - "tailscale", "prometheus", "loki", "grafana", ]); +const CONTRACT_SCENARIOS = new Set([ + "wire-reconnect-idempotency", + "gui-auth-isolation", + "desktop-viewport", + "mobile-viewport", +]); const COMMIT_PATTERN = /^[0-9a-f]{40}$/u; const SHA_PATTERN = /^[0-9a-f]{64}$/u; @@ -30,11 +34,17 @@ const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; const REPOSITORY_PATTERN = /^[a-z0-9.-]+(?::[0-9]+)?\/[a-z0-9._/-]+$/u; const ARTIFACT_PATH_PATTERN = /^artifacts\/cluster-proof\/[A-Za-z0-9][A-Za-z0-9._/-]{0,255}$/u; const ASSERTION_PATTERN = /^[a-z0-9][a-z0-9._-]{0,127}$/u; -const SENSITIVE_KEY_PATTERN = /authorization|cookie|credential|password|prompt|secret|token|transcript/iu; +const SENSITIVE_KEY_PATTERN = /api.?key|private.?key|authorization|auth|bearer|cookie|credential|password|prompt|secret|token|transcript/iu; const MAX_FRAME_BYTES = 64 * 1024; const MAX_DEPTH = 12; const MAX_OBJECT_FIELDS = 128; const MAX_ARRAY_ITEMS = 256; +const OBSERVATION_HOSTS = Object.freeze({ + woodpecker: new Set(["woodpecker-ci-dev.tailb18de3.ts.net"]), + prometheus: new Set(["interview-responder-prometheus.tailb18de3.ts.net"]), + loki: new Set(["interview-responder-loki.tailb18de3.ts.net"]), + grafana: new Set(["grafana.tailb18de3.ts.net"]), +}); function fail(message) { throw new Error(`T4 cluster proof ${message}`); @@ -106,6 +116,19 @@ function httpsUrl(value, label) { return value; } +function observationUrl(value, system, label) { + if (system === "kubernetes") { + if (value !== null) fail(`${label} must be null because Kubernetes was captured through bounded kubectl GETs`); + return value; + } + httpsUrl(value, label); + const url = new URL(value); + if (url.port || !OBSERVATION_HOSTS[system]?.has(url.hostname)) { + fail(`${label} is outside the exact ${system} tailnet allowlist`); + } + return value; +} + function fileEvidence(value, label, extraFields = []) { const fields = ["path", "sha256", ...extraFields]; exactFields(value, fields, label); @@ -117,9 +140,10 @@ function fileEvidence(value, label, extraFields = []) { } function visualEvidence(value, label) { - const fields = ["path", "sha256", "redacted", ...(value.viewport === undefined ? [] : ["viewport"] )]; + const fields = ["path", "sha256", "redacted", "evidenceType", ...(value.viewport === undefined ? [] : ["viewport"] )]; fileEvidence(value, label, fields.slice(2)); if (value.redacted !== true) fail(`${label} must be redacted`); + if (!["contract", "live"].includes(value.evidenceType)) fail(`${label}.evidenceType is invalid`); if (value.viewport !== undefined && !["desktop", "mobile"].includes(value.viewport)) { fail(`${label}.viewport is invalid`); } @@ -163,7 +187,13 @@ export function validateImageEntries(images, sourceCommit) { fail(`${label}.reference must use the immutable digest`); } fileEvidence(image.sbom, `${label}.sbom`); - fileEvidence(image.provenance, `${label}.provenance`); + fileEvidence(image.provenance, `${label}.provenance`, ["mode", "signatureVerified"]); + if ( + !["buildkit-content", "cosign-keyless"].includes(image.provenance.mode) || + image.provenance.signatureVerified !== (image.provenance.mode === "cosign-keyless") + ) { + fail(`${label}.provenance signer verification claim is not truthful`); + } fileEvidence(image.vulnerability, `${label}.vulnerability`, ["scanner", "critical", "high"]); if ( image.vulnerability.scanner !== "trivy" || @@ -180,9 +210,11 @@ function validateScenarioEntries(scenarios) { exactSet(scenarios, PROOF_SCENARIOS, "scenario"); for (const [index, scenario] of scenarios.entries()) { const label = `scenarios[${index}]`; - exactFields(scenario, ["id", "status", "observedAt", "assertions", "evidence"], label); + exactFields(scenario, ["id", "status", "evidenceType", "observedAt", "assertions", "evidence"], label); if (!PROOF_SCENARIOS.includes(scenario.id)) fail(`${label}.id is invalid`); if (scenario.status !== "passed") fail(`${label} must be passed`); + const expectedType = CONTRACT_SCENARIOS.has(scenario.id) ? "contract" : "live"; + if (scenario.evidenceType !== expectedType) fail(`${label}.evidenceType must be ${expectedType}`); timestamp(scenario.observedAt, `${label}.observedAt`); if ( !Array.isArray(scenario.assertions) || @@ -210,7 +242,7 @@ function validateObservationEntries(observations) { exactFields(observation, ["system", "observedAt", "url", "ids", "evidence"], label); if (!OBSERVATION_SYSTEMS.includes(observation.system)) fail(`${label}.system is invalid`); timestamp(observation.observedAt, `${label}.observedAt`); - httpsUrl(observation.url, `${label}.url`); + observationUrl(observation.url, observation.system, `${label}.url`); if ( !Array.isArray(observation.ids) || observation.ids.length < 1 || @@ -226,24 +258,25 @@ function validateObservationEntries(observations) { } function validateArtifacts(artifacts) { - exactFields(artifacts, ["frames", "screenshots", "videos"], "artifacts"); + exactFields(artifacts, ["frames", "screenshots"], "artifacts"); const bounds = { frames: [1, 32], screenshots: [2, 32], - videos: [2, 8], }; for (const [kind, [minimum, maximum]] of Object.entries(bounds)) { const entries = artifacts[kind]; if (!Array.isArray(entries) || entries.length < minimum || entries.length > maximum) { fail(`artifacts.${kind} is outside its bound`); } - entries.forEach((entry, index) => visualEvidence(entry, `artifacts.${kind}[${index}]`)); + const expectedType = kind === "frames" ? "live" : "contract"; + entries.forEach((entry, index) => { + visualEvidence(entry, `artifacts.${kind}[${index}]`); + if (entry.evidenceType !== expectedType) fail(`artifacts.${kind} must be ${expectedType} evidence`); + }); } - for (const kind of ["screenshots", "videos"]) { - const viewports = new Set(artifacts[kind].map(({ viewport }) => viewport)); - if (!viewports.has("desktop") || !viewports.has("mobile")) { - fail(`artifacts.${kind} must include desktop and mobile evidence`); - } + const viewports = new Set(artifacts.screenshots.map(({ viewport }) => viewport)); + if (!viewports.has("desktop") || !viewports.has("mobile")) { + fail("artifacts.screenshots must include desktop and mobile contract evidence"); } return artifacts; } diff --git a/scripts/cluster-ci/publish-artifact.sh b/scripts/cluster-ci/publish-artifact.sh index ec1a5468..2deeb99f 100755 --- a/scripts/cluster-ci/publish-artifact.sh +++ b/scripts/cluster-ci/publish-artifact.sh @@ -15,7 +15,7 @@ case "$mode" in repository_suffix=t4-cluster-proof tag="${CI_COMMIT_SHA:-}-${CI_PIPELINE_NUMBER:-}" artifact_type=application/vnd.t4.cluster.proof.v1 - files="artifacts/cluster-proof/manifest.json artifacts/cluster-proof/scenarios/* artifacts/cluster-proof/observations/* artifacts/cluster-proof/frames/* artifacts/cluster-proof/screenshots/* artifacts/cluster-proof/videos/*" + files="artifacts/cluster-proof/manifest.json artifacts/cluster-proof/scenarios/* artifacts/cluster-proof/observations/* artifacts/cluster-proof/frames/* artifacts/cluster-proof/screenshots/*" ;; *) echo "artifact mode must be images or proof" >&2 @@ -38,13 +38,16 @@ fi : "${HARBOR_REGISTRY:?HARBOR_REGISTRY is required}" : "${HARBOR_PROJECT:?HARBOR_PROJECT is required}" +if [ "$HARBOR_REGISTRY" != "harbor.tailb18de3.ts.net" ]; then + echo "HARBOR_REGISTRY must be the exact HTTPS tailnet Harbor host" >&2 + exit 64 +fi auth_dir=${T4_REGISTRY_AUTH_DIR:-${CI_WORKSPACE:-$PWD}/.cluster-ci/registry-auth} test -r "$auth_dir/config.json" export DOCKER_CONFIG="$auth_dir" reference="$HARBOR_REGISTRY/$HARBOR_PROJECT/$repository_suffix:$tag" # shellcheck disable=SC2086 oras push \ - --plain-http \ --artifact-type "$artifact_type" \ --format json \ "$reference" \ diff --git a/scripts/cluster-ci/readonly-cluster-proof.mjs b/scripts/cluster-ci/readonly-cluster-proof.mjs index ee816e1c..9baaaf12 100644 --- a/scripts/cluster-ci/readonly-cluster-proof.mjs +++ b/scripts/cluster-ci/readonly-cluster-proof.mjs @@ -3,13 +3,21 @@ import { promisify } from "node:util"; const execFileAsync = promisify(execFile); const MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +const MAX_LEASE_AGE_MS = 30_000; +const MAX_CLOCK_SKEW_MS = 5_000; const NAMESPACE_PATTERN = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; +const DIGEST_PATTERN = /@sha256:[0-9a-f]{64}$/u; const FORBIDDEN_SESSION_NODES = new Set(["k3s-worker-02", "k3s-worker-03"]); const WORKLOAD_KINDS = new Set(["CronJob", "DaemonSet", "Deployment", "Job", "Pod", "StatefulSet"]); +const LEASE_NAME = "t4-cluster-operator.cluster.t4.dev"; +const DEPLOYMENTS = Object.freeze([ + { name: "t4-cluster-controller", component: "controller", container: "controller", replicas: 2, maxUnavailable: 1, minimumAvailable: 1 }, + { name: "t4-cluster-server", component: "server", container: "server", replicas: 3, maxUnavailable: 0, minimumAvailable: 2 }, +]); const REQUESTS = Object.freeze([ ["deployments", ["get", "deployments", "-n", "$NAMESPACE", "-l", "app.kubernetes.io/part-of=t4-cluster", "-o", "json"]], - ["leases", ["get", "leases", "-n", "$NAMESPACE", "-l", "app.kubernetes.io/part-of=t4-cluster", "-o", "json"]], + ["lease", ["get", "lease", LEASE_NAME, "-n", "$NAMESPACE", "-o", "json"]], [ "customresourcedefinitions", [ @@ -42,9 +50,9 @@ function list(snapshot, key) { return value.items; } -function named(items, fragment, label) { - const matches = items.filter(({ metadata }) => metadata?.name?.includes(fragment)); - if (matches.length !== 1) fail(`expected exactly one ${label}`); +function exactNamed(items, name, label) { + const matches = items.filter(({ metadata }) => metadata?.name === name); + if (matches.length !== 1) fail(`expected exactly one ${label} named ${name}`); return matches[0]; } @@ -52,15 +60,55 @@ function positive(value) { return Number.isSafeInteger(value) && value > 0; } -function deploymentContract(deployment, replicas, minimumAvailable, label) { +function observed(resource, label) { + if (!positive(resource.metadata?.generation) || resource.status?.observedGeneration !== resource.metadata.generation) { + fail(`${label} status does not observe metadata.generation exactly`); + } +} + +function label(resource, key, expected, resourceLabel) { + if (resource.metadata?.labels?.[key] !== expected) { + fail(`${resourceLabel} label ${key} is not exactly ${expected}`); + } +} + +function readyPod(pod, labelText) { + if ( + pod.status?.phase !== "Running" || + !Array.isArray(pod.status?.conditions) || + !pod.status.conditions.some(({ type, status }) => type === "Ready" && status === "True") + ) { + fail(`${labelText} is not Running and Ready`); + } +} + +function currentContainer(pod, name, labelText) { + const statuses = pod.status?.containerStatuses; + const matches = Array.isArray(statuses) ? statuses.filter((status) => status?.name === name) : []; + const imageDigest = matches[0]?.image?.match(/@(sha256:[0-9a-f]{64})$/u)?.[1]; + if ( + matches.length !== 1 || + matches[0].ready !== true || + !imageDigest || + !DIGEST_PATTERN.test(matches[0].imageID ?? "") || + !matches[0].imageID.endsWith(`@${imageDigest}`) + ) { + fail(`${labelText} has no exact ready ${name} container at its declared current digest`); + } + return matches[0]; +} + +function deploymentContract(deployment, contract) { + label(deployment, "app.kubernetes.io/part-of", "t4-cluster", `${contract.name} Deployment`); + label(deployment, "app.kubernetes.io/component", contract.component, `${contract.name} Deployment`); + observed(deployment, `${contract.name} Deployment`); if ( - deployment.spec?.replicas !== replicas || + deployment.spec?.replicas !== contract.replicas || deployment.spec?.strategy?.type !== "RollingUpdate" || - ![0, "0"].includes(deployment.spec?.strategy?.rollingUpdate?.maxUnavailable) || - !positive(deployment.status?.observedGeneration) || - (deployment.status?.availableReplicas ?? 0) < minimumAvailable + ![contract.maxUnavailable, String(contract.maxUnavailable)].includes(deployment.spec?.strategy?.rollingUpdate?.maxUnavailable) || + (deployment.status?.availableReplicas ?? 0) < contract.minimumAvailable ) { - fail(`${label} Deployment did not satisfy its HA rollout contract`); + fail(`${contract.name} Deployment did not satisfy its exact HA rollout contract`); } } @@ -91,93 +139,147 @@ export async function collectReadOnlyClusterSnapshot({ namespace, run = defaultR return snapshot; } -export function validateClusterSnapshot(snapshot) { +function validateCiMapping(sessions, expected) { + const mappings = sessions.filter(({ spec }) => { + const ci = spec?.ci; + return ( + ci && + typeof ci.repositoryId === "string" && ci.repositoryId.length > 0 && ci.repositoryId.length <= 128 && + typeof ci.ref === "string" && ci.ref.length > 0 && ci.ref.length <= 256 && + typeof ci.commit === "string" && /^[0-9a-f]{40}$/u.test(ci.commit) + ); + }); + if (mappings.length < 1) fail("no live T4Session carries an exact CI mapping"); + if (expected) { + const matches = mappings.filter(({ spec }) => + spec.ci.repositoryId === expected.repositoryId && + spec.ci.ref === expected.ref && + spec.ci.commit === expected.commit, + ); + if (matches.length !== 1) fail("live T4Session CI mapping does not exactly match this repository, ref, and commit"); + return matches; + } + return mappings; +} + +export function validateClusterSnapshot(snapshot, { now = Date.now(), ciMapping } = {}) { const deployments = list(snapshot, "deployments"); - deploymentContract(named(deployments, "controller", "controller Deployment"), 2, 1, "controller"); - deploymentContract(named(deployments, "server", "cluster-server Deployment"), 3, 2, "cluster-server"); + for (const contract of DEPLOYMENTS) { + deploymentContract(exactNamed(deployments, contract.name, `${contract.component} Deployment`), contract); + } - const leases = list(snapshot, "leases"); - const leaderLease = named(leases, "controller", "controller leader Lease"); + const lease = snapshot?.lease; + const renewTime = Date.parse(lease?.spec?.renewTime ?? ""); if ( - typeof leaderLease.spec?.holderIdentity !== "string" || - leaderLease.spec.holderIdentity.length < 1 || - leaderLease.spec.holderIdentity.length > 253 || - !Number.isFinite(Date.parse(leaderLease.spec?.renewTime ?? "")) + !lease || + lease.metadata?.name !== LEASE_NAME || + typeof lease.spec?.holderIdentity !== "string" || + lease.spec.holderIdentity.length < 1 || + lease.spec.holderIdentity.length > 253 || + !Number.isFinite(renewTime) || + renewTime > now + MAX_CLOCK_SKEW_MS || + now - renewTime > MAX_LEASE_AGE_MS ) { - fail("controller leader Lease is not currently held"); + fail(`controller leader Lease ${LEASE_NAME} is not freshly held`); } const crds = list(snapshot, "customresourcedefinitions"); - const requiredCrds = new Set(["t4clusterhosts", "t4workspaces", "t4sessions"]); - for (const crd of crds) { - const version = crd.spec?.versions?.find(({ name }) => name === "v1alpha1"); + for (const plural of ["t4clusterhosts", "t4workspaces", "t4sessions"]) { + const crd = exactNamed(crds, `${plural}.cluster.t4.dev`, `${plural} CRD`); + const versions = Array.isArray(crd.spec?.versions) ? crd.spec.versions : []; + const version = versions.find(({ name }) => name === "v1alpha1"); if ( - crd.spec?.group === "cluster.t4.dev" && - crd.spec?.scope === "Namespaced" && - version?.served === true && - version?.storage === true + crd.spec?.group !== "cluster.t4.dev" || + crd.spec?.scope !== "Namespaced" || + crd.spec?.names?.plural !== plural || + version?.served !== true || + version?.storage !== true ) { - requiredCrds.delete(crd.spec?.names?.plural); + fail(`${plural} CRD does not satisfy the exact v1alpha1 contract`); } } - if (requiredCrds.size > 0) fail(`CRD contract is missing ${[...requiredCrds].join(",")}`); const hosts = list(snapshot, "t4clusterhosts"); - if (hosts.length < 1 || !hosts.every((host) => positive(host.status?.observedGeneration))) { - fail("T4ClusterHost status has not observed the desired generation"); - } + if (hosts.length < 1) fail("T4ClusterHost proof resource is absent"); + hosts.forEach((host) => observed(host, `T4ClusterHost ${host.metadata?.name ?? "unknown"}`)); const workspaces = list(snapshot, "t4workspaces"); const sessions = list(snapshot, "t4sessions"); const pvcs = list(snapshot, "persistentvolumeclaims"); + const pods = list(snapshot, "pods"); if (workspaces.length < 1 || sessions.length < 1 || pvcs.length < 1) { fail("workspace/session/storage proof resources are absent"); } for (const workspace of workspaces) { - if ( - !positive(workspace.status?.observedGeneration) || - workspace.status?.phase !== "Ready" || - !["Retain", "Delete"].includes(workspace.spec?.retentionPolicy) - ) { - fail(`workspace ${workspace.metadata?.name ?? "unknown"} is not reconciled Ready`); + const name = workspace.metadata?.name ?? "unknown"; + observed(workspace, `workspace ${name}`); + if (workspace.status?.phase !== "Ready" || !["Retain", "Delete"].includes(workspace.spec?.retentionPolicy)) { + fail(`workspace ${name} is not reconciled Ready`); + } + const pvcName = workspace.status?.pvcName; + if (typeof pvcName !== "string" || pvcName.length < 1 || pvcName.length > 63) { + fail(`workspace ${name} status.pvcName is invalid`); } - const pvcName = workspace.status?.pvcRef?.name; - const pvc = pvcs.find(({ metadata }) => metadata?.name === pvcName); - if (!pvc || pvc.status?.phase !== "Bound") fail(`workspace ${workspace.metadata?.name ?? "unknown"} PVC is not Bound`); + const pvc = exactNamed(pvcs, pvcName, `workspace ${name} PVC`); + label(pvc, "app.kubernetes.io/part-of", "t4-cluster", `workspace ${name} PVC`); + label(pvc, "cluster.t4.dev/workspace", name, `workspace ${name} PVC`); + if (pvc.status?.phase !== "Bound") fail(`workspace ${name} PVC is not Bound`); if ( !Array.isArray(pvc.spec?.accessModes) || pvc.spec.accessModes.length !== 1 || pvc.spec.accessModes[0] !== "ReadWriteMany" ) { - fail(`workspace ${workspace.metadata?.name ?? "unknown"} PVC is not ReadWriteMany`); + fail(`workspace ${name} PVC is not ReadWriteMany`); } if (typeof pvc.spec?.storageClassName !== "string" || pvc.spec.storageClassName.length === 0) { - fail(`workspace ${workspace.metadata?.name ?? "unknown"} PVC has no StorageClass`); + fail(`workspace ${name} PVC has no StorageClass`); } } + for (const session of sessions) { - if (!positive(session.status?.observedGeneration) || session.status?.phase !== "Running") { - fail(`session ${session.metadata?.name ?? "unknown"} is not reconciled Running`); - } + const name = session.metadata?.name ?? "unknown"; + observed(session, `session ${name}`); + if (session.status?.phase !== "Running") fail(`session ${name} is not reconciled Running`); if (!workspaces.some(({ metadata }) => metadata?.name === session.spec?.workspaceRef)) { - fail(`session ${session.metadata?.name ?? "unknown"} references an unknown workspace`); + fail(`session ${name} references an unknown workspace`); + } + const podName = session.status?.podName; + if (typeof podName !== "string" || podName.length < 1 || podName.length > 63) { + fail(`session ${name} status.podName is invalid`); + } + const pod = exactNamed(pods, podName, `session ${name} Pod`); + label(pod, "app.kubernetes.io/name", "t4-session-runtime", `session ${name} Pod`); + label(pod, "app.kubernetes.io/part-of", "t4-cluster", `session ${name} Pod`); + label(pod, "cluster.t4.dev/session", podName, `session ${name} Pod`); + readyPod(pod, `session ${name} Pod`); + currentContainer(pod, "session-runtime", `session ${name} Pod`); + if (FORBIDDEN_SESSION_NODES.has(pod.spec?.nodeName)) { + fail(`durable session placement is invalid for ${podName}`); } } + validateCiMapping(sessions, ciMapping); - const pods = list(snapshot, "pods"); - const sessionPods = pods.filter(({ metadata }) => metadata?.labels?.["cluster.t4.dev/session"]); - if (sessionPods.length < 1) fail("no durable session pod was observed"); - for (const pod of sessionPods) { - if (pod.status?.phase !== "Running" || FORBIDDEN_SESSION_NODES.has(pod.spec?.nodeName)) { - fail(`durable session placement is invalid for ${pod.metadata?.name ?? "unknown"}`); + for (const contract of DEPLOYMENTS) { + const matchingPods = pods.filter((pod) => + pod.metadata?.labels?.["app.kubernetes.io/part-of"] === "t4-cluster" && + pod.metadata?.labels?.["app.kubernetes.io/component"] === contract.component, + ); + if (matchingPods.length < contract.minimumAvailable) { + fail(`${contract.name} has too few exactly labelled pods`); + } + for (const pod of matchingPods) { + readyPod(pod, `${contract.name} Pod ${pod.metadata?.name ?? "unknown"}`); + currentContainer(pod, contract.container, `${contract.name} Pod ${pod.metadata?.name ?? "unknown"}`); } } const services = list(snapshot, "services"); - const serverService = named(services, "cluster-server", "cluster-server Service"); + const serverService = exactNamed(services, "t4-cluster-server", "cluster-server Service"); + label(serverService, "app.kubernetes.io/part-of", "t4-cluster", "cluster-server Service"); + label(serverService, "app.kubernetes.io/component", "server", "cluster-server Service"); const ports = new Map((serverService.spec?.ports ?? []).map(({ name, port }) => [name, port])); - if (ports.get("omp-app") !== 8080 || ports.get("admin") !== 9090) { - fail("cluster-server Service does not expose the fixed public/admin ports"); + if (ports.get("websocket") !== 8080 || ports.get("admin") !== 9090 || ports.size !== 2) { + fail("cluster-server Service does not expose the exact websocket/admin ports"); } return snapshot; } @@ -191,28 +293,36 @@ export function validateDefaultOffRender(documents) { return { clusterOperatorEnabled: false, workloadCount: 0 }; } -export function summarizeClusterSnapshot(snapshot) { - validateClusterSnapshot(snapshot); +export function summarizeClusterSnapshot(snapshot, options = {}) { + validateClusterSnapshot(snapshot, options); const deployments = list(snapshot, "deployments"); - const lease = named(list(snapshot, "leases"), "controller", "controller leader Lease"); const workspaces = list(snapshot, "t4workspaces"); const sessions = list(snapshot, "t4sessions"); const pvcs = list(snapshot, "persistentvolumeclaims"); const allPods = list(snapshot, "pods"); - const pods = allPods.filter(({ metadata }) => metadata?.labels?.["cluster.t4.dev/session"]); + const lease = snapshot.lease; + const capturedAt = new Date(options.now ?? Date.now()).toISOString(); return { schemaVersion: "t4-cluster-readonly-snapshot/1", - observedAt: new Date().toISOString(), + observedAt: capturedAt, deployments: deployments.map(({ metadata, spec, status }) => ({ name: metadata.name, + component: metadata.labels["app.kubernetes.io/component"], replicas: spec.replicas, + maxUnavailable: spec.strategy.rollingUpdate.maxUnavailable, availableReplicas: status.availableReplicas, + generation: metadata.generation, observedGeneration: status.observedGeneration, })), leader: { lease: lease.metadata.name, holderIdentity: lease.spec.holderIdentity, renewTime: lease.spec.renewTime }, - crds: list(snapshot, "customresourcedefinitions").map(({ spec }) => `${spec.names.plural}.${spec.group}/v1alpha1`), - workspaces: workspaces.map(({ metadata, status }) => ({ name: metadata.name, phase: status.phase, pvc: status.pvcRef.name })), - sessions: sessions.map(({ metadata, status }) => ({ name: metadata.name, phase: status.phase })), + crds: list(snapshot, "customresourcedefinitions").map(({ metadata }) => metadata.name), + workspaces: workspaces.map(({ metadata, status }) => ({ name: metadata.name, phase: status.phase, pvc: status.pvcName })), + sessions: sessions.map(({ metadata, spec, status }) => ({ + name: metadata.name, + phase: status.phase, + pod: status.podName, + ...(spec.ci ? { ci: { repositoryId: spec.ci.repositoryId, ref: spec.ci.ref, commit: spec.ci.commit } } : {}), + })), storage: pvcs.map(({ metadata, spec, status }) => ({ name: metadata.name, storageClassName: spec.storageClassName, @@ -220,10 +330,20 @@ export function summarizeClusterSnapshot(snapshot) { phase: status.phase, capacity: status.capacity?.storage ?? "unknown", })), - placements: pods.map(({ metadata, spec }) => ({ name: metadata.name, node: spec.nodeName })), + placements: allPods + .filter(({ metadata }) => metadata?.labels?.["cluster.t4.dev/session"]) + .map(({ metadata, spec }) => ({ name: metadata.name, node: spec.nodeName })), images: allPods.flatMap(({ metadata, status }) => - (status?.containerStatuses ?? []).map(({ name, image, imageID }) => ({ + (status?.containerStatuses ?? []).map(({ name, image, imageID, ready }) => ({ pod: metadata.name, + labels: { + name: metadata.labels?.["app.kubernetes.io/name"], + component: metadata.labels?.["app.kubernetes.io/component"], + session: metadata.labels?.["cluster.t4.dev/session"], + partOf: metadata.labels?.["app.kubernetes.io/part-of"], + }, + phase: status.phase, + ready: ready === true, container: name, image, imageID, From 81add12afa2a064259810f34015f3674bae636f5 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 20 Jul 2026 17:37:58 +0000 Subject: [PATCH 26/98] fix(cluster): enforce negotiated client projections --- .../session-runtime/session-navigation.ts | 30 +++ .../targets/ClusterOperatorSection.tsx | 233 ++++++++++++----- .../src/features/targets/TargetsScreen.tsx | 2 +- .../src/features/targets/cluster-operator.ts | 124 ++++++++- apps/web/src/platform/live-workspace.ts | 10 +- apps/web/src/router.tsx | 11 +- apps/web/test/cluster-mobile.test.ts | 18 +- apps/web/test/cluster-operator.test.tsx | 238 ++++++++++++++++-- apps/web/test/session-navigation.test.ts | 51 +++- e2e/cluster-operator.spec.ts | 190 ++++++++++---- packages/client/src/desktop-runtime.ts | 58 ++++- packages/client/src/omp-client-cursor.ts | 3 + packages/client/src/projection-cache.ts | 24 ++ packages/client/src/projection.ts | 107 +++++++- packages/client/test/client.test.ts | 32 +++ packages/client/test/cluster-operator.test.ts | 15 ++ packages/client/test/desktop-runtime.test.ts | 103 +++++++- packages/client/test/projection.test.ts | 73 ++++++ packages/host-service/src/server.ts | 9 +- packages/host-service/test/appserver.test.ts | 1 + .../test/remote-transport.test.ts | 81 +++++- 21 files changed, 1266 insertions(+), 147 deletions(-) diff --git a/apps/web/src/features/session-runtime/session-navigation.ts b/apps/web/src/features/session-runtime/session-navigation.ts index e6b8e95f..1eab24ce 100644 --- a/apps/web/src/features/session-runtime/session-navigation.ts +++ b/apps/web/src/features/session-runtime/session-navigation.ts @@ -1,7 +1,11 @@ +import type { DesktopRuntimeSnapshot } from "@t4-code/client"; + import type { SessionListView, WorkspaceSession, } from "../../lib/workspace-data.ts"; +import { resolveLiveSession } from "../../platform/live-workspace.ts"; +import type { SessionPreviewSelection } from "../../state/workspace-store.ts"; export type CompletedSessionManagementAction = "archive" | "restore" | "delete"; @@ -43,3 +47,29 @@ export function resolveSessionManagementNavigation( )?.id ?? null; return { view, destinationSessionId, navigate: true }; } + +/** + * Carry a host-advertised GUI preview through navigation. Trust metadata comes + * only from the matching warm projection; a not-yet-attached preview keeps the + * id without inventing authority or opting the user into an unknown profile. + */ +export function previewSelectionForNavigation( + snapshot: DesktopRuntimeSnapshot, + sessionViewId: string, + previewId: string, +): SessionPreviewSelection | null { + const address = resolveLiveSession(snapshot, sessionViewId); + if (address === null) return null; + const preview = [ + ...(snapshot.projection.sessions.get( + `${address.hostId}\u0000${address.sessionId}`, + )?.previews.values() ?? []), + ].find((candidate) => candidate.previewId === previewId); + if (preview === undefined) return { previewId, optIn: false }; + return { + previewId, + optInKind: preview.authority?.kind ?? null, + optInAuthorityId: preview.authority?.id ?? null, + optIn: true, + }; +} diff --git a/apps/web/src/features/targets/ClusterOperatorSection.tsx b/apps/web/src/features/targets/ClusterOperatorSection.tsx index f0046196..d60fe677 100644 --- a/apps/web/src/features/targets/ClusterOperatorSection.tsx +++ b/apps/web/src/features/targets/ClusterOperatorSection.tsx @@ -1,22 +1,37 @@ import type { DesktopRuntimeController, DesktopRuntimeSnapshot } from "@t4-code/client"; -import { CLUSTER_OPERATOR_FEATURE } from "@t4-code/protocol"; +import type { SessionRef } from "@t4-code/protocol"; import { Button, Spinner } from "@t4-code/ui"; import { useMemo, useState } from "react"; import { FIELD_CLASS } from "../settings/controls.tsx"; -import { deriveWorkspaceData } from "../../platform/live-workspace.ts"; +import { deriveWorkspaceData, sessionViewId } from "../../platform/live-workspace.ts"; +import type { WorkspaceSession } from "../../lib/workspace-data.ts"; import { + clusterCiAvailability, + clusterCreationTargets, + clusterGuiAvailability, clusterOperatorAvailability, createClusterSession, createClusterWorkspace, runClusterCi, } from "./cluster-operator.ts"; +const CLUSTER_FIELD_CLASS = `${FIELD_CLASS} min-h-11 sm:min-h-8`; + export interface ClusterOperatorSectionProps { readonly controller: DesktopRuntimeController; readonly snapshot: DesktopRuntimeSnapshot; readonly onOpenSession: (sessionId: string) => void; - readonly onOpenPreview: (sessionId: string) => void; + readonly onOpenPreview: (sessionId: string, previewId: string) => void; +} + +export function clusterSessionMatchesWorkspace( + session: WorkspaceSession, + ref: SessionRef | undefined, + hostId: string, + workspaceId: string, +): boolean { + return String(ref?.hostId ?? "") === hostId && session.cluster?.workspaceId === workspaceId; } export function ClusterOperatorSection({ @@ -26,16 +41,40 @@ export function ClusterOperatorSection({ onOpenPreview, }: ClusterOperatorSectionProps) { const [query, setQuery] = useState(""); + const [creationHostId, setCreationHostId] = useState(""); const [displayName, setDisplayName] = useState(""); const [capacity, setCapacity] = useState("20Gi"); const [sessionTitle, setSessionTitle] = useState>({}); const [pending, setPending] = useState(null); const [message, setMessage] = useState(null); const data = deriveWorkspaceData(snapshot); - const advertised = [...snapshot.hosts.values()].some( - (host) => host.grantedFeatures.includes(CLUSTER_OPERATOR_FEATURE), + const creationTargets = useMemo(() => clusterCreationTargets(snapshot), [snapshot]); + const selectedCreationTarget = creationTargets.find( + (target) => target.hostId === creationHostId, ); + const creationAvailability = + selectedCreationTarget === undefined + ? { + enabled: false, + reason: "Choose an advertised cluster host for creation.", + } + : clusterOperatorAvailability( + snapshot, + selectedCreationTarget.targetId, + selectedCreationTarget.hostId, + "manage", + ); const clusterWorkspaces = data.clusterWorkspaces ?? []; + const sessionRefs = useMemo( + () => + new Map( + [...snapshot.projection.sessionIndex.values()].map((ref) => [ + sessionViewId(String(ref.hostId), String(ref.sessionId)), + ref, + ]), + ), + [snapshot.projection.sessionIndex], + ); const normalizedQuery = query.trim().toLocaleLowerCase(); const workspaces = useMemo( () => @@ -49,13 +88,7 @@ export function ClusterOperatorSection({ [clusterWorkspaces, normalizedQuery], ); - if (snapshot.clusterOperatorEnabled !== true || !advertised) return null; - const firstHost = [...snapshot.hosts.values()].find((host) => - host.grantedFeatures.includes(CLUSTER_OPERATOR_FEATURE), - ); - if (firstHost === undefined) return null; - const targetId = firstHost.targetId; - const availability = clusterOperatorAvailability(snapshot, targetId, "manage"); + if (snapshot.clusterOperatorEnabled !== true || creationTargets.length === 0) return null; const act = async (id: string, operation: () => Promise) => { setPending(id); @@ -78,13 +111,13 @@ export function ClusterOperatorSection({ Cluster workspaces

- Infrastructure, sessions, CI, and GUI state reported by the connected cluster host. + Infrastructure, sessions, CI, and GUI state reported by connected cluster hosts.