diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..1c4b766d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.git +.gitignore +.cluster-ci +.cluster-ci/registry-auth +**/.cluster-ci/registry-auth +artifacts +**/artifacts +node_modules +**/node_modules +.env +.env.* +*.log +coverage +**/coverage +test-results +playwright-report diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb08c40c..527de4df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,7 @@ jobs: timeout-minutes: 5 outputs: continuity: ${{ steps.classify.outputs.continuity }} + cluster: ${{ steps.classify.outputs.cluster }} tooling: ${{ steps.classify.outputs.tooling }} android_debug: ${{ steps.classify.outputs.android_debug }} flutter: ${{ steps.classify.outputs.flutter }} @@ -173,6 +174,70 @@ jobs: if-no-files-found: error retention-days: 14 + cluster: + needs: changes + if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.cluster == 'true' }} + runs-on: ubuntu-24.04 + timeout-minutes: 25 + steps: + - name: Check out source + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Install pnpm + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + with: + version: 11.10.0 + + - name: Install Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 24.13.1 + cache: pnpm + + - name: Install Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.3.14 + + - name: Install Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: "1.25.5" + cache-dependency-path: packages/cluster-operator/go.sum + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Install pinned Helm + shell: bash + run: | + set -euo pipefail + curl -fsSLo /tmp/helm.tar.gz https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz + echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496 /tmp/helm.tar.gz" | sha256sum -c - + tar -xzf /tmp/helm.tar.gz -C /tmp + sudo install -m 0755 /tmp/linux-amd64/helm /usr/local/bin/helm + + - name: Run cluster CI proof contracts + run: pnpm test:cluster:ci + + - name: Run cluster server tests + run: pnpm --filter @t4-code/cluster-server test + + - name: Typecheck cluster server + run: pnpm --filter @t4-code/cluster-server typecheck + + - name: Run cluster protocol contracts + run: | + pnpm --filter @t4-code/host-wire exec bun test test/cluster-operator.test.ts + pnpm --filter @t4-code/host-service exec bun test test/cluster-default-off.test.ts + + - name: Run operator tests + working-directory: packages/cluster-operator + run: go test ./... + + - name: Lint cluster chart + run: helm lint deploy/charts/t4-cluster + tooling: needs: changes if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.tooling == 'true' }} @@ -395,7 +460,7 @@ jobs: verify: name: verify if: ${{ always() }} - needs: [changes, core, legacy-bridge-continuity, tooling, android-debug, flutter, flutter-android, flutter-apple] + needs: [changes, core, legacy-bridge-continuity, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple] runs-on: ubuntu-24.04 timeout-minutes: 5 steps: @@ -405,6 +470,7 @@ jobs: CHANGES_RESULT: ${{ needs.changes.result }} CORE_RESULT: ${{ needs.core.result }} CONTINUITY_RESULT: ${{ needs.legacy-bridge-continuity.result }} + CLUSTER_RESULT: ${{ needs.cluster.result }} TOOLING_RESULT: ${{ needs.tooling.result }} ANDROID_RESULT: ${{ needs.android-debug.result }} FLUTTER_RESULT: ${{ needs.flutter.result }} @@ -416,6 +482,7 @@ jobs: test "$CORE_RESULT" = success for result in \ "$CONTINUITY_RESULT" \ + "$CLUSTER_RESULT" \ "$TOOLING_RESULT" \ "$ANDROID_RESULT" \ "$FLUTTER_RESULT" \ diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 00000000..09724a03 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,914 @@ +when: + - event: + - push + - pull_request + - manual + +clone: + git: + image: mirror.gcr.io/woodpeckerci/plugin-git:2.6.5@sha256:2b34ecbbfd2f318405bd5d2765a750f6b02dae5dfb8707e16c943b9faeef5fc5 + settings: + depth: 50 + partial: false + +steps: + dependencies: + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + commands: + - corepack enable + - pnpm install --frozen-lockfile + - mkdir -p .ci && cp /etc/ssl/certs/ca-certificates.crt .ci/ca-certificates.crt + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 512Mi + ephemeral-storage: 1Gi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 4Gi + + bun-runtime: + depends_on: [dependencies] + image: mirror.gcr.io/oven/bun:1.3.14@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 + commands: + - cp "$(command -v bun)" .ci/bun + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 64Mi + ephemeral-storage: 64Mi + limits: + cpu: 250m + memory: 256Mi + ephemeral-storage: 256Mi + + upstream-core: + depends_on: [cluster-ci-contracts, bun-runtime] + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + commands: + - export PATH="$PWD/.ci:$PATH" + - corepack enable + - pnpm check:release && pnpm check:provenance && pnpm lint && pnpm --filter '!@t4-code/flutter' -r typecheck + - VP_RUN_CONCURRENCY_LIMIT=1 pnpm --filter '!@t4-code/flutter' -r test + - pnpm --filter '!@t4-code/flutter' -r build + - pnpm exec playwright install --with-deps chromium + - pnpm test:e2e + - pnpm test:packaging + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 1Gi + ephemeral-storage: 2Gi + limits: + cpu: "2" + memory: 6Gi + ephemeral-storage: 8Gi + + upstream-tooling: + depends_on: [upstream-core] + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + commands: + - apt-get update && apt-get install -y --no-install-recommends jq && rm -rf /var/lib/apt/lists/* + - corepack enable + - pnpm test:tooling + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 512Mi + ephemeral-storage: 512Mi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 2Gi + + legacy-authority-source: + depends_on: [upstream-tooling] + image: mirror.gcr.io/alpine/git:2.49.1@sha256:c0280cf9572316299b08544065d3bf35db65043d5e3963982ec50647d2746e26 + commands: + - git init .continuity/omp + - git -C .continuity/omp remote add origin https://github.com/lyc-aon/oh-my-pi.git + - git -C .continuity/omp fetch --depth=1 origin 8476f4451ed95c5d5401785d279a93d3c659fac4 + - git -C .continuity/omp checkout --detach FETCH_HEAD + - test "$(git -C .continuity/omp rev-parse HEAD)" = 8476f4451ed95c5d5401785d279a93d3c659fac4 + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 128Mi + ephemeral-storage: 256Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 1Gi + + legacy-authority-build: + depends_on: [legacy-authority-source] + image: mirror.gcr.io/oven/bun:1.3.14@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 + commands: + - cd .continuity/omp && bun install --frozen-lockfile + - cd .continuity/omp && bun run build:native + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 512Mi + ephemeral-storage: 1Gi + limits: + cpu: "2" + memory: 4Gi + ephemeral-storage: 6Gi + + legacy-bridge-continuity: + depends_on: [legacy-authority-build] + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + environment: + T4_OMP_SOURCE_DIR: .continuity/omp + commands: + - corepack enable + - pnpm test:legacy-bridge-continuity + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 512Mi + ephemeral-storage: 512Mi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 2Gi + + android-debug: + depends_on: [legacy-bridge-continuity] + image: cimg/android:2026.03.1-node@sha256:b62bbea69ba6bb28fe165ff913c23bde5220d0305fed7247121fb2919ecffd0e + commands: + - corepack enable + - pnpm --filter @t4-code/mobile check:android:debug + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 1Gi + ephemeral-storage: 2Gi + limits: + cpu: "2" + memory: 5Gi + ephemeral-storage: 8Gi + + cluster-ci-contracts: + depends_on: [dependencies] + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + commands: + - corepack enable + - pnpm test:cluster:ci + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 1Gi + + cluster-server-tests: + depends_on: [cluster-ci-contracts] + image: mirror.gcr.io/oven/bun:1.3.14@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 + environment: + T4_PROOF_OUTPUT_DIR: artifacts/cluster-proof/scenarios + commands: + - export SSL_CERT_FILE="$PWD/.ci/ca-certificates.crt" + - export NODE_EXTRA_CA_CERTS="$SSL_CERT_FILE" + - (cd packages/cluster-server && bun --bun run test) + - (cd packages/cluster-server && bun --bun run typecheck) + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 2Gi + + cluster-wire-tests: + depends_on: [cluster-server-tests, bun-runtime] + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + commands: + - export PATH="$PWD/.ci:$PATH" + - export SSL_CERT_FILE="$PWD/.ci/ca-certificates.crt" + - export NODE_EXTRA_CA_CERTS="$SSL_CERT_FILE" + - (cd packages/host-wire && bun test test/cluster-operator.test.ts) + - (cd packages/host-service && bun test test/cluster-default-off.test.ts) + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 2Gi + + cluster-operator-tests: + depends_on: [cluster-wire-tests] + image: mirror.gcr.io/library/golang:1.25.1-bookworm@sha256:c423747fbd96fd8f0b1102d947f51f9b266060217478e5f9bf86f145969562ee + directory: packages/cluster-operator + commands: + - GOMAXPROCS=1 GOFLAGS=-p=1 go test ./api/... ./controllers/... ./cmd/... + - mkdir -p ../../artifacts/cluster-proof + - CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS=-p=1 go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 2Gi + + cluster-chart-tests: + depends_on: [cluster-operator-tests] + image: mirror.gcr.io/alpine/helm:3.19.0@sha256:aef9b56f64e866207d9591d0abd8f6d767b36aadd12edf68f8a719716d9d29c9 + directory: packages/cluster-operator/charttests + commands: + - helm lint ../../../deploy/charts/t4-cluster + - ../../../artifacts/cluster-proof/chart-contract.test + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 128Mi + ephemeral-storage: 128Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 512Mi + + harbor-auth: + depends_on: [cluster-chart-tests, android-debug] + image: harbor.tailb18de3.ts.net/linkedin-bot/woodpecker-buildkit-deploy-tools@sha256:7936be1edac8ce3c225b148cff0e89b1446fc88185d72379398617f7279ad8cc + commands: + - sh scripts/cluster-ci/load-registry-auth.sh + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-dev-verifier + resources: + requests: + cpu: "0" + memory: 128Mi + ephemeral-storage: 128Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 512Mi + + build-controller: + depends_on: [harbor-auth] + image: mirror.gcr.io/moby/buildkit:v0.30.0-rootless@sha256:d76eb1caecac5733ef7553c1e90a1b21f1bb218cd1142d3553de0747b4a14ba9 + environment: + BUILDKIT_ADDR: tcp://woodpecker-buildkitd.linkedin-ci.svc.cluster.local:1234 + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + commands: + - scripts/cluster-ci/build-image.sh controller t4-cluster-operator cluster/images/controller/Dockerfile + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 1Gi + + sbom-controller: + depends_on: [build-session-runtime] + image: anchore/syft:v1.33.0-debug@sha256:421a1367ee81b35b094defc7c1c42774e9f0c2f818e278848175015987c4a034 + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + PATH: "/busybox:/" + entrypoint: + - /busybox/sh + - -c + - echo "$CI_SCRIPT" | /busybox/base64 -d | /busybox/sh -e + commands: + - sh scripts/cluster-ci/capture-image-evidence.sh sbom controller t4-cluster-operator + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 2Gi + + vulnerability-controller: + depends_on: [sbom-controller] + image: aquasec/trivy:0.66.0@sha256:086971aaf400beebd94e8300fd8ea623774419597169156cec56eec5b00dfb1e + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + commands: + - scripts/cluster-ci/capture-image-evidence.sh vulnerability controller t4-cluster-operator + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 512Mi + ephemeral-storage: 1Gi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 4Gi + + provenance-controller: + depends_on: [vulnerability-controller] + image: gcr.io/projectsigstore/cosign:v2.6.0-dev@sha256:927acebad5fd845802b560f2a1b2cfa7c7170a5056511d2cae137a5e4fc39a4c + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + entrypoint: + - /busybox/sh + - -c + - echo "$CI_SCRIPT" | /busybox/base64 -d | /busybox/sh -e + commands: + - sh scripts/cluster-ci/capture-image-evidence.sh provenance controller t4-cluster-operator + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 128Mi + ephemeral-storage: 128Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 1Gi + + build-cluster-server: + depends_on: [build-controller] + image: mirror.gcr.io/moby/buildkit:v0.30.0-rootless@sha256:d76eb1caecac5733ef7553c1e90a1b21f1bb218cd1142d3553de0747b4a14ba9 + environment: + BUILDKIT_ADDR: tcp://woodpecker-buildkitd.linkedin-ci.svc.cluster.local:1234 + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + commands: + - scripts/cluster-ci/build-image.sh cluster-server t4-cluster-server cluster/images/cluster-server/Dockerfile + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 1Gi + + sbom-cluster-server: + depends_on: [provenance-controller] + image: anchore/syft:v1.33.0-debug@sha256:421a1367ee81b35b094defc7c1c42774e9f0c2f818e278848175015987c4a034 + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + PATH: "/busybox:/" + entrypoint: + - /busybox/sh + - -c + - echo "$CI_SCRIPT" | /busybox/base64 -d | /busybox/sh -e + commands: + - sh scripts/cluster-ci/capture-image-evidence.sh sbom cluster-server t4-cluster-server + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 2Gi + + vulnerability-cluster-server: + depends_on: [sbom-cluster-server] + image: aquasec/trivy:0.66.0@sha256:086971aaf400beebd94e8300fd8ea623774419597169156cec56eec5b00dfb1e + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + commands: + - scripts/cluster-ci/capture-image-evidence.sh vulnerability cluster-server t4-cluster-server + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 512Mi + ephemeral-storage: 1Gi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 4Gi + + provenance-cluster-server: + depends_on: [vulnerability-cluster-server] + image: gcr.io/projectsigstore/cosign:v2.6.0-dev@sha256:927acebad5fd845802b560f2a1b2cfa7c7170a5056511d2cae137a5e4fc39a4c + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + entrypoint: + - /busybox/sh + - -c + - echo "$CI_SCRIPT" | /busybox/base64 -d | /busybox/sh -e + commands: + - sh scripts/cluster-ci/capture-image-evidence.sh provenance cluster-server t4-cluster-server + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 128Mi + ephemeral-storage: 128Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 1Gi + + build-session-runtime: + depends_on: [build-cluster-server] + image: mirror.gcr.io/moby/buildkit:v0.30.0-rootless@sha256:d76eb1caecac5733ef7553c1e90a1b21f1bb218cd1142d3553de0747b4a14ba9 + environment: + BUILDKIT_ADDR: tcp://woodpecker-buildkitd.linkedin-ci.svc.cluster.local:1234 + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + commands: + - scripts/cluster-ci/build-image.sh session-runtime t4-session-runtime cluster/images/session-runtime/Dockerfile + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 1Gi + + sbom-session-runtime: + depends_on: [provenance-cluster-server] + image: anchore/syft:v1.33.0-debug@sha256:421a1367ee81b35b094defc7c1c42774e9f0c2f818e278848175015987c4a034 + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + PATH: "/busybox:/" + entrypoint: + - /busybox/sh + - -c + - echo "$CI_SCRIPT" | /busybox/base64 -d | /busybox/sh -e + commands: + - sh scripts/cluster-ci/capture-image-evidence.sh sbom session-runtime t4-session-runtime + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 2Gi + + vulnerability-session-runtime: + depends_on: [sbom-session-runtime] + image: aquasec/trivy:0.66.0@sha256:086971aaf400beebd94e8300fd8ea623774419597169156cec56eec5b00dfb1e + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + commands: + - scripts/cluster-ci/capture-image-evidence.sh vulnerability session-runtime t4-session-runtime + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 512Mi + ephemeral-storage: 1Gi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 4Gi + + provenance-session-runtime: + depends_on: [vulnerability-session-runtime] + image: gcr.io/projectsigstore/cosign:v2.6.0-dev@sha256:927acebad5fd845802b560f2a1b2cfa7c7170a5056511d2cae137a5e4fc39a4c + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + entrypoint: + - /busybox/sh + - -c + - echo "$CI_SCRIPT" | /busybox/base64 -d | /busybox/sh -e + commands: + - sh scripts/cluster-ci/capture-image-evidence.sh provenance session-runtime t4-session-runtime + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 128Mi + ephemeral-storage: 128Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 1Gi + + promote-images: + depends_on: [image-publication-manifest] + image: ghcr.io/oras-project/oras:v1.3.0@sha256:6ce045ce069a89934d6666b8b49f9c4c0145201bd6de6dbe2aee267814c55468 + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + commands: + - scripts/cluster-ci/promote-images.sh + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 128Mi + ephemeral-storage: 128Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 1Gi + + image-publication-manifest: + depends_on: [provenance-session-runtime] + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + commands: + - node scripts/cluster-ci/assemble-image-manifest.mjs + - node scripts/cluster-ci/validate-proof.mjs artifacts/cluster-proof/image-publication.json --images-only + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 128Mi + ephemeral-storage: 128Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 1Gi + + publish-image-evidence: + depends_on: [promote-images] + image: ghcr.io/oras-project/oras:v1.3.0@sha256:6ce045ce069a89934d6666b8b49f9c4c0145201bd6de6dbe2aee267814c55468 + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + commands: + - scripts/cluster-ci/publish-artifact.sh images + when: + - event: push + branch: main + - event: manual + branch: [main, agent/t4-cluster-operator] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 128Mi + ephemeral-storage: 128Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 1Gi + + cleanup-image-registry-auth: + depends_on: [publish-image-evidence] + image: mirror.gcr.io/library/busybox:1.37.0-musl@sha256:222ad6d973c0d198014546a65cd02c5fdedcc172123c5b4c2bf0af636550bd94 + commands: + - rm -rf .cluster-ci/registry-auth + when: + - event: push + branch: main + status: [success, failure] + - event: manual + branch: [main, agent/t4-cluster-operator] + status: [success, failure] + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 16Mi + ephemeral-storage: 16Mi + limits: + cpu: 100m + memory: 64Mi + ephemeral-storage: 64Mi + + live-cluster-observations: + depends_on: [cleanup-image-registry-auth] + image: harbor.tailb18de3.ts.net/linkedin-bot/woodpecker-buildkit-deploy-tools@sha256:7936be1edac8ce3c225b148cff0e89b1446fc88185d72379398617f7279ad8cc + environment: + T4_CLUSTER_NAMESPACE: t4-development + commands: + - node scripts/cluster-ci/collect-readonly-cluster.mjs + when: + - event: manual + branch: [main, agent/t4-cluster-operator] + evaluate: 'T4_CLUSTER_PROOF_ENABLED == "true"' + backend_options: + kubernetes: + serviceAccountName: woodpecker-dev-verifier + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 1Gi + + live-frame-proof: + depends_on: [live-cluster-observations] + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + environment: + T4_CLUSTER_BASE_URL: https://t4-dev.tailb18de3.ts.net/ + commands: + - node scripts/cluster-ci/capture-redacted-frames.mjs + when: + - event: manual + branch: [main, agent/t4-cluster-operator] + evaluate: 'T4_CLUSTER_PROOF_ENABLED == "true"' + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 1Gi + ephemeral-storage: 1Gi + limits: + cpu: "2" + memory: 4Gi + ephemeral-storage: 6Gi + + live-proof-assembly: + depends_on: [live-frame-proof] + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + commands: + - node scripts/cluster-ci/assemble-proof.mjs + - node scripts/cluster-ci/validate-proof.mjs artifacts/cluster-proof/manifest.json + when: + - event: manual + branch: [main, agent/t4-cluster-operator] + evaluate: 'T4_CLUSTER_PROOF_ENABLED == "true"' + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 1Gi + + harbor-auth-live-proof: + depends_on: [live-proof-assembly] + image: harbor.tailb18de3.ts.net/linkedin-bot/woodpecker-buildkit-deploy-tools@sha256:7936be1edac8ce3c225b148cff0e89b1446fc88185d72379398617f7279ad8cc + commands: + - sh scripts/cluster-ci/load-registry-auth.sh + when: + - event: manual + branch: [main, agent/t4-cluster-operator] + evaluate: 'T4_CLUSTER_PROOF_ENABLED == "true"' + backend_options: + kubernetes: + serviceAccountName: woodpecker-dev-verifier + resources: + requests: + cpu: "0" + memory: 128Mi + ephemeral-storage: 128Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 512Mi + + publish-live-proof: + depends_on: [harbor-auth-live-proof] + image: ghcr.io/oras-project/oras:v1.3.0@sha256:6ce045ce069a89934d6666b8b49f9c4c0145201bd6de6dbe2aee267814c55468 + environment: + HARBOR_REGISTRY: harbor.tailb18de3.ts.net + HARBOR_PROJECT: linkedin-bot + commands: + - scripts/cluster-ci/publish-artifact.sh proof + when: + - event: manual + branch: [main, agent/t4-cluster-operator] + evaluate: 'T4_CLUSTER_PROOF_ENABLED == "true"' + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 128Mi + ephemeral-storage: 128Mi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 1Gi + + cleanup-live-registry-auth: + depends_on: [publish-live-proof] + image: mirror.gcr.io/library/busybox:1.37.0-musl@sha256:222ad6d973c0d198014546a65cd02c5fdedcc172123c5b4c2bf0af636550bd94 + commands: + - rm -rf .cluster-ci/registry-auth + when: + - event: manual + branch: [main, agent/t4-cluster-operator] + status: [success, failure] + evaluate: 'T4_CLUSTER_PROOF_ENABLED == "true"' + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 16Mi + ephemeral-storage: 16Mi + limits: + cpu: 100m + memory: 64Mi + ephemeral-storage: 64Mi + + verify: + depends_on: + - cluster-chart-tests + image: mirror.gcr.io/library/alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 + commands: + - test -f package.json + when: + - event: + - push + - pull_request + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 32Mi + ephemeral-storage: 16Mi + limits: + cpu: 100m + memory: 64Mi + ephemeral-storage: 64Mi diff --git a/apps/desktop/src/cluster-operator-flag.ts b/apps/desktop/src/cluster-operator-flag.ts new file mode 100644 index 00000000..87558820 --- /dev/null +++ b/apps/desktop/src/cluster-operator-flag.ts @@ -0,0 +1,7 @@ +export const DESKTOP_CLUSTER_OPERATOR_SWITCH = "--t4-cluster-operator"; + +export function desktopClusterOperatorEnabled( + arguments_: readonly string[] = process.argv, +): boolean { + return arguments_.includes(DESKTOP_CLUSTER_OPERATOR_SWITCH); +} diff --git a/apps/desktop/src/lifecycle.ts b/apps/desktop/src/lifecycle.ts index 52d03bdc..1aedbeab 100644 --- a/apps/desktop/src/lifecycle.ts +++ b/apps/desktop/src/lifecycle.ts @@ -6,7 +6,11 @@ import { parsePairDeepLink, PendingPairQueue, type PendingPair } from "@t4-code/ import { decodeLocalProfileId, type ConnectionStateEvent, type ServiceAvailabilityIssue } from "@t4-code/protocol/desktop-ipc"; import type { ServiceManager } from "@t4-code/service-manager"; import type { RemoteTargetRegistry } from "./remote-runtime/registry.ts"; -import { createDesktopWindow, type DesktopWindowHandle } from "./window.ts"; +import { + createDesktopWindow, + type DesktopWindowHandle, + type DesktopWindowOptions, +} from "./window.ts"; import { DesktopIpcRegistry, runtimeError, type IpcRuntime } from "./ipc.ts"; import { BrowserRuntime, type BrowserRuntimeOptions } from "./browser-runtime.ts"; import { @@ -65,7 +69,8 @@ export function appserverLogsDirectory( export interface DesktopLifecycleOptions { readonly app?: typeof app; readonly getAllWindows?: () => readonly BrowserWindow[]; - readonly createWindow?: () => DesktopWindowHandle; + readonly createWindow?: (options?: DesktopWindowOptions) => DesktopWindowHandle; + readonly clusterOperatorEnabled?: boolean; readonly createBrowserRuntime?: (options: BrowserRuntimeOptions) => BrowserRuntime; readonly createIpcRegistry?: (runtime: IpcRuntime) => DesktopIpcRegistry; readonly loadIdentity?: () => DeviceIdentity; @@ -99,7 +104,8 @@ export class DesktopLifecycle { private readonly pendingPairs = new PendingPairQueue(8); private readonly electronApp: typeof app; private readonly allWindows: () => readonly BrowserWindow[]; - private readonly windowFactory: () => DesktopWindowHandle; + private readonly windowFactory: (options?: DesktopWindowOptions) => DesktopWindowHandle; + private readonly clusterOperatorEnabled: boolean; private readonly ipcFactory: (runtime: IpcRuntime) => DesktopIpcRegistry; private readonly browserRuntimeFactory: (options: BrowserRuntimeOptions) => BrowserRuntime; private readonly identityFactory: () => DeviceIdentity; @@ -151,6 +157,7 @@ export class DesktopLifecycle { constructor(options: DesktopLifecycleOptions = {}) { this.electronApp = options.app ?? app; + this.clusterOperatorEnabled = options.clusterOperatorEnabled === true; this.browserRuntimeFactory = options.createBrowserRuntime ?? ((runtimeOptions) => new BrowserRuntime(runtimeOptions)); this.allWindows = options.getAllWindows ?? (() => BrowserWindow.getAllWindows()); @@ -272,6 +279,7 @@ export class DesktopLifecycle { registry: remoteRegistry, localProfiles: () => this.localProfileRegistry?.list() ?? Promise.resolve([]), ...(credentials === undefined ? {} : { credentials }), + ...(this.clusterOperatorEnabled ? { clusterOperatorEnabled: true } : {}), deviceId: identity.deviceId, deviceName: identity.deviceName, events: { @@ -293,7 +301,11 @@ export class DesktopLifecycle { this.speechService = this.speechServiceFactory({ discoverExecutable: () => this.discoverServiceExecutable(), }); - this.bindWindow(this.windowFactory()); + this.bindWindow( + this.windowFactory( + this.clusterOperatorEnabled ? { clusterOperatorEnabled: true } : undefined, + ), + ); await this.acquireServiceManager(); if (this.stopping) return; await this.profileRuntime.startAutomaticProfiles((profileId, error) => { @@ -307,7 +319,12 @@ export class DesktopLifecycle { this.electronApp.on("before-quit", this.beforeQuitHandler); this.electronApp.on("activate", () => { if (this.stopping) return; - if (this.allWindows().length === 0) this.bindWindow(this.windowFactory()); + if (this.allWindows().length === 0) + this.bindWindow( + this.windowFactory( + this.clusterOperatorEnabled ? { clusterOperatorEnabled: true } : undefined, + ), + ); }); } @@ -696,7 +713,11 @@ export class DesktopLifecycle { private openUpdatesFromMenu(): void { if (this.stopping) return; if (this.mainWindow === undefined && this.manager !== undefined) { - this.bindWindow(this.windowFactory()); + this.bindWindow( + this.windowFactory( + this.clusterOperatorEnabled ? { clusterOperatorEnabled: true } : undefined, + ), + ); } this.mainWindow?.show(); this.mainWindow?.focus(); diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index da65cf7d..0dca08aa 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -1,5 +1,6 @@ import { app } from "electron"; import { DesktopLifecycle } from "./lifecycle.ts"; +import { desktopClusterOperatorEnabled } from "./cluster-operator-flag.ts"; const MAX_RUNTIME_REJECTION_REPORTS = 10; const MAX_FAILURE_MESSAGE_LENGTH = 1_024; @@ -124,7 +125,9 @@ export function bootstrapDesktopMain(options: MainRuntimeOptions): Promise } } -const lifecycle = new DesktopLifecycle(); +const lifecycle = new DesktopLifecycle({ + clusterOperatorEnabled: desktopClusterOperatorEnabled(), +}); void bootstrapDesktopMain({ app: app as unknown as DesktopApp, lifecycle, diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index e8636f3b..f153866a 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -49,10 +49,12 @@ import { } from "@t4-code/protocol/desktop-ipc"; import { BROWSER_CHANNELS, decodeBrowserCall, decodeBrowserEvent, decodeBrowserResult, type BrowserCall, type BrowserCallResult, type BrowserEvent } from "@t4-code/protocol/browser-ipc"; import type { BrowserShellPort } from "@t4-code/client"; +import { desktopClusterOperatorEnabled } from "./cluster-operator-flag.ts"; export interface OmpShellBridge { readonly kind: "desktop"; readonly platform: "linux" | "darwin"; + readonly clusterOperatorEnabled?: true; readonly bootstrap: () => Promise; readonly confirm: (request: ConfirmRequest) => Promise; readonly terminalInput: (request: TerminalInputRequest) => Promise; @@ -169,6 +171,7 @@ function createBrowserBridge(): BrowserShellPort { const bridge: OmpShellBridge = { kind: "desktop", platform: process.platform === "darwin" ? "darwin" : "linux", + ...(desktopClusterOperatorEnabled() ? { clusterOperatorEnabled: true as const } : {}), speakText: (request) => invoke("omp:speech:speak", request), stopSpeaking: () => invoke("omp:speech:stop", {}), bootstrap: () => invoke("omp:bootstrap", {}), diff --git a/apps/desktop/src/target-manager.ts b/apps/desktop/src/target-manager.ts index 5a00ca28..ceb644dc 100644 --- a/apps/desktop/src/target-manager.ts +++ b/apps/desktop/src/target-manager.ts @@ -1,4 +1,4 @@ -import { createOmpClient, isConfirmationDecisionConsumed, OmpClientError, type CommandIntent, type CursorStore, type OmpClient, type OmpPairOk, type PublicOmpServerEvent } from "@t4-code/client"; +import { clusterOperatorRequestedCapabilities, clusterOperatorRequestedFeatures, createOmpClient, isConfirmationDecisionConsumed, OmpClientError, type CommandIntent, type CursorStore, type OmpClient, type OmpPairOk, type PublicOmpServerEvent } from "@t4-code/client"; import { commandResultError, type CommandResult, type ConnectionStateEvent, type RuntimeErrorEvent } from "@t4-code/protocol/desktop-ipc"; import type { ConfirmRequest, ConfirmResult, TerminalCloseRequest, TerminalInputRequest, TerminalResizeRequest, TerminalResult } from "@t4-code/protocol/desktop-ipc"; import { ADDITIVE_FEATURES, DEVICE_CAPABILITIES, type DeviceCapability } from "@t4-code/protocol"; @@ -8,11 +8,6 @@ import { validateRemoteTarget, type CredentialStore, type PublicRemoteTarget, ty import { DEFAULT_LOCAL_PROFILE, localTargetId, type LocalProfileRecord } from "./local-profiles.ts"; const DEFAULT_CAPABILITIES: readonly DeviceCapability[] = Object.freeze([...DEVICE_CAPABILITIES]); const REQUESTED_FEATURES: readonly string[] = ADDITIVE_FEATURES; -const COMPATIBILITY_FEATURES: readonly string[] = Object.freeze( - REQUESTED_FEATURES.filter( - (feature) => feature !== "prompt.images" && feature !== "transcript.images", - ), -); export type DesktopTargetState = "disconnected" | "connecting" | "connected" | "pairing-required" | "error"; export interface PublicDesktopTarget { @@ -44,6 +39,7 @@ export interface TargetManagerOptions { readonly capabilities?: readonly DeviceCapability[]; readonly deviceId?: string; readonly deviceName?: string; + readonly clusterOperatorEnabled?: boolean; } type Transport = UnixWebSocketTransport | RemoteWebSocketTransport; interface Runtime { @@ -99,6 +95,9 @@ export class DesktopTargetManager { private readonly deviceId: string; private readonly deviceName: string; private readonly capabilities: readonly DeviceCapability[]; + private readonly clusterOperatorEnabled: boolean; + private readonly requestedFeatures: readonly string[]; + private readonly compatibilityRequestedFeatures: readonly string[]; private readonly connectAttempts = new Map }>(); private readonly registryQueue = { tail: Promise.resolve() }; private readonly targetQueues = new Map }>(); @@ -127,7 +126,19 @@ export class DesktopTargetManager { this.registry = options.registry; this.deviceId = options.deviceId ?? "desktop"; this.credentials = options.credentials; - this.capabilities = Object.freeze([...(options.capabilities ?? DEFAULT_CAPABILITIES)]); + this.clusterOperatorEnabled = options.clusterOperatorEnabled === true; + this.capabilities = this.effectiveCapabilities( + options.capabilities ?? DEFAULT_CAPABILITIES, + ) as readonly DeviceCapability[]; + this.requestedFeatures = clusterOperatorRequestedFeatures( + REQUESTED_FEATURES, + this.clusterOperatorEnabled, + ); + this.compatibilityRequestedFeatures = Object.freeze( + this.requestedFeatures.filter( + (feature) => feature !== "prompt.images" && feature !== "transcript.images", + ), + ); this.deviceName = options.deviceName ?? "T4 Code Desktop"; } @@ -190,8 +201,12 @@ export class DesktopTargetManager { return enqueueTarget(this.targetQueues, targetId, async () => { if (localProfileId(targetId) === undefined) { const remote = await this.remoteTarget(targetId); + const effectiveCapabilities = this.effectiveCapabilities(remote.requestedCapabilities); const current = this.runtimes.get(targetId); - if (current !== undefined && !sameCapabilities(current.requestedCapabilities, remote.requestedCapabilities)) { + if ( + current !== undefined && + !sameCapabilities(current.requestedCapabilities, effectiveCapabilities) + ) { await this.closeRuntime(targetId); const attempt = await this.connectNow(targetId); await attempt.result; @@ -321,6 +336,12 @@ export class DesktopTargetManager { this.targetQueues.clear(); } + private effectiveCapabilities(capabilities: readonly string[]): readonly string[] { + return Object.freeze([ + ...clusterOperatorRequestedCapabilities(capabilities, this.clusterOperatorEnabled), + ]); + } + private async remoteTarget(targetId: string): Promise { if (this.registry === undefined) throw new Error("target not found"); const target = await this.registry.get(targetId); @@ -331,7 +352,9 @@ export class DesktopTargetManager { if (this.closed) throw new Error("target manager is closed"); const local = await this.localProfile(targetId); const remote = local === undefined ? await this.remoteTarget(targetId) : undefined; - const requestedCapabilities = Object.freeze([...(local !== undefined ? this.capabilities : remote!.requestedCapabilities)]); + const requestedCapabilities = this.effectiveCapabilities( + local !== undefined ? this.capabilities : remote!.requestedCapabilities, + ); const existing = this.runtimes.get(targetId); if (existing !== undefined && sameCapabilities(existing.requestedCapabilities, requestedCapabilities)) { if (existing.client.state === "ready") { @@ -381,8 +404,8 @@ export class DesktopTargetManager { }), cursorStore: this.cursorStoreFactory(targetId), capabilities: requestedCapabilities, - requestedFeatures: REQUESTED_FEATURES, - compatibilityRequestedFeatures: COMPATIBILITY_FEATURES, + requestedFeatures: this.requestedFeatures, + compatibilityRequestedFeatures: this.compatibilityRequestedFeatures, client: { name: "T4 Code", version: "0.1.30", build: "desktop", platform: process.platform }, reconnect: { baseMs: 250, maxMs: 10_000 }, }; diff --git a/apps/desktop/src/window.ts b/apps/desktop/src/window.ts index 2300c35e..afc22442 100644 --- a/apps/desktop/src/window.ts +++ b/apps/desktop/src/window.ts @@ -7,6 +7,7 @@ import { rendererUrl, type TrustedRenderer, } from "./security.ts"; +import { DESKTOP_CLUSTER_OPERATOR_SWITCH } from "./cluster-operator-flag.ts"; export const WINDOW_MIN_WIDTH = 840; export const WINDOW_MIN_HEIGHT = 620; @@ -17,6 +18,7 @@ export interface DesktopWindowOptions { readonly devServerUrl?: string; readonly webRoot?: string; readonly isPackaged?: boolean; + readonly clusterOperatorEnabled?: boolean; } export interface DesktopWindowHandle { @@ -49,6 +51,9 @@ export function createDesktopWindow(options: DesktopWindowOptions = {}): Desktop : { titleBarStyle: "default" as const }), webPreferences: { preload: join(__dirname, "preload.cjs"), + ...(options.clusterOperatorEnabled + ? { additionalArguments: [DESKTOP_CLUSTER_OPERATOR_SWITCH] } + : {}), contextIsolation: true, sandbox: true, nodeIntegration: false, diff --git a/apps/desktop/test/cluster-operator-flag.test.ts b/apps/desktop/test/cluster-operator-flag.test.ts new file mode 100644 index 00000000..12e97278 --- /dev/null +++ b/apps/desktop/test/cluster-operator-flag.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; +import { + DESKTOP_CLUSTER_OPERATOR_SWITCH, + desktopClusterOperatorEnabled, +} from "../src/cluster-operator-flag.ts"; + +describe("desktop cluster operator feature switch", () => { + it("is default-off and accepts only the exact explicit switch", () => { + expect(desktopClusterOperatorEnabled([])).toBe(false); + expect(desktopClusterOperatorEnabled([`${DESKTOP_CLUSTER_OPERATOR_SWITCH}=true`])).toBe(false); + expect(desktopClusterOperatorEnabled([`prefix${DESKTOP_CLUSTER_OPERATOR_SWITCH}`])).toBe(false); + expect(desktopClusterOperatorEnabled([DESKTOP_CLUSTER_OPERATOR_SWITCH])).toBe(true); + }); +}); diff --git a/apps/desktop/test/lifecycle-runtime.test.ts b/apps/desktop/test/lifecycle-runtime.test.ts index c7a6abaa..a7ef98d9 100644 --- a/apps/desktop/test/lifecycle-runtime.test.ts +++ b/apps/desktop/test/lifecycle-runtime.test.ts @@ -144,6 +144,7 @@ function setup( readonly createServiceManager?: NonNullable; readonly createProjectionCache?: NonNullable; readonly createBrowserRuntime?: NonNullable; + readonly clusterOperatorEnabled?: boolean; } = {}, ) { const app = new FakeApp(); @@ -164,6 +165,7 @@ function setup( let updateScheduleCount = 0; let updateDisposeCount = 0; let menuOptions: ApplicationMenuOptions | undefined; + let windowOptions: { readonly clusterOperatorEnabled?: boolean } | undefined; let localProfileState: LocalProfileRegistryState = { version: 1, records: [DEFAULT_LOCAL_PROFILE], @@ -208,7 +210,8 @@ function setup( const lifecycle = new DesktopLifecycle({ app: app as never, getAllWindows: () => windows.filter((window) => !window.destroyed) as never, - createWindow: () => { + createWindow: (options?: { readonly clusterOperatorEnabled?: boolean }) => { + windowOptions = options; const next = new FakeWindow(); windows.push(next); return { @@ -230,6 +233,9 @@ function setup( ...(overrides.createProjectionCache === undefined ? {} : { createProjectionCache: overrides.createProjectionCache }), + ...(overrides.clusterOperatorEnabled === undefined + ? {} + : { clusterOperatorEnabled: overrides.clusterOperatorEnabled }), discoverExecutable: overrides.discoverExecutable ?? (serviceManager === undefined ? async () => undefined : async () => "/opt/omp/bin/omp"), @@ -278,10 +284,26 @@ function setup( get menuOptions() { return menuOptions; }, + get windowOptions() { + return windowOptions; + }, }; } describe("desktop Electron lifecycle", () => { + it("propagates explicit cluster operator opt-in to main and renderer runtimes", async () => { + const disabled = setup(); + await disabled.lifecycle.start(); + expect(disabled.managerOptions?.clusterOperatorEnabled).toBeUndefined(); + expect(disabled.windowOptions?.clusterOperatorEnabled).not.toBe(true); + await disabled.lifecycle.stop(); + + const enabled = setup(undefined, async () => true, { clusterOperatorEnabled: true }); + await enabled.lifecycle.start(); + expect(enabled.managerOptions?.clusterOperatorEnabled).toBe(true); + expect(enabled.windowOptions?.clusterOperatorEnabled).toBe(true); + await enabled.lifecycle.stop(); + }); it("queues initial argv, second-instance, and open-url links until renderer load", async () => { const original = [...process.argv]; process.argv.push("t4-code://pair/argv-host/123456"); diff --git a/apps/desktop/test/speech.test.ts b/apps/desktop/test/speech.test.ts index 320f2607..0a66c1b3 100644 --- a/apps/desktop/test/speech.test.ts +++ b/apps/desktop/test/speech.test.ts @@ -3,25 +3,25 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { EventEmitter } from "node:events"; import { describe, expect, it } from "vite-plus/test"; -import type { ChildProcess } from "node:child_process"; +import type { ChildProcess, spawn } from "node:child_process"; import { createDesktopSpeechService } from "../src/speech.ts"; class FakeChild extends EventEmitter { pid = undefined; exitCode: number | null = null; signalCode: NodeJS.Signals | null = null; killed = false; kill(signal?: NodeJS.Signals): boolean { this.killed = true; this.signalCode = signal ?? "SIGTERM"; this.emit("exit", null, this.signalCode); return true; } } -const fakeSpawn = (child: FakeChild, args: (value: string, next: readonly string[], options?: unknown) => void = () => undefined) => ((_: string, next: readonly string[], options?: unknown) => { args(_, next, options); return child as unknown as ChildProcess; }) as typeof import("node:child_process").spawn; +const fakeSpawn = (child: FakeChild, args: (value: string, next: readonly string[], options?: unknown) => void = () => undefined) => ((_: string, next: readonly string[], options?: unknown) => { args(_, next, options); return child as unknown as ChildProcess; }) as typeof spawn; describe("desktop read-aloud process bridge", () => { it("uses a private temp file and keeps text out of argv", async () => { - const root = await mkdtemp(join(tmpdir(), "t4-speech-test-")); const child = new FakeChild(); let argv: readonly string[] = []; - const service = createDesktopSpeechService({ discoverExecutable: async () => "/opt/omp", makeTempDirectory: async () => mkdtemp(join(root, "run-")), spawn: fakeSpawn(child, (_, next, options) => { argv = next; expect((options as { shell?: boolean }).shell).toBe(false); }) }); - const speaking = service.speakText({ text: "private sentence" }); await new Promise((resolve) => setTimeout(resolve, 50)); const file = argv[2] as string; - expect(argv[0]).toBe("say"); expect(argv[1]).toBe("--file"); expect((argv[2] as string).endsWith("/speech.txt")).toBe(true); expect(argv.join(" ")).not.toContain("private sentence"); expect((await stat(file)).mode & 0o777).toBe(0o600); expect(await readFile(file, "utf8")).toBe("private sentence"); + const root = await mkdtemp(join(tmpdir(), "t4-speech-test-")); const child = new FakeChild(); let argv: readonly string[] = []; let spawnOptions: unknown; const spawned = Promise.withResolvers(); + const service = createDesktopSpeechService({ discoverExecutable: async () => "/opt/omp", makeTempDirectory: async () => mkdtemp(join(root, "run-")), spawn: fakeSpawn(child, (_, next, options) => { argv = next; spawnOptions = options; spawned.resolve(); }) }); + const speaking = service.speakText({ text: "private sentence" }); await spawned.promise; const file = argv[2] as string; + expect((spawnOptions as { shell?: boolean }).shell).toBe(false); expect(argv[0]).toBe("say"); expect(argv[1]).toBe("--file"); expect((argv[2] as string).endsWith("/speech.txt")).toBe(true); expect(argv.join(" ")).not.toContain("private sentence"); expect((await stat(file)).mode & 0o777).toBe(0o600); expect(await readFile(file, "utf8")).toBe("private sentence"); child.emit("exit", 0, null); expect(await speaking).toEqual({ accepted: true }); await expect(stat(file)).rejects.toThrow(); await service.dispose(); await rm(root, { recursive: true, force: true }); }); it("replaces and cancels active speech, settling both requests", async () => { - const root = await mkdtemp(join(tmpdir(), "t4-speech-test-")); const children: FakeChild[] = []; - const service = createDesktopSpeechService({ discoverExecutable: async () => "/opt/omp", makeTempDirectory: async () => mkdtemp(join(root, "run-")), spawn: ((_: string, _args: readonly string[]) => { const child = new FakeChild(); children.push(child); return child as unknown as ChildProcess; }) as typeof import("node:child_process").spawn }); - const first = service.speakText({ text: "first" }); await new Promise((resolve) => setTimeout(resolve, 50)); const second = service.speakText({ text: "second" }); await new Promise((resolve) => setTimeout(resolve, 50)); expect(children).toHaveLength(2); expect(children[0]?.killed).toBe(true); await service.stopSpeaking(); expect(children[1]?.killed).toBe(true); expect(await first).toMatchObject({ accepted: false }); expect(await second).toMatchObject({ accepted: false }); await service.dispose(); await rm(root, { recursive: true, force: true }); + const root = await mkdtemp(join(tmpdir(), "t4-speech-test-")); const children: FakeChild[] = []; const firstSpawned = Promise.withResolvers(); const secondSpawned = Promise.withResolvers(); + const service = createDesktopSpeechService({ discoverExecutable: async () => "/opt/omp", makeTempDirectory: async () => mkdtemp(join(root, "run-")), spawn: ((_: string, _args: readonly string[]) => { const child = new FakeChild(); children.push(child); (children.length === 1 ? firstSpawned : secondSpawned).resolve(); return child as unknown as ChildProcess; }) as typeof spawn }); + const first = service.speakText({ text: "first" }); await firstSpawned.promise; const second = service.speakText({ text: "second" }); await secondSpawned.promise; expect(children).toHaveLength(2); expect(children[0]?.killed).toBe(true); await service.stopSpeaking(); expect(children[1]?.killed).toBe(true); expect(await first).toMatchObject({ accepted: false }); expect(await second).toMatchObject({ accepted: false }); await service.dispose(); await rm(root, { recursive: true, force: true }); }); it("settles and cleans up on process exit", async () => { - const root = await mkdtemp(join(tmpdir(), "t4-speech-test-")); const child = new FakeChild(); const service = createDesktopSpeechService({ discoverExecutable: async () => "/opt/omp", makeTempDirectory: async () => mkdtemp(join(root, "run-")), spawn: fakeSpawn(child) }); - const speaking = service.speakText({ text: "exit cleanup" }); await new Promise((resolve) => setTimeout(resolve, 50)); child.emit("exit", 0, null); expect(await speaking).toEqual({ accepted: true }); await service.dispose(); await rm(root, { recursive: true, force: true }); + const root = await mkdtemp(join(tmpdir(), "t4-speech-test-")); const child = new FakeChild(); const spawned = Promise.withResolvers(); const service = createDesktopSpeechService({ discoverExecutable: async () => "/opt/omp", makeTempDirectory: async () => mkdtemp(join(root, "run-")), spawn: fakeSpawn(child, () => spawned.resolve()) }); + const speaking = service.speakText({ text: "exit cleanup" }); await spawned.promise; child.emit("exit", 0, null); expect(await speaking).toEqual({ accepted: true }); await service.dispose(); await rm(root, { recursive: true, force: true }); }); }); diff --git a/apps/desktop/test/target-manager.test.ts b/apps/desktop/test/target-manager.test.ts index 627f3bf1..767cacc7 100644 --- a/apps/desktop/test/target-manager.test.ts +++ b/apps/desktop/test/target-manager.test.ts @@ -7,8 +7,10 @@ import { type PublicOmpServerEvent, } from "@t4-code/client"; import { + CI_TRIGGER_CAPABILITY, commandId, confirmationId, + CLUSTER_OPERATOR_FEATURE, DEVICE_CAPABILITIES, hostId, revision, @@ -224,6 +226,68 @@ async function settlesBeforeTurnLimit(promise: Promise): Promise { } describe("desktop target manager boundaries", () => { + it("requests cluster.operator only after explicit desktop opt-in", async () => { + const defaultTransport = new Transport(); + const defaultRuntime = new DesktopTargetManager({ + cursorStore: new Store(), + localTransportFactory: () => defaultTransport as never, + events: { onEvent: () => {}, onState: () => {}, onError: () => {} }, + }); + await defaultRuntime.connect("local"); + const defaultHello = JSON.parse(defaultTransport.sent[0] ?? "{}") as { + readonly requestedFeatures?: readonly string[]; + }; + expect(defaultHello.requestedFeatures).not.toContain(CLUSTER_OPERATOR_FEATURE); + await defaultRuntime.close(); + + const enabledTransport = new Transport(); + const enabledRuntime = new DesktopTargetManager({ + cursorStore: new Store(), + clusterOperatorEnabled: true, + localTransportFactory: () => enabledTransport as never, + events: { onEvent: () => {}, onState: () => {}, onError: () => {} }, + }); + await enabledRuntime.connect("local"); + const enabledHello = JSON.parse(enabledTransport.sent[0] ?? "{}") as { + readonly requestedFeatures?: readonly string[]; + }; + expect(enabledHello.requestedFeatures).toContain(CLUSTER_OPERATOR_FEATURE); + await enabledRuntime.close(); + + const registry = new Registry(); + await registry.put({ + ...target("cluster"), + requestedCapabilities: [...DEVICE_CAPABILITIES], + }); + const disabledRemoteTransport = new Transport(); + const disabledRemoteRuntime = new DesktopTargetManager({ + cursorStore: new Store(), + registry, + remoteTransportFactory: () => disabledRemoteTransport as never, + events: { onEvent: () => {}, onState: () => {}, onError: () => {} }, + }); + await disabledRemoteRuntime.connect("cluster"); + const disabledRemoteHello = JSON.parse(disabledRemoteTransport.sent[0] ?? "{}") as { + readonly capabilities?: { readonly client?: readonly string[] }; + }; + expect(disabledRemoteHello.capabilities?.client).not.toContain(CI_TRIGGER_CAPABILITY); + await disabledRemoteRuntime.close(); + + const enabledRemoteTransport = new Transport(); + const enabledRemoteRuntime = new DesktopTargetManager({ + cursorStore: new Store(), + registry, + clusterOperatorEnabled: true, + remoteTransportFactory: () => enabledRemoteTransport as never, + events: { onEvent: () => {}, onState: () => {}, onError: () => {} }, + }); + await enabledRemoteRuntime.connect("cluster"); + const enabledRemoteHello = JSON.parse(enabledRemoteTransport.sent[0] ?? "{}") as { + readonly capabilities?: { readonly client?: readonly string[] }; + }; + expect(enabledRemoteHello.capabilities?.client).toContain(CI_TRIGGER_CAPABILITY); + await enabledRemoteRuntime.close(); + }); it("lists and connects named local profiles through profile-scoped transports", async () => { const transports: Transport[] = []; const requestedProfiles: string[] = []; @@ -573,7 +637,7 @@ describe("desktop target manager boundaries", () => { expect(transports.every((item) => item.closed)).toBe(true); await runtime.close(); }); - it("keeps local full scope and isolates each remote scope across reconnects", async () => { + it("keeps local cluster control default-off and isolates each remote scope across reconnects", async () => { const localTransports: Transport[] = []; const local = new DesktopTargetManager({ cursorStore: new Store(), @@ -588,7 +652,9 @@ describe("desktop target manager boundaries", () => { const localHello = JSON.parse(localTransports[0]?.sent[0] ?? "{}") as Record; expect(localHello).toMatchObject({ type: "hello", - capabilities: { client: [...DEVICE_CAPABILITIES] }, + capabilities: { + client: DEVICE_CAPABILITIES.filter((capability) => capability !== "ci.trigger"), + }, }); await local.close(); @@ -653,8 +719,9 @@ describe("desktop target manager boundaries", () => { const registry = new Registry(); const remote = { ...target("observe-pair"), - requestedCapabilities: ["sessions.read", "catalog.read"], + requestedCapabilities: ["sessions.read", "catalog.read", CI_TRIGGER_CAPABILITY], }; + const effectiveCapabilities = ["sessions.read", "catalog.read"]; await registry.put(remote); const credentials = { withCredential(): T { @@ -678,7 +745,7 @@ describe("desktop target manager boundaries", () => { expect(await runtime.connect("observe-pair")).toBe("connecting"); expect(transports).toHaveLength(1); const hello = JSON.parse(transports[0]?.sent[0] ?? "{}") as Record; - expect(hello).toMatchObject({ capabilities: { client: remote.requestedCapabilities } }); + expect(hello).toMatchObject({ capabilities: { client: effectiveCapabilities } }); const pairTransport = transports[0]; if (pairTransport === undefined) throw new Error("pairing transport was not created"); const pairRequestPromise = pairTransport.waitForSent(1); @@ -686,8 +753,9 @@ describe("desktop target manager boundaries", () => { const pairRequest = await pairRequestPromise; expect(pairRequest).toMatchObject({ type: "pair.start", - requestedCapabilities: remote.requestedCapabilities, + requestedCapabilities: effectiveCapabilities, }); + expect(pairRequest).not.toMatchObject({ requestedCapabilities: remote.requestedCapabilities }); expect(pairRequest).not.toMatchObject({ requestedCapabilities: [...DEVICE_CAPABILITIES] }); pairTransport.receive({ v: V, @@ -697,11 +765,12 @@ describe("desktop target manager boundaries", () => { deviceId: "desktop", deviceName: "T4 Code Desktop", platform: process.platform, - requestedCapabilities: remote.requestedCapabilities, - grantedCapabilities: remote.requestedCapabilities, + requestedCapabilities: effectiveCapabilities, + grantedCapabilities: effectiveCapabilities, deviceToken: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", expiresAt: "2030-01-01T00:00:00Z", }); + expect(transports).toHaveLength(1); const result = await pairing; expect(result).toEqual({ targetId: "observe-pair", paired: true }); await runtime.close(); diff --git a/apps/web/src/components/MobileConnectionScreen.tsx b/apps/web/src/components/MobileConnectionScreen.tsx index 2a707dce..13f2146e 100644 --- a/apps/web/src/components/MobileConnectionScreen.tsx +++ b/apps/web/src/components/MobileConnectionScreen.tsx @@ -62,6 +62,7 @@ export function TailnetAddressForm({ const id = useId(); const [address, setAddress] = useState(""); const [profileId, setProfileId] = useState(""); + const [clusterOperatorEnabled, setClusterOperatorEnabled] = useState(false); const [message, setMessage] = useState(initialMessage ?? null); const [checking, setChecking] = useState(false); const activeProbe = useRef(null); @@ -73,6 +74,7 @@ export function TailnetAddressForm({ ); const addressId = `${id}-address`; const profileIdId = `${id}-profile`; + const clusterOperatorId = `${id}-cluster-operator`; const helpId = `${id}-help`; const statusId = `${id}-status`; @@ -85,7 +87,7 @@ export function TailnetAddressForm({ setMessage(null); let backend; try { - backend = parseTailnetBackend(address, profileId); + backend = parseTailnetBackend(address, profileId, clusterOperatorEnabled); } catch (error) { setMessage(error instanceof Error ? error.message : "Enter a valid Tailnet address."); return; @@ -141,13 +143,35 @@ export function TailnetAddressForm({ autoComplete="off" autoCorrect="off" className="h-12 w-full rounded-lg border border-input bg-background px-3 font-mono text-base outline-none transition-shadow duration-(--motion-duration-fast) placeholder:text-muted-foreground focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-1 focus-visible:ring-offset-background" - disabled={checking} + disabled={checking || clusterOperatorEnabled} id={profileIdId} onChange={(event) => setProfileId(event.target.value)} placeholder="default route" spellCheck={false} value={profileId} /> +

Use the full HTTPS address shown by the T4 gateway on your computer.

diff --git a/apps/web/src/features/agent-view/AgentViewScreen.tsx b/apps/web/src/features/agent-view/AgentViewScreen.tsx index 38ac7276..bf640bf1 100644 --- a/apps/web/src/features/agent-view/AgentViewScreen.tsx +++ b/apps/web/src/features/agent-view/AgentViewScreen.tsx @@ -470,6 +470,16 @@ export function AgentViewScreen({

{group.projectName} · {group.session.model} · {group.session.freshness}

+ {group.session.ci !== undefined && ( +

+ {group.session.ci.status} + {group.session.ci.currentStage ?? "CI stage unknown"} + {group.session.ci.branch ?? group.session.ci.ref ?? "CI branch unknown"} + + {group.session.ci.commit ?? "CI commit unknown"} + +

+ )} {group.session.status !== null && ( diff --git a/apps/web/src/features/preview/PreviewWorkspace.tsx b/apps/web/src/features/preview/PreviewWorkspace.tsx index a5072090..af061155 100644 --- a/apps/web/src/features/preview/PreviewWorkspace.tsx +++ b/apps/web/src/features/preview/PreviewWorkspace.tsx @@ -409,7 +409,7 @@ export function PreviewWorkspace({ setText(event.target.value)} value={text} />
{support("fill").supported && } - {support("type").supported && } + {support("type").supported && }
} {support("select").supported && <> diff --git a/apps/web/src/features/session-runtime/session-navigation.ts b/apps/web/src/features/session-runtime/session-navigation.ts index e6b8e95f..1eab24ce 100644 --- a/apps/web/src/features/session-runtime/session-navigation.ts +++ b/apps/web/src/features/session-runtime/session-navigation.ts @@ -1,7 +1,11 @@ +import type { DesktopRuntimeSnapshot } from "@t4-code/client"; + import type { SessionListView, WorkspaceSession, } from "../../lib/workspace-data.ts"; +import { resolveLiveSession } from "../../platform/live-workspace.ts"; +import type { SessionPreviewSelection } from "../../state/workspace-store.ts"; export type CompletedSessionManagementAction = "archive" | "restore" | "delete"; @@ -43,3 +47,29 @@ export function resolveSessionManagementNavigation( )?.id ?? null; return { view, destinationSessionId, navigate: true }; } + +/** + * Carry a host-advertised GUI preview through navigation. Trust metadata comes + * only from the matching warm projection; a not-yet-attached preview keeps the + * id without inventing authority or opting the user into an unknown profile. + */ +export function previewSelectionForNavigation( + snapshot: DesktopRuntimeSnapshot, + sessionViewId: string, + previewId: string, +): SessionPreviewSelection | null { + const address = resolveLiveSession(snapshot, sessionViewId); + if (address === null) return null; + const preview = [ + ...(snapshot.projection.sessions.get( + `${address.hostId}\u0000${address.sessionId}`, + )?.previews.values() ?? []), + ].find((candidate) => candidate.previewId === previewId); + if (preview === undefined) return { previewId, optIn: false }; + return { + previewId, + optInKind: preview.authority?.kind ?? null, + optInAuthorityId: preview.authority?.id ?? null, + optIn: true, + }; +} diff --git a/apps/web/src/features/targets/ClusterOperatorSection.tsx b/apps/web/src/features/targets/ClusterOperatorSection.tsx new file mode 100644 index 00000000..63785917 --- /dev/null +++ b/apps/web/src/features/targets/ClusterOperatorSection.tsx @@ -0,0 +1,652 @@ +import type { DesktopRuntimeController, DesktopRuntimeSnapshot } from "@t4-code/client"; +import { + AppWireError, + CI_TRIGGER_CAPABILITY, + CLUSTER_MAX_REFERENCE_BYTES, + CLUSTER_MAX_REPOSITORY_ID_BYTES, + decodeClusterSessionCreateArguments, + type ClusterSessionCreateArguments, + type SessionRef, +} from "@t4-code/protocol"; +import { Button, cn, Spinner } from "@t4-code/ui"; +import { useMemo, useState } from "react"; + +import { FIELD_CLASS } from "../settings/controls.tsx"; +import { deriveWorkspaceData, sessionViewId } from "../../platform/live-workspace.ts"; +import type { WorkspaceSession } from "../../lib/workspace-data.ts"; +import { + clusterCiAvailability, + clusterCreationTargets, + clusterGuiAvailability, + clusterOperatorAvailability, + createClusterSession, + createClusterWorkspace, + runClusterCi, +} from "./cluster-operator.ts"; + +const CLUSTER_FIELD_CLASS = `${FIELD_CLASS} min-h-11 sm:min-h-8`; + +export interface ClusterSessionCiDraft { + readonly repositoryId: string; + readonly ref: string; + readonly commit: string; +} + +type ClusterSessionCiInvalidField = keyof ClusterSessionCiDraft | "correlation"; + +export type ClusterSessionCreationPreparation = + | { + readonly args: ClusterSessionCreateArguments; + readonly error: null; + readonly invalidField: null; + } + | { + readonly args: null; + readonly error: string; + readonly invalidField: ClusterSessionCiInvalidField; + }; + +const EMPTY_SESSION_CI_DRAFT: ClusterSessionCiDraft = Object.freeze({ + repositoryId: "", + ref: "", + commit: "", +}); +const WOODPECKER_PARTIAL_REASON = + "Enter repository ID, ref, and commit together, or leave all three Woodpecker correlation fields empty."; +const WOODPECKER_COMMIT_REASON = + "Woodpecker commit must be exactly 40 lowercase hexadecimal characters."; +const WOODPECKER_REPOSITORY_REASON = + "Woodpecker repository ID must be 1 to 128 UTF-8 bytes, start with a letter or number, and otherwise use only letters, numbers, '.', '_', ':', '/', or '-' without '..' or '://'."; +const WOODPECKER_REF_REASON = + "Woodpecker ref must be 1 to 256 UTF-8 bytes and contain no control characters."; +const WOODPECKER_UNAVAILABLE_REASON = + "Woodpecker correlation is unavailable because this host did not grant CI trigger access."; + +function rejectedSessionCreation( + error: string, + invalidField: ClusterSessionCiInvalidField, +): ClusterSessionCreationPreparation { + return { args: null, error, invalidField }; +} + +export function prepareClusterSessionCreation( + workspaceId: string, + title: string | undefined, + ciDraft: ClusterSessionCiDraft, +): ClusterSessionCreationPreparation { + const normalizedTitle = title?.trim(); + const args: ClusterSessionCreateArguments = { + workspaceId, + ...(normalizedTitle === undefined || normalizedTitle === "" ? {} : { title: normalizedTitle }), + runtimeProfile: "default", + guiEnabled: true, + }; + const hasRepository = ciDraft.repositoryId !== ""; + const hasRef = ciDraft.ref !== ""; + const hasCommit = ciDraft.commit !== ""; + if (!hasRepository && !hasRef && !hasCommit) { + return { args, error: null, invalidField: null }; + } + if (!hasRepository || !hasRef || !hasCommit) { + return rejectedSessionCreation(WOODPECKER_PARTIAL_REASON, "correlation"); + } + + let ci: ClusterSessionCreateArguments["ci"]; + try { + ci = decodeClusterSessionCreateArguments({ + workspaceId: "workspace", + runtimeProfile: "default", + guiEnabled: true, + ci: { + provider: "woodpecker", + repositoryId: ciDraft.repositoryId, + ref: ciDraft.ref, + commit: ciDraft.commit, + }, + }).ci; + } catch (error) { + if (error instanceof AppWireError && error.path === "args.ci.repositoryId") { + return rejectedSessionCreation(WOODPECKER_REPOSITORY_REASON, "repositoryId"); + } + if (error instanceof AppWireError && error.path === "args.ci.ref") { + return rejectedSessionCreation(WOODPECKER_REF_REASON, "ref"); + } + if (error instanceof AppWireError && error.path === "args.ci.commit") { + return rejectedSessionCreation(WOODPECKER_COMMIT_REASON, "commit"); + } + return rejectedSessionCreation("Woodpecker correlation is invalid.", "correlation"); + } + if (!/^[0-9a-f]{40}$/u.test(ciDraft.commit)) { + return rejectedSessionCreation(WOODPECKER_COMMIT_REASON, "commit"); + } + if (ci === undefined) { + return rejectedSessionCreation("Woodpecker correlation is invalid.", "correlation"); + } + return { args: { ...args, ci }, error: null, invalidField: null }; +} + +export interface ClusterOperatorSectionProps { + readonly controller: DesktopRuntimeController; + readonly snapshot: DesktopRuntimeSnapshot; + readonly onOpenSession: (sessionId: string) => void; + readonly onOpenPreview: (sessionId: string, previewId: string) => void; +} + +export function clusterSessionMatchesWorkspace( + session: WorkspaceSession, + ref: SessionRef | undefined, + hostId: string, + workspaceId: string, +): boolean { + return String(ref?.hostId ?? "") === hostId && session.cluster?.workspaceId === workspaceId; +} + +export function ClusterOperatorSection({ + controller, + snapshot, + onOpenSession, + onOpenPreview, +}: ClusterOperatorSectionProps) { + const [query, setQuery] = useState(""); + const [creationHostId, setCreationHostId] = useState(""); + const [displayName, setDisplayName] = useState(""); + const [capacity, setCapacity] = useState("20Gi"); + const [sessionTitle, setSessionTitle] = useState>({}); + const [sessionCi, setSessionCi] = useState>({}); + const [sessionCiErrors, setSessionCiErrors] = useState< + Record | undefined> + >({}); + const [pending, setPending] = useState(null); + const [message, setMessage] = useState(null); + const data = deriveWorkspaceData(snapshot); + const creationTargets = useMemo(() => clusterCreationTargets(snapshot), [snapshot]); + const selectedCreationTarget = creationTargets.find( + (target) => target.hostId === creationHostId, + ); + const creationAvailability = + selectedCreationTarget === undefined + ? { + enabled: false, + reason: "Choose an advertised cluster host for creation.", + } + : clusterOperatorAvailability( + snapshot, + selectedCreationTarget.targetId, + selectedCreationTarget.hostId, + "manage", + ); + const clusterWorkspaces = data.clusterWorkspaces ?? []; + const sessionRefs = useMemo( + () => + new Map( + [...snapshot.projection.sessionIndex.values()].map((ref) => [ + sessionViewId(String(ref.hostId), String(ref.sessionId)), + ref, + ]), + ), + [snapshot.projection.sessionIndex], + ); + const normalizedQuery = query.trim().toLocaleLowerCase(); + const workspaces = useMemo( + () => + normalizedQuery === "" + ? clusterWorkspaces + : clusterWorkspaces.filter(({ infrastructure }) => + `${infrastructure.displayName} ${infrastructure.id} ${infrastructure.phase}` + .toLocaleLowerCase() + .includes(normalizedQuery), + ), + [clusterWorkspaces, normalizedQuery], + ); + + if (snapshot.clusterOperatorEnabled !== true || creationTargets.length === 0) return null; + + const act = async (id: string, operation: () => Promise) => { + setPending(id); + setMessage(null); + try { + await operation(); + setMessage("Request accepted. Status will update from the host projection."); + } catch (error) { + setMessage(error instanceof Error ? error.message : "The request failed."); + } finally { + setPending(null); + } + }; + + const updateSessionCi = ( + workspaceKey: string, + field: keyof ClusterSessionCiDraft, + value: string, + ) => { + setSessionCi((current) => ({ + ...current, + [workspaceKey]: { ...(current[workspaceKey] ?? EMPTY_SESSION_CI_DRAFT), [field]: value }, + })); + setSessionCiErrors((current) => { + if (current[workspaceKey] === undefined) return current; + const next = { ...current }; + delete next[workspaceKey]; + return next; + }); + }; + + return ( +
+
+
+

+ Cluster workspaces +

+

+ Infrastructure, sessions, CI, and GUI state reported by connected cluster hosts. +

+
+ +
+ +
{ + event.preventDefault(); + if (displayName.trim() === "" || selectedCreationTarget === undefined) return; + void act("create-workspace", () => + createClusterWorkspace( + controller, + selectedCreationTarget.targetId, + selectedCreationTarget.hostId, + { + displayName: displayName.trim(), + retentionPolicy: "Retain", + capacity, + }, + ), + ); + }} + > + + + + + {!creationAvailability.enabled &&

{creationAvailability.reason}

} +
+ + {workspaces.length === 0 ? ( +

+ {normalizedQuery === "" + ? "No cluster workspaces are projected yet. Create one to begin." + : "No projected workspace matches this search."} +

+ ) : ( +
    + {workspaces.map(({ hostId, targetId: workspaceTargetId, infrastructure }) => { + const workspaceKey = `${hostId}\u0000${infrastructure.id}`; + const workspaceAvailability = clusterOperatorAvailability( + snapshot, + workspaceTargetId, + hostId, + "manage", + ); + const ciDraft = sessionCi[workspaceKey] ?? EMPTY_SESSION_CI_DRAFT; + const ciError = sessionCiErrors[workspaceKey]; + const ciCorrelationEnabled = + snapshot.hosts + .get(hostId) + ?.grantedCapabilities.includes(CI_TRIGGER_CAPABILITY) === true; + const ciReasonId = `cluster-session-ci-reason-${encodeURIComponent(workspaceKey)}`; + const sessions = data.sessions.filter((session) => + clusterSessionMatchesWorkspace( + session, + sessionRefs.get(session.id), + hostId, + infrastructure.id, + ), + ); + const condition = infrastructure.condition; + return ( +
  • +
    +
    +

    {infrastructure.displayName}

    +

    {infrastructure.id}

    +
    +

    + {infrastructure.phase} + {condition === undefined ? " · Condition unknown" : ` · ${condition.reason}: ${condition.message}`} +

    +
    +

    + {infrastructure.capacity ?? "Capacity unknown"} · {infrastructure.storageClass ?? "Storage class unknown"} · {infrastructure.accessMode} · {infrastructure.retentionPolicy} +

    +
    { + event.preventDefault(); + const hasCiDraft = + ciDraft.repositoryId !== "" || ciDraft.ref !== "" || ciDraft.commit !== ""; + const preparation = + !ciCorrelationEnabled && hasCiDraft + ? rejectedSessionCreation(WOODPECKER_UNAVAILABLE_REASON, "correlation") + : prepareClusterSessionCreation( + infrastructure.id, + sessionTitle[workspaceKey], + ciDraft, + ); + if (preparation.args === null) { + setMessage(null); + setSessionCiErrors((current) => ({ + ...current, + [workspaceKey]: preparation, + })); + const invalidField = + preparation.invalidField === "correlation" + ? ciDraft.repositoryId === "" + ? "ciRepositoryId" + : ciDraft.ref === "" + ? "ciRef" + : "ciCommit" + : preparation.invalidField === "repositoryId" + ? "ciRepositoryId" + : preparation.invalidField === "ref" + ? "ciRef" + : "ciCommit"; + const invalidInput = event.currentTarget.elements.namedItem(invalidField); + if (invalidInput instanceof HTMLInputElement) invalidInput.focus(); + return; + } + setSessionCiErrors((current) => { + if (current[workspaceKey] === undefined) return current; + const next = { ...current }; + delete next[workspaceKey]; + return next; + }); + void act(`session:${workspaceKey}`, () => + createClusterSession( + controller, + workspaceTargetId, + hostId, + preparation.args, + ), + ); + }} + > + +
    + + Woodpecker correlation (optional) + +
    + + + +
    +
    + + {(ciError !== undefined || !ciCorrelationEnabled) && ( +

    + {ciError?.error ?? WOODPECKER_UNAVAILABLE_REASON} +

    + )} +
    + {!workspaceAvailability.enabled && ( +

    {workspaceAvailability.reason}

    + )} + {sessions.length > 0 && ( +
      + {sessions.map((session) => { + const ref = sessionRefs.get(session.id); + const ci = session.ci; + const gui = session.cluster?.gui; + const ciAvailability = clusterCiAvailability( + snapshot, + workspaceTargetId, + hostId, + ref?.revision, + ci, + ); + const guiAvailability = clusterGuiAvailability( + snapshot, + workspaceTargetId, + hostId, + gui, + ); + const actionReasonKey = encodeURIComponent(session.id); + const guiReasonId = `cluster-gui-reason-${actionReasonKey}`; + const ciReasonId = `cluster-ci-reason-${actionReasonKey}`; + return ( +
    • +
      +

      {session.title}

      +

      + {session.cluster?.phase ?? "Runtime phase unknown"} · GUI {gui?.state ?? "unknown"} + {gui?.reason === undefined || gui.reason === "" ? "" : ` · ${gui.reason}`} +

      +

      + {ci === undefined + ? "CI status unknown" + : `${ci.branch ?? ci.ref ?? "Branch unknown"} · ${ci.commit ?? "Commit unknown"} · ${ci.status}${ci.currentStage === undefined ? "" : ` · ${ci.currentStage}`}`} +

      + {ci?.link !== undefined && ( + + Open CI pipeline + + )} +
      +
      +
      + + + +
      + {!guiAvailability.enabled && ( +

      GUI: {guiAvailability.reason}

      + )} + {!ciAvailability.enabled && ( +

      CI: {ciAvailability.reason}

      + )} +
      +
      • + ); + })} +
    + )} +
    • + ); + })} +
+ )} + {message !== null &&

{message}

} +
+ ); +} diff --git a/apps/web/src/features/targets/TargetsScreen.tsx b/apps/web/src/features/targets/TargetsScreen.tsx index 45bc4d6c..08b863b0 100644 --- a/apps/web/src/features/targets/TargetsScreen.tsx +++ b/apps/web/src/features/targets/TargetsScreen.tsx @@ -4,7 +4,7 @@ // runtime snapshot or a completed desktop call — connection words are the // runtime's words, and removing a host says exactly what it does: it // deletes the credential stored on this computer, nothing more. -import type { DesktopRuntimeSnapshot } from "@t4-code/client"; +import type { DesktopRuntimeController, DesktopRuntimeSnapshot } from "@t4-code/client"; import type { LocalProfile, PhoneSetupState, ServiceInspection } from "@t4-code/protocol/desktop-ipc"; import { Badge, @@ -25,13 +25,14 @@ import { useEffect, useRef, useState } from "react"; import { QRCodeSVG } from "qrcode.react"; import { ToneBadge } from "../onboarding/bits.tsx"; +import { ClusterOperatorSection } from "./ClusterOperatorSection.tsx"; import { FIELD_CLASS } from "../settings/controls.tsx"; import { capabilityDiff, CONNECTION_STATE_META, deriveTargetRows, pairCommandForTarget, - TARGET_CAPABILITY_GROUPS, + selectTargetCapabilityGroups, type TargetCapabilityGroupId, type TargetRow, } from "./model.ts"; @@ -720,7 +721,13 @@ function TargetCard({ api, row }: { readonly api: TargetsStoreApi; readonly row: // ─── Add-host form ────────────────────────────────────────────────────────── -function AddHostForm({ api }: { readonly api: TargetsStoreApi }) { +function AddHostForm({ + api, + clusterOperatorEnabled, +}: { + readonly api: TargetsStoreApi; + readonly clusterOperatorEnabled: boolean; +}) { const draft = useTargets(api, (state) => state.draft); const errors = useTargets(api, (state) => state.draftErrors); const addError = useTargets(api, (state) => state.addError); @@ -811,7 +818,7 @@ function AddHostForm({ api }: { readonly api: TargetsStoreApi }) { Ask that host for permission to - {TARGET_CAPABILITY_GROUPS.map((group) => { + {selectTargetCapabilityGroups(clusterOperatorEnabled).map((group) => { const locked = group.id === "observe"; const checked = locked || draft.groups.has(group.id); return ( @@ -954,6 +961,9 @@ function RemoveProfileDialog({ api }: { readonly api: TargetsStoreApi }) { export function TargetsScreen({ api, + controller, + onOpenSession, + onOpenPreview, snapshot, serviceAvailable, profilesAvailable, @@ -961,6 +971,9 @@ export function TargetsScreen({ onBack, }: { readonly api: TargetsStoreApi; + readonly controller: DesktopRuntimeController; + readonly onOpenSession: (sessionId: string) => void; + readonly onOpenPreview: (sessionId: string, previewId: string) => void; readonly snapshot: DesktopRuntimeSnapshot; /** Whether this desktop build exposes local service management. */ readonly serviceAvailable: boolean; @@ -1018,7 +1031,13 @@ export function TargetsScreen({ )} - + + diff --git a/apps/web/src/features/targets/cluster-operator.ts b/apps/web/src/features/targets/cluster-operator.ts new file mode 100644 index 00000000..db3f58fc --- /dev/null +++ b/apps/web/src/features/targets/cluster-operator.ts @@ -0,0 +1,232 @@ +import type { DesktopRuntimeController, DesktopRuntimeSnapshot } from "@t4-code/client"; +import { + CI_TRIGGER_CAPABILITY, + CLUSTER_OPERATOR_FEATURE, + hostId, + sessionId, + type CiRunArguments, + type ClusterSessionCreateArguments, + type ClusterWorkspaceCreateArguments, + type Revision, + type SessionCiState, + type SessionClusterState, +} from "@t4-code/protocol"; + +import { resolveCurrentHostTargetId } from "../../lib/host-target.ts"; +import { previewHostSupport } from "../preview/preview-model.ts"; + +export type ClusterOperation = "read" | "manage" | "ci"; + +export interface ClusterOperatorAvailability { + readonly enabled: boolean; + readonly reason?: string; +} + +export interface ClusterCreationTarget { + readonly targetId: string; + readonly hostId: string; + readonly label: string; +} + +export function clusterCreationTargets( + snapshot: DesktopRuntimeSnapshot, +): readonly ClusterCreationTarget[] { + if (snapshot.clusterOperatorEnabled !== true) return []; + const targets: ClusterCreationTarget[] = []; + for (const [hostIdValue, host] of snapshot.hosts) { + if (!host.grantedFeatures.includes(CLUSTER_OPERATOR_FEATURE)) continue; + const targetId = resolveCurrentHostTargetId(snapshot, hostIdValue); + if (targetId === null) continue; + targets.push({ + targetId, + hostId: hostIdValue, + label: snapshot.targets.get(targetId)?.label ?? hostIdValue, + }); + } + targets.sort( + (left, right) => + left.label.localeCompare(right.label) || left.hostId.localeCompare(right.hostId), + ); + return targets; +} + +export function clusterOperatorAvailability( + snapshot: DesktopRuntimeSnapshot, + targetId: string, + hostIdValue: string, + operation: ClusterOperation, + expectedRevision?: Revision, +): ClusterOperatorAvailability { + if (snapshot.clusterOperatorEnabled !== true) { + return { enabled: false, reason: "Cluster operator is disabled in this app." }; + } + if (snapshot.targetHosts.get(targetId) !== hostIdValue) { + return { + enabled: false, + reason: "This cluster host is no longer bound to the selected target.", + }; + } + if (snapshot.connections.get(targetId) !== "connected") { + return { enabled: false, reason: "Reconnect this host to inspect cluster workspaces." }; + } + const host = snapshot.hosts.get(hostIdValue); + if (host === undefined || !host.grantedFeatures.includes(CLUSTER_OPERATOR_FEATURE)) { + return { enabled: false, reason: "This host does not advertise cluster operator support." }; + } + if (!host.grantedCapabilities.includes("sessions.read")) { + return { enabled: false, reason: "This host did not grant session read access." }; + } + if (operation === "manage" && !host.grantedCapabilities.includes("sessions.manage")) { + return { + enabled: false, + reason: "This host did not grant workspace and session management.", + }; + } + if (operation === "ci" && !host.grantedCapabilities.includes(CI_TRIGGER_CAPABILITY)) { + return { enabled: false, reason: "This host did not grant CI trigger access." }; + } + if (operation === "ci" && expectedRevision === undefined) { + return { enabled: false, reason: "Waiting for the latest session revision." }; + } + return { enabled: true }; +} + +export function clusterCiAvailability( + snapshot: DesktopRuntimeSnapshot, + targetId: string, + hostIdValue: string, + expectedRevision: Revision | undefined, + ci: SessionCiState | undefined, +): ClusterOperatorAvailability { + const availability = clusterOperatorAvailability( + snapshot, + targetId, + hostIdValue, + "ci", + expectedRevision, + ); + if (!availability.enabled) return availability; + if (ci === undefined) { + return { enabled: false, reason: "CI status is unavailable for this session." }; + } + if (ci.correlation !== "exact") { + return { + enabled: false, + reason: "CI correlation is unknown; a run cannot be triggered.", + }; + } + return { enabled: true }; +} + +export function clusterGuiAvailability( + snapshot: DesktopRuntimeSnapshot, + targetId: string, + hostIdValue: string, + gui: SessionClusterState["gui"] | undefined, +): ClusterOperatorAvailability { + const availability = clusterOperatorAvailability( + snapshot, + targetId, + hostIdValue, + "read", + ); + if (!availability.enabled) return availability; + const support = previewHostSupport(snapshot.hosts.get(hostIdValue)); + if (!support.supported) { + return { + enabled: false, + reason: support.reason ?? "This host does not permit browser preview reads.", + }; + } + if (!support.controlSupported) { + return { + enabled: false, + reason: "This host does not permit browser preview control.", + }; + } + if (gui === undefined) { + return { enabled: false, reason: "GUI status is unavailable for this session." }; + } + if (gui.state !== "Ready") { + return { + enabled: false, + reason: + gui.reason === undefined || gui.reason === "" + ? `GUI is ${gui.state.toLocaleLowerCase()}.` + : gui.reason, + }; + } + if (gui.previewId === undefined) { + return { enabled: false, reason: "Waiting for the negotiated GUI preview." }; + } + return { enabled: true }; +} + +function requireAvailability( + controller: DesktopRuntimeController, + targetId: string, + hostIdValue: string, + operation: ClusterOperation, + expectedRevision?: Revision, +): void { + const availability = clusterOperatorAvailability( + controller.getSnapshot(), + targetId, + hostIdValue, + operation, + expectedRevision, + ); + if (!availability.enabled) throw new Error(availability.reason); +} + +export async function createClusterWorkspace( + controller: DesktopRuntimeController, + targetId: string, + hostIdValue: string, + args: ClusterWorkspaceCreateArguments, +) { + requireAvailability(controller, targetId, hostIdValue, "manage"); + const result = await controller.command(targetId, { + hostId: hostId(hostIdValue), + command: "workspace.create", + args: { ...args }, + }); + if (!result.accepted) throw new Error("Cluster workspace creation was rejected."); + return result; +} + +export async function createClusterSession( + controller: DesktopRuntimeController, + targetId: string, + hostIdValue: string, + args: ClusterSessionCreateArguments, +) { + requireAvailability(controller, targetId, hostIdValue, "manage"); + const result = await controller.command(targetId, { + hostId: hostId(hostIdValue), + command: "session.create", + args: { ...args }, + }); + if (!result.accepted) throw new Error("Cluster session creation was rejected."); + return result; +} + +export async function runClusterCi( + controller: DesktopRuntimeController, + targetId: string, + hostIdValue: string, + sessionIdValue: string, + expectedRevision: Revision, + args: CiRunArguments, +) { + requireAvailability(controller, targetId, hostIdValue, "ci", expectedRevision); + const result = await controller.command(targetId, { + hostId: hostId(hostIdValue), + sessionId: sessionId(sessionIdValue), + command: "ci.run", + expectedRevision, + args: { ...args }, + }); + if (!result.accepted) throw new Error("CI run was rejected."); + return result; +} diff --git a/apps/web/src/features/targets/model.ts b/apps/web/src/features/targets/model.ts index b505bcc4..5d3ccb4b 100644 --- a/apps/web/src/features/targets/model.ts +++ b/apps/web/src/features/targets/model.ts @@ -9,7 +9,7 @@ import type { ConnectionState, DesktopTarget, TargetAddRequest } from "@t4-code/ // ─── Capability groups ────────────────────────────────────────────────────── /** Plain-language capability groups offered when adding a target. */ -export type TargetCapabilityGroupId = "observe" | "control" | "shell" | "files" | "settings"; +export type TargetCapabilityGroupId = "observe" | "control" | "shell" | "files" | "settings" | "cluster"; export interface TargetCapabilityGroup { readonly id: TargetCapabilityGroupId; @@ -54,8 +54,25 @@ export const TARGET_CAPABILITY_GROUPS: readonly TargetCapabilityGroup[] = [ impact: "Edit that host's settings from this app.", capabilities: ["config.write"], }, + { + id: "cluster", + label: "Cluster sessions", + impact: "Trigger CI and view or control isolated session GUIs on that host.", + capabilities: ["ci.trigger", "preview.read", "preview.control", "preview.input"], + }, ]; +const DEFAULT_TARGET_CAPABILITY_GROUPS = TARGET_CAPABILITY_GROUPS.filter( + (group) => group.id !== "cluster", +); + +/** Groups offered by the add-host form; cluster access is an explicit app opt-in. */ +export function selectTargetCapabilityGroups( + clusterOperatorEnabled?: boolean, +): readonly TargetCapabilityGroup[] { + return clusterOperatorEnabled === true ? TARGET_CAPABILITY_GROUPS : DEFAULT_TARGET_CAPABILITY_GROUPS; +} + /** Wire capabilities for a set of chosen groups, in catalog order. */ export function capabilitiesForGroups(groups: ReadonlySet): readonly string[] { const out: string[] = []; diff --git a/apps/web/src/lib/workspace-data.ts b/apps/web/src/lib/workspace-data.ts index 74afca80..d5f34f97 100644 --- a/apps/web/src/lib/workspace-data.ts +++ b/apps/web/src/lib/workspace-data.ts @@ -5,6 +5,11 @@ // is feeding it. Display data only; never runtime authority. import type { SessionStatus } from "@t4-code/ui"; import type { RuntimeKind } from "@t4-code/client"; +import type { + SessionCiState, + SessionClusterState, + WorkspaceInfrastructureProjection, +} from "@t4-code/protocol"; /** How current the projection of a session is. */ export type SessionFreshness = "live" | "cached" | "offline"; @@ -62,10 +67,21 @@ export interface WorkspaceSession { * SessionControlDisplayKind in session-observer.ts. */ readonly control?: "observer" | "suspect" | "reconciling" | "unclear"; + /** Cluster runtime and GUI truth, present only after local opt-in and host grant. */ + readonly cluster?: SessionClusterState; + /** Strict CI correlation and progress from the authoritative session projection. */ + readonly ci?: SessionCiState; +} + +export interface WorkspaceClusterWorkspace { + readonly hostId: string; + readonly targetId: string; + readonly infrastructure: WorkspaceInfrastructureProjection; } export interface WorkspaceData { readonly hosts: readonly WorkspaceHost[]; readonly projects: readonly WorkspaceProject[]; readonly sessions: readonly WorkspaceSession[]; + readonly clusterWorkspaces?: readonly WorkspaceClusterWorkspace[]; } diff --git a/apps/web/src/platform/browser-shell-port.ts b/apps/web/src/platform/browser-shell-port.ts index 325e8419..6ee3c729 100644 --- a/apps/web/src/platform/browser-shell-port.ts +++ b/apps/web/src/platform/browser-shell-port.ts @@ -11,6 +11,8 @@ // or explicit query parameters. An ordinary page URL is never used as an // implicit backend endpoint. import { + clusterOperatorRequestedCapabilities, + clusterOperatorRequestedFeatures, createOmpClient, isConfirmationDecisionConsumed, type OmpClient, @@ -69,11 +71,6 @@ import { } from "./native-mobile.ts"; const TARGET_ID = "remote"; -const COMPATIBILITY_FEATURES: readonly string[] = Object.freeze( - ADDITIVE_FEATURES.filter( - (feature) => feature !== "prompt.images" && feature !== "transcript.images", - ), -); const MAX_URL_LENGTH = 2048; const MAX_LABEL_LENGTH = 128; @@ -84,6 +81,7 @@ export interface BrowserBackendConfig { readonly label: string; readonly deviceId?: string; readonly deviceToken?: string; + readonly clusterOperatorEnabled?: true; } function boundedText(value: unknown, name: string, max: number): string { @@ -114,13 +112,35 @@ function validatedWsUrl(value: unknown): string { } return parsed.toString(); } +function validateClusterWsUrl(value: string): string { + const parsed = new URL(value); + if ( + parsed.protocol !== "wss:" || + parsed.username !== "" || + parsed.password !== "" || + parsed.port !== "" || + !parsed.hostname.endsWith(".ts.net") || + parsed.pathname !== "/v1/ws" + ) { + throw new Error("cluster operator requires one secure WSS cluster target"); + } + if (parsed.search !== "" || parsed.hash !== "") { + throw new Error("cluster operator target must not contain query credentials or fragments"); + } + return parsed.toString(); +} function parseBackendPayload(value: unknown): BrowserBackendConfig { if (value === null || typeof value !== "object" || Array.isArray(value)) { throw new Error("invalid browser backend payload"); } const data = value as Record; - const wsUrl = validatedWsUrl(data.wsUrl); + const clusterOperatorEnabled = data.clusterOperatorEnabled === true; + if (data.clusterOperatorEnabled !== undefined && !clusterOperatorEnabled) { + throw new Error("invalid browser backend clusterOperatorEnabled"); + } + const rawWsUrl = validatedWsUrl(data.wsUrl); + const wsUrl = clusterOperatorEnabled ? validateClusterWsUrl(rawWsUrl) : rawWsUrl; const label = data.label === undefined ? "T4 host" : boundedText(data.label, "label", MAX_LABEL_LENGTH); const auth = data.auth; @@ -138,6 +158,7 @@ function parseBackendPayload(value: unknown): BrowserBackendConfig { return { wsUrl, label, + ...(clusterOperatorEnabled ? { clusterOperatorEnabled: true as const } : {}), ...(deviceId === undefined ? {} : { deviceId: boundedText(deviceId, "deviceId", MAX_DEVICE_ID_LENGTH) }), @@ -211,6 +232,19 @@ export function createBrowserShellPort( const config = detectBackend(); if (config === null) return null; const backendConfig = config; + const requestedFeatures = clusterOperatorRequestedFeatures( + ADDITIVE_FEATURES, + backendConfig.clusterOperatorEnabled === true, + ); + const requestedCapabilities = clusterOperatorRequestedCapabilities( + DEVICE_CAPABILITIES, + backendConfig.clusterOperatorEnabled === true, + ); + const compatibilityRequestedFeatures = Object.freeze( + requestedFeatures.filter( + (feature) => feature !== "prompt.images" && feature !== "transcript.images", + ), + ); const nativeEndpointKey = currentNativeMobileBackend()?.endpointKey; const mobilePlatform = nativeMobilePlatform(); const platform: "linux" | "darwin" = (() => { @@ -292,9 +326,9 @@ export function createBrowserShellPort( const c = (options.clientFactory ?? createOmpClient)({ transport: transportFactory, - capabilities: DEVICE_CAPABILITIES, - requestedFeatures: ADDITIVE_FEATURES, - compatibilityRequestedFeatures: COMPATIBILITY_FEATURES, + capabilities: requestedCapabilities, + requestedFeatures, + compatibilityRequestedFeatures, authentication: () => authentication, privilegedPairResult: async (result) => { await persistNativeMobileCredentials(result, nativeEndpointKey); @@ -393,7 +427,7 @@ export function createBrowserShellPort( } const shell: DesktopShellPort = { kind: "desktop" as const, - + clusterOperatorEnabled: backendConfig.clusterOperatorEnabled === true, speakText, stopSpeaking, platform, @@ -536,7 +570,7 @@ export function createBrowserShellPort( ? "T4 Browser" : `T4 ${mobilePlatform === "android" ? "Android" : "iOS"}`, platform: mobilePlatform ?? platform, - requestedCapabilities: DEVICE_CAPABILITIES, + requestedCapabilities, }); return { targetId: request.targetId, paired: true }; }, diff --git a/apps/web/src/platform/desktop-runtime.ts b/apps/web/src/platform/desktop-runtime.ts index d392545d..feb8fa66 100644 --- a/apps/web/src/platform/desktop-runtime.ts +++ b/apps/web/src/platform/desktop-runtime.ts @@ -66,6 +66,7 @@ export function acquireRuntimeController( const cacheStore = projectionCacheStore(shell); const controller = createDesktopRuntimeController({ shell, + clusterOperatorEnabled: shell.clusterOperatorEnabled === true, ...(cacheStore === undefined ? {} : { projection: createProjectionStore({ cacheStore }) }), diff --git a/apps/web/src/platform/live-workspace.ts b/apps/web/src/platform/live-workspace.ts index 5a81f29a..0844df46 100644 --- a/apps/web/src/platform/live-workspace.ts +++ b/apps/web/src/platform/live-workspace.ts @@ -8,9 +8,11 @@ import { readSessionAttention, type SessionProjection, } from "@t4-code/client"; +import { CLUSTER_OPERATOR_FEATURE } from "@t4-code/protocol"; import type { SessionStatus } from "@t4-code/ui"; import type { + WorkspaceClusterWorkspace, WorkspaceData, WorkspaceHost, SessionLifecycle, @@ -186,6 +188,21 @@ function hostConnection( ? { targetId: null, state: null } : { targetId, state: snapshot.connections.get(targetId) ?? null }; } +function clusterHostTarget( + snapshot: DesktopRuntimeSnapshot, + hostId: string, +): string | null { + if (snapshot.clusterOperatorEnabled !== true) return null; + const host = snapshot.hosts.get(hostId); + if ( + host === undefined || + !host.grantedFeatures.includes(CLUSTER_OPERATOR_FEATURE) || + !host.grantedCapabilities.includes("sessions.read") + ) { + return null; + } + return resolveCurrentHostTargetId(snapshot, hostId); +} const derived = new WeakMap(); @@ -193,6 +210,7 @@ const EMPTY_WORKSPACE: WorkspaceData = Object.freeze({ hosts: Object.freeze([]), projects: Object.freeze([]), sessions: Object.freeze([]), + clusterWorkspaces: Object.freeze([]), }); /** @@ -214,6 +232,12 @@ export function deriveWorkspaceData(snapshot: DesktopRuntimeSnapshot): Workspace for (const ref of snapshot.projection.sessionIndex.values()) { hostIds.add(String(ref.hostId)); } + if (snapshot.clusterOperatorEnabled === true) { + for (const key of snapshot.projection.workspaces.keys()) { + const separator = key.indexOf("\u0000"); + if (separator > 0) hostIds.add(key.slice(0, separator)); + } + } for (const hostId of hostIds) { const meta = snapshot.hosts.get(hostId); if (meta === undefined) { @@ -254,6 +278,17 @@ export function deriveWorkspaceData(snapshot: DesktopRuntimeSnapshot): Workspace }); } + const clusterWorkspaces: WorkspaceClusterWorkspace[] = []; + if (snapshot.clusterOperatorEnabled === true) { + for (const [key, infrastructure] of snapshot.projection.workspaces) { + const separator = key.indexOf("\u0000"); + if (separator <= 0) continue; + const hostId = key.slice(0, separator); + const targetId = clusterHostTarget(snapshot, hostId); + if (targetId !== null) clusterWorkspaces.push({ hostId, targetId, infrastructure }); + } + } + const projects = new Map(); const projectsWithAdvertisedNames = new Set(); const sessions: WorkspaceSession[] = []; @@ -337,16 +372,23 @@ export function deriveWorkspaceData(snapshot: DesktopRuntimeSnapshot): Workspace lastActivity: "", ...(archivedAt === null ? {} : { archivedAt }), ...(controlKind === undefined ? {} : { control: controlKind }), + ...(clusterHostTarget(snapshot, hostId) === null || ref.liveState?.cluster === undefined + ? {} + : { cluster: ref.liveState.cluster }), + ...(clusterHostTarget(snapshot, hostId) === null || ref.liveState?.ci === undefined + ? {} + : { ci: ref.liveState.ci }), }); } const data: WorkspaceData = - sessions.length === 0 && hosts.length === 0 + sessions.length === 0 && hosts.length === 0 && clusterWorkspaces.length === 0 ? EMPTY_WORKSPACE : Object.freeze({ hosts: Object.freeze(hosts), projects: Object.freeze([...projects.values()]), sessions: Object.freeze(sessions), + clusterWorkspaces: Object.freeze(clusterWorkspaces), }); derived.set(snapshot, data); return data; diff --git a/apps/web/src/platform/native-mobile-backend.ts b/apps/web/src/platform/native-mobile-backend.ts index 7a2aa736..10b655dd 100644 --- a/apps/web/src/platform/native-mobile-backend.ts +++ b/apps/web/src/platform/native-mobile-backend.ts @@ -12,6 +12,7 @@ export interface StoredMobileBackend { readonly profileId: string; readonly wsUrl: string; readonly label: string; + readonly clusterOperatorEnabled?: true; } export interface StoredMobileBackendDirectory { @@ -54,12 +55,19 @@ function websocketUrlFor(origin: string, profileId: string): string { return websocket.toString(); } -export function parseTailnetBackend(value: string, profileId?: string): StoredMobileBackend { +export function parseTailnetBackend( + value: string, + profileId?: string, + clusterOperatorEnabled = false, +): StoredMobileBackend { const trimmed = value.trim(); if (trimmed.length === 0) throw new Error("Enter the HTTPS address shown by T4 Code on your computer."); if (trimmed.length > MAX_URL_LENGTH) throw new Error("That address is too long."); const selectedProfile = normalizeMobileProfileId(profileId); + if (clusterOperatorEnabled && selectedProfile !== DEFAULT_MOBILE_PROFILE_ID) { + throw new Error("The cluster operator uses the default secure WSS route."); + } const candidate = trimmed.includes("://") ? trimmed : `https://${trimmed}`; let parsed: URL; try { @@ -77,6 +85,9 @@ export function parseTailnetBackend(value: string, profileId?: string): StoredMo if (hostname === "ts.net" || !hostname.endsWith(".ts.net")) { throw new Error("Use the full Tailscale hostname ending in .ts.net."); } + if (clusterOperatorEnabled && parsed.port !== "") { + throw new Error("The cluster operator requires the standard secure WSS route, not a NodePort."); + } const origin = parsed.origin; return { version: 3, @@ -85,5 +96,6 @@ export function parseTailnetBackend(value: string, profileId?: string): StoredMo profileId: selectedProfile, wsUrl: websocketUrlFor(origin, selectedProfile), label: requiredLabel(`T4 on ${hostname.slice(0, hostname.indexOf("."))}`), + ...(clusterOperatorEnabled ? { clusterOperatorEnabled: true as const } : {}), }; } diff --git a/apps/web/src/platform/native-mobile.ts b/apps/web/src/platform/native-mobile.ts index 95f2cf0f..bbe21aa3 100644 --- a/apps/web/src/platform/native-mobile.ts +++ b/apps/web/src/platform/native-mobile.ts @@ -32,6 +32,7 @@ export interface NativeMobileBackendConfig { readonly label: string; readonly deviceId?: string; readonly deviceToken?: string; + readonly clusterOperatorEnabled?: true; } interface T4SecureStoragePlugin { @@ -196,14 +197,23 @@ function storedMobileBackend(value: unknown): StoredMobileBackend { throw new Error("The saved host list is from an unsupported app version."); } if (typeof data.origin !== "string") throw new Error("The saved host list is damaged. Add the host again."); + if ( + data.clusterOperatorEnabled !== undefined && + (data.version !== 3 || data.clusterOperatorEnabled !== true) + ) { + throw new Error("The saved host list is damaged. Add the host again."); + } const parsed = parseTailnetBackend( data.origin, data.version === 3 && typeof data.profileId === "string" ? data.profileId : undefined, + data.version === 3 && data.clusterOperatorEnabled === true, ); if ( data.wsUrl !== parsed.wsUrl || data.label !== parsed.label || - (data.version === 3 && data.endpointKey !== parsed.endpointKey) + (data.version === 3 && data.endpointKey !== parsed.endpointKey) || + (data.version === 3 && data.clusterOperatorEnabled === true) !== + (parsed.clusterOperatorEnabled === true) ) { throw new Error("The saved host list is inconsistent. Add the host again."); } @@ -275,6 +285,12 @@ export function writeStoredMobileBackend( const canonical = storedMobileBackend(backend); const current = readStoredMobileBackendDirectory(storage); const existing = current?.backends.filter((item) => item.endpointKey !== canonical.endpointKey) ?? []; + if ( + canonical.clusterOperatorEnabled === true && + existing.some((item) => item.clusterOperatorEnabled === true) + ) { + throw new Error("This phone can save only one cluster operator endpoint."); + } if (current === null && existing.length >= MAX_SAVED_MOBILE_BACKENDS) { throw new Error(`This phone can save up to ${MAX_SAVED_MOBILE_BACKENDS} T4 hosts.`); } @@ -377,6 +393,7 @@ export async function prepareNativeMobileBackend(): Promise { profileId: backend.profileId, wsUrl: backend.wsUrl, label: backend.label, + ...(backend.clusterOperatorEnabled === true ? { clusterOperatorEnabled: true as const } : {}), ...(credentials === null ? {} : credentials), }; return { kind: "ready", backend }; diff --git a/apps/web/src/router.tsx b/apps/web/src/router.tsx index d9a62b68..b1f0ac59 100644 --- a/apps/web/src/router.tsx +++ b/apps/web/src/router.tsx @@ -36,6 +36,7 @@ import { FixturePreviewWorkspace } from "./features/preview/FixturePreviewWorksp import { LiveAttentionInbox } from "./features/attention/index.ts"; import { LiveTranscriptSearch } from "./features/transcript-search/index.ts"; import { TRANSCRIPT_SEARCH_ROUTE } from "./features/transcript-search/route.ts"; +import { previewSelectionForNavigation } from "./features/session-runtime/session-navigation.ts"; import { SettingsWorkspace } from "./features/settings/index.ts"; import { LiveSettingsScreen } from "./features/settings/LiveSettingsScreen.tsx"; import { TargetsScreen } from "./features/targets/TargetsScreen.tsx"; @@ -463,6 +464,17 @@ function HostsRoute() { return (
{ + const selection = previewSelectionForNavigation(snapshot, sessionId, previewId); + if (selection !== null) { + workspaceStore.getState().setSessionPreview(sessionId, selection); + } + void navigate({ to: "/sessions/$sessionId/preview", params: { sessionId } }); + }} + onOpenSession={(sessionId) => + void navigate({ to: "/sessions/$sessionId", params: { sessionId } }) + } api={targetsStoreInstance} onBack={() => void navigate({ to: "/settings" })} profilesAvailable={localProfiles !== undefined} diff --git a/apps/web/test/cluster-mobile.test.ts b/apps/web/test/cluster-mobile.test.ts new file mode 100644 index 00000000..1a29227e --- /dev/null +++ b/apps/web/test/cluster-mobile.test.ts @@ -0,0 +1,128 @@ +import { CI_TRIGGER_CAPABILITY, CLUSTER_OPERATOR_FEATURE } from "@t4-code/protocol"; +import type { OmpClient, OmpClientOptions } from "@t4-code/client"; +import { afterEach, describe, expect, it } from "vite-plus/test"; + +import { createBrowserShellPort, detectBackend } from "../src/platform/browser-shell-port.ts"; +import { parseTailnetBackend } from "../src/platform/native-mobile-backend.ts"; + +const originalDocument = globalThis.document; +const originalWindow = globalThis.window; + +function backendScript(value: unknown): void { + Object.defineProperty(globalThis, "document", { + configurable: true, + value: { + getElementById: () => ({ textContent: JSON.stringify(value) }), + }, + }); + Object.defineProperty(globalThis, "window", { + configurable: true, + value: { location: { search: "" } }, + }); +} + +function captureClientOptions(): { + readonly options: () => OmpClientOptions; + readonly factory: (value: OmpClientOptions) => OmpClient; +} { + let captured: OmpClientOptions | undefined; + return { + options: () => { + if (captured === undefined) throw new Error("client was not built"); + return captured; + }, + factory: (value) => { + captured = value; + return { + state: "idle", + connect: async () => undefined, + close: async () => undefined, + wake: () => undefined, + onEvent: () => () => undefined, + onState: () => () => undefined, + onError: () => () => undefined, + } as unknown as OmpClient; + }, + }; +} + +afterEach(() => { + Object.defineProperty(globalThis, "document", { configurable: true, value: originalDocument }); + Object.defineProperty(globalThis, "window", { configurable: true, value: originalWindow }); +}); + +describe("mobile cluster target", () => { + it("keeps the saved Tailnet target operator-disabled by default", () => { + expect(parseTailnetBackend("https://operator.tailnet.ts.net")).toEqual({ + version: 3, + endpointKey: "https://operator.tailnet.ts.net#profile=default", + origin: "https://operator.tailnet.ts.net", + profileId: "default", + wsUrl: "wss://operator.tailnet.ts.net/v1/ws", + label: "T4 on operator", + }); + expect( + parseTailnetBackend("https://operator.tailnet.ts.net", "default", true), + ).toMatchObject({ + clusterOperatorEnabled: true, + wsUrl: "wss://operator.tailnet.ts.net/v1/ws", + }); + }); + + it("requests no cluster feature for an ordinary browser/mobile backend", async () => { + backendScript({ wsUrl: "wss://operator.tailnet.ts.net/v1/ws", label: "Operator" }); + const capture = captureClientOptions(); + const shell = createBrowserShellPort({ clientFactory: capture.factory }); + if (shell === null) throw new Error("shell was not created"); + await shell.bootstrap(); + + expect(capture.options().requestedFeatures).not.toContain(CLUSTER_OPERATOR_FEATURE); + expect(capture.options().compatibilityRequestedFeatures).not.toContain( + CLUSTER_OPERATOR_FEATURE, + ); + expect(capture.options().capabilities).not.toContain(CI_TRIGGER_CAPABILITY); + expect(capture.options().capabilities).toEqual( + expect.arrayContaining(["sessions.read", "preview.read", "preview.control", "preview.input"]), + ); + }); + + it("allows one explicit secure cluster target and rejects insecure or credentialed URLs", async () => { + backendScript({ + wsUrl: "wss://operator.tailnet.ts.net/v1/ws", + label: "Operator", + clusterOperatorEnabled: true, + }); + const capture = captureClientOptions(); + const shell = createBrowserShellPort({ clientFactory: capture.factory }); + if (shell === null) throw new Error("shell was not created"); + await shell.bootstrap(); + + expect(detectBackend()).toMatchObject({ clusterOperatorEnabled: true }); + expect(capture.options().requestedFeatures).toContain(CLUSTER_OPERATOR_FEATURE); + expect(capture.options().compatibilityRequestedFeatures).toContain(CLUSTER_OPERATOR_FEATURE); + expect(capture.options().requestedFeatures).toContain("preview.control"); + expect(capture.options().capabilities).toEqual( + expect.arrayContaining([ + "sessions.read", + "sessions.manage", + CI_TRIGGER_CAPABILITY, + "preview.read", + "preview.control", + "preview.input", + ]), + ); + + backendScript({ + wsUrl: "ws://operator.tailnet.ts.net/v1/ws", + label: "Operator", + clusterOperatorEnabled: true, + }); + expect(() => detectBackend()).toThrow(/secure WSS cluster target/u); + backendScript({ + wsUrl: "wss://operator.tailnet.ts.net/v1/ws?token=secret", + label: "Operator", + clusterOperatorEnabled: true, + }); + expect(() => detectBackend()).toThrow(/query|credential/u); + }); +}); diff --git a/apps/web/test/cluster-operator.test.tsx b/apps/web/test/cluster-operator.test.tsx new file mode 100644 index 00000000..d03f765f --- /dev/null +++ b/apps/web/test/cluster-operator.test.tsx @@ -0,0 +1,587 @@ +import { + CI_TRIGGER_CAPABILITY, + CLUSTER_OPERATOR_FEATURE, + hostId, + revision, + sessionId, + type SessionRef, + type WorkspaceInfrastructureProjection, +} from "@t4-code/protocol"; +import { + createProjectionSnapshot, + type DesktopRuntimeController, + type DesktopRuntimeSnapshot, +} from "@t4-code/client"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { + clusterCiAvailability, + clusterCreationTargets, + clusterGuiAvailability, + clusterOperatorAvailability, + createClusterSession, + createClusterWorkspace, + runClusterCi, +} from "../src/features/targets/cluster-operator.ts"; +import { + ClusterOperatorSection, + clusterSessionMatchesWorkspace, + prepareClusterSessionCreation, +} from "../src/features/targets/ClusterOperatorSection.tsx"; +import { deriveWorkspaceData } from "../src/platform/live-workspace.ts"; + +const HOST = "cluster-host"; +const TARGET = "cluster-target"; +const workspace: WorkspaceInfrastructureProjection = { + id: "workspace-a", + displayName: "Release train", + phase: "Ready", + retentionPolicy: "Retain", + storageClass: "t4-workspaces-rwx", + capacity: "20Gi", + accessMode: "ReadWriteMany", + revision: revision("workspace-r2"), + condition: { + type: "StorageReady", + status: "True", + reason: "Bound", + message: "The RWX claim is bound.", + observedGeneration: 2, + }, +}; + +function snapshot(options: { + readonly enabled?: boolean; + readonly connected?: boolean; + readonly features?: readonly string[]; + readonly capabilities?: readonly string[]; + readonly session?: SessionRef; + readonly workspace?: WorkspaceInfrastructureProjection; +} = {}): DesktopRuntimeSnapshot { + const projection = createProjectionSnapshot(); + return { + version: 1, + integration: { kind: "omp", displayName: "OMP", level: "first-party" }, + platform: "linux", + desktopVersion: "test", + startState: "started", + clusterOperatorEnabled: options.enabled ?? false, + targets: new Map([[TARGET, { targetId: TARGET, label: "Cluster", kind: "remote", state: options.connected === false ? "disconnected" : "connected", paired: true }]]), + connections: new Map([[TARGET, options.connected === false ? "disconnected" : "connected"]]), + targetHosts: new Map([[TARGET, HOST]]), + hosts: new Map([[HOST, { + targetId: TARGET, + hostId: HOST, + ompVersion: "17.0.5", + ompBuild: "8476f445", + appserverVersion: "1", + appserverBuild: "test", + epoch: "host-epoch", + grantedCapabilities: [...(options.capabilities ?? ["sessions.read", "sessions.manage", "sessions.prompt", "sessions.control", CI_TRIGGER_CAPABILITY])], + grantedFeatures: [...(options.features ?? [CLUSTER_OPERATOR_FEATURE])], + negotiatedLimits: {}, + authentication: "paired", + resumed: false, + }]]), + catalogs: new Map(), + settings: new Map(), + projection: { + ...projection, + workspaces: new Map(options.workspace === undefined ? [] : [[`${HOST}\u0000${options.workspace.id}`, options.workspace]]), + workspaceCursors: new Map(options.workspace === undefined ? [] : [[HOST, { epoch: "workspace-epoch", seq: 2 }]]), + sessionIndex: new Map(options.session === undefined ? [] : [[`${HOST}\u0000${String(options.session.sessionId)}`, options.session]]), + sessionIndexMetadata: new Map(options.session === undefined ? [] : [[HOST, { totalCount: 1, truncated: false }]]), + }, + runtimeErrors: [], + } as DesktopRuntimeSnapshot; +} + +const session: SessionRef = { + hostId: hostId(HOST), + sessionId: sessionId("session-a"), + project: { projectId: "cluster/workspace-a" as never, name: "Release train" }, + revision: revision("session-r3"), + title: "Ship release", + status: "active", + updatedAt: "2026-07-20T12:00:00.000Z", + liveState: { + phase: "running", + cluster: { + workspaceId: workspace.id, + phase: "Running", + gui: { state: "Ready", previewId: "preview-a" }, + }, + ci: { + provider: "woodpecker", + correlation: "exact", + repositoryId: "repo-a", + branch: "main", + ref: "refs/heads/main", + commit: "0123456789abcdef", + pipelineNumber: 42, + status: "running", + currentStage: "verify", + startedAt: "2026-07-20T12:01:00.000Z", + link: "https://ci.tailnet.ts.net/repos/repo-a/pipeline/42", + }, + }, +}; + +describe("cluster operator presentation", () => { + it("fails closed with exact disabled, transport, feature, capability, and revision reasons", () => { + expect(clusterOperatorAvailability(snapshot(), TARGET, HOST, "read")).toEqual({ + enabled: false, + reason: "Cluster operator is disabled in this app.", + }); + expect(clusterOperatorAvailability(snapshot({ enabled: true, connected: false }), TARGET, HOST, "read")).toEqual({ + enabled: false, + reason: "Reconnect this host to inspect cluster workspaces.", + }); + expect(clusterOperatorAvailability(snapshot({ enabled: true, features: [] }), TARGET, HOST, "read")).toEqual({ + enabled: false, + reason: "This host does not advertise cluster operator support.", + }); + expect(clusterOperatorAvailability(snapshot({ enabled: true, capabilities: [] }), TARGET, HOST, "read")).toEqual({ + enabled: false, + reason: "This host did not grant session read access.", + }); + expect(clusterOperatorAvailability(snapshot({ enabled: true, capabilities: ["sessions.read"] }), TARGET, HOST, "manage")).toEqual({ + enabled: false, + reason: "This host did not grant workspace and session management.", + }); + expect(clusterOperatorAvailability(snapshot({ enabled: true, capabilities: ["sessions.read", "sessions.manage"] }), TARGET, HOST, "ci", revision("session-r3"))).toEqual({ + enabled: false, + reason: "This host did not grant CI trigger access.", + }); + expect( + clusterOperatorAvailability( + snapshot({ enabled: true, capabilities: ["sessions.read", CI_TRIGGER_CAPABILITY] }), + TARGET, + HOST, + "ci", + revision("session-r3"), + ), + ).toEqual({ enabled: true }); + expect(clusterOperatorAvailability(snapshot({ enabled: true }), TARGET, HOST, "ci")).toEqual({ + enabled: false, + reason: "Waiting for the latest session revision.", + }); + }); + + it("binds every operation and creation choice to the advertised target and host", () => { + const base = snapshot({ enabled: true }); + const otherHost = "cluster-host-b"; + const otherTarget = "cluster-target-b"; + const multiHost = { + ...base, + targets: new Map(base.targets).set(otherTarget, { + targetId: otherTarget, + label: "Alpha cluster", + kind: "remote", + state: "connected", + paired: true, + }), + connections: new Map(base.connections).set(otherTarget, "connected"), + targetHosts: new Map(base.targetHosts).set(otherTarget, otherHost), + hosts: new Map(base.hosts).set(otherHost, { + ...base.hosts.get(HOST)!, + targetId: otherTarget, + hostId: otherHost, + }), + } as DesktopRuntimeSnapshot; + + expect(clusterCreationTargets(multiHost)).toEqual([ + { targetId: otherTarget, hostId: otherHost, label: "Alpha cluster" }, + { targetId: TARGET, hostId: HOST, label: "Cluster" }, + ]); + expect(clusterOperatorAvailability(multiHost, TARGET, otherHost, "manage")).toEqual({ + enabled: false, + reason: "This cluster host is no longer bound to the selected target.", + }); + }); + + it("disables CI and GUI with capability, negotiation, correlation, and host reasons", () => { + expect( + clusterCiAvailability( + snapshot({ enabled: true }), + TARGET, + HOST, + revision("session-r3"), + { ...session.liveState!.ci!, correlation: "unknown" }, + ), + ).toEqual({ + enabled: false, + reason: "CI correlation is unknown; a run cannot be triggered.", + }); + expect( + clusterGuiAvailability(snapshot({ enabled: true }), TARGET, HOST, session.liveState!.cluster!.gui), + ).toEqual({ + enabled: false, + reason: "This host does not advertise browser preview control.", + }); + expect( + clusterGuiAvailability( + snapshot({ + enabled: true, + features: [CLUSTER_OPERATOR_FEATURE, "preview.control"], + capabilities: ["sessions.read"], + }), + TARGET, + HOST, + session.liveState!.cluster!.gui, + ), + ).toEqual({ + enabled: false, + reason: "This host does not permit browser preview reads.", + }); + expect( + clusterGuiAvailability( + snapshot({ + enabled: true, + features: [CLUSTER_OPERATOR_FEATURE, "preview.control"], + capabilities: ["sessions.read", "preview.read"], + }), + TARGET, + HOST, + session.liveState!.cluster!.gui, + ), + ).toEqual({ + enabled: false, + reason: "This host does not permit browser preview control.", + }); + expect( + clusterGuiAvailability( + snapshot({ + enabled: true, + features: [CLUSTER_OPERATOR_FEATURE, "preview.control"], + capabilities: ["sessions.read", "preview.read", "preview.control"], + }), + TARGET, + HOST, + { state: "Failed", reason: "GUI pod did not become ready." }, + ), + ).toEqual({ enabled: false, reason: "GUI pod did not become ready." }); + expect( + clusterGuiAvailability( + snapshot({ + enabled: true, + features: [CLUSTER_OPERATOR_FEATURE, "preview.control"], + capabilities: ["sessions.read", "preview.read", "preview.control"], + }), + TARGET, + HOST, + session.liveState!.cluster!.gui, + ), + ).toEqual({ enabled: true }); + }); + + it("derives infrastructure, CI, and GUI truth from canonical projections", () => { + const data = deriveWorkspaceData(snapshot({ enabled: true, workspace, session })); + + expect(data.clusterWorkspaces).toEqual([{ hostId: HOST, targetId: TARGET, infrastructure: workspace }]); + expect(data.sessions[0]).toMatchObject({ + cluster: session.liveState?.cluster, + ci: session.liveState?.ci, + }); + expect(data.sessions[0]?.ci?.currentStage).toBe("verify"); + expect(data.sessions[0]?.cluster?.gui).toEqual({ state: "Ready", previewId: "preview-a" }); + }); + + it("host-qualifies colliding workspace ids and renders exact disabled action reasons", () => { + const denied = snapshot({ + enabled: true, + capabilities: ["sessions.read", "sessions.manage"], + workspace, + session, + }); + const projected = deriveWorkspaceData(denied).sessions[0]!; + expect(clusterSessionMatchesWorkspace(projected, session, HOST, workspace.id)).toBe(true); + expect( + clusterSessionMatchesWorkspace( + projected, + { ...session, hostId: hostId("other-cluster-host") }, + HOST, + workspace.id, + ), + ).toBe(false); + + const markup = renderToStaticMarkup( + denied } as unknown as DesktopRuntimeController} + onOpenPreview={() => undefined} + onOpenSession={() => undefined} + snapshot={denied} + />, + ); + expect(markup).toContain("GUI: This host does not advertise browser preview control."); + expect(markup).toContain("CI: This host did not grant CI trigger access."); + expect(markup).toMatch(/]*disabled=""[^>]*>Open GUI<\/button>/u); + expect(markup).toMatch(/]*disabled=""[^>]*>Run CI<\/button>/u); + expect(markup).toContain( + "Woodpecker correlation is unavailable because this host did not grant CI trigger access.", + ); + expect(markup).toMatch(/]*disabled=""[^>]*>/u); + expect(markup).toMatch(/]*>Create session with GUI<\/button>/u); + expect(markup).not.toMatch( + /]*\sdisabled=""[^>]*>Create session with GUI<\/button>/u, + ); + const failedSession: SessionRef = { + ...session, + liveState: { + ...session.liveState!, + cluster: { + ...session.liveState!.cluster!, + gui: { state: "Failed", reason: "GUI pod did not become ready." }, + }, + }, + }; + const failed = snapshot({ + enabled: true, + features: [CLUSTER_OPERATOR_FEATURE, "preview.control"], + capabilities: [ + "sessions.read", + "sessions.manage", + CI_TRIGGER_CAPABILITY, + "preview.read", + "preview.control", + ], + workspace, + session: failedSession, + }); + const failedMarkup = renderToStaticMarkup( + failed } as unknown as DesktopRuntimeController} + onOpenPreview={() => undefined} + onOpenSession={() => undefined} + snapshot={failed} + />, + ); + expect(failedMarkup.match(/GUI pod did not become ready\./gu)).toHaveLength(2); + }); + + it("projects no cluster truth without local opt-in and sessions.read", () => { + for (const denied of [ + snapshot({ workspace, session }), + snapshot({ enabled: true, capabilities: [], workspace, session }), + ]) { + const data = deriveWorkspaceData(denied); + expect(data.clusterWorkspaces).toEqual([]); + expect(data.sessions[0]).not.toHaveProperty("cluster"); + expect(data.sessions[0]).not.toHaveProperty("ci"); + } + const defaultOff = snapshot({ workspace, session }); + expect( + renderToStaticMarkup( + defaultOff } as unknown as DesktopRuntimeController} + onOpenPreview={() => undefined} + onOpenSession={() => undefined} + snapshot={defaultOff} + />, + ), + ).toBe(""); + }); + + it("keeps session creation unchanged when Woodpecker correlation is empty", async () => { + const command = vi.fn( + async (..._args: Parameters) => ({ + accepted: true, + result: {}, + }), + ); + const controller = { + command, + getSnapshot: () => snapshot({ enabled: true, workspace }), + } as unknown as DesktopRuntimeController; + const preparation = prepareClusterSessionCreation(workspace.id, " Ship release ", { + repositoryId: "", + ref: "", + commit: "", + }); + + expect(preparation).toEqual({ + args: { + workspaceId: workspace.id, + title: "Ship release", + runtimeProfile: "default", + guiEnabled: true, + }, + error: null, + invalidField: null, + }); + if (preparation.args === null) throw new Error(preparation.error); + await createClusterSession(controller, TARGET, HOST, preparation.args); + expect(command).toHaveBeenCalledWith(TARGET, { + hostId: HOST, + command: "session.create", + args: { + workspaceId: workspace.id, + title: "Ship release", + runtimeProfile: "default", + guiEnabled: true, + }, + }); + + const markup = renderToStaticMarkup( + undefined} + onOpenSession={() => undefined} + snapshot={snapshot({ enabled: true, workspace })} + />, + ); + expect(markup).toContain("Woodpecker correlation"); + expect(markup).toContain("Repository ID"); + expect(markup).toMatch(/maxlength="128"/iu); + expect(markup).toMatch(/maxlength="256"/iu); + expect(markup).toMatch(/maxlength="40"/iu); + }); + + it("transmits an exact valid Woodpecker correlation on session creation", async () => { + const command = vi.fn( + async (..._args: Parameters) => ({ + accepted: true, + result: {}, + }), + ); + const controller = { + command, + getSnapshot: () => snapshot({ enabled: true, workspace }), + } as unknown as DesktopRuntimeController; + const commit = "0123456789abcdef0123456789abcdef01234567"; + const preparation = prepareClusterSessionCreation(workspace.id, undefined, { + repositoryId: "t4-code", + ref: "refs/heads/agent/t4-cluster-operator", + commit, + }); + + expect(preparation).toEqual({ + args: { + workspaceId: workspace.id, + runtimeProfile: "default", + guiEnabled: true, + ci: { + provider: "woodpecker", + repositoryId: "t4-code", + ref: "refs/heads/agent/t4-cluster-operator", + commit, + }, + }, + error: null, + invalidField: null, + }); + if (preparation.args === null) throw new Error(preparation.error); + await createClusterSession(controller, TARGET, HOST, preparation.args); + expect(command).toHaveBeenCalledWith(TARGET, { + hostId: HOST, + command: "session.create", + args: { + workspaceId: workspace.id, + runtimeProfile: "default", + guiEnabled: true, + ci: { + provider: "woodpecker", + repositoryId: "t4-code", + ref: "refs/heads/agent/t4-cluster-operator", + commit, + }, + }, + }); + }); + + it("rejects partial, non-exact commit, and wire-bounds failures before submission", async () => { + const command = vi.fn( + async (..._args: Parameters) => ({ + accepted: true, + result: {}, + }), + ); + const controller = { + command, + getSnapshot: () => snapshot({ enabled: true, workspace }), + } as unknown as DesktopRuntimeController; + const commit = "0123456789abcdef0123456789abcdef01234567"; + const attempts = [ + { + draft: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "" }, + error: + "Enter repository ID, ref, and commit together, or leave all three Woodpecker correlation fields empty.", + invalidField: "correlation", + }, + { + draft: { repositoryId: "t4-code", ref: "refs/heads/main", commit: commit.slice(1) }, + error: "Woodpecker commit must be exactly 40 lowercase hexadecimal characters.", + invalidField: "commit", + }, + { + draft: { repositoryId: "t4-code", ref: "refs/heads/main", commit: commit.toUpperCase() }, + error: "Woodpecker commit must be exactly 40 lowercase hexadecimal characters.", + invalidField: "commit", + }, + { + draft: { repositoryId: "t4-code", ref: "refs/heads/main", commit: `g${commit.slice(1)}` }, + error: "Woodpecker commit must be exactly 40 lowercase hexadecimal characters.", + invalidField: "commit", + }, + { + draft: { repositoryId: "a".repeat(129), ref: "refs/heads/main", commit }, + error: + "Woodpecker repository ID must be 1 to 128 UTF-8 bytes, start with a letter or number, and otherwise use only letters, numbers, '.', '_', ':', '/', or '-' without '..' or '://'.", + invalidField: "repositoryId", + }, + { + draft: { repositoryId: "t4-code", ref: "é".repeat(129), commit }, + error: "Woodpecker ref must be 1 to 256 UTF-8 bytes and contain no control characters.", + invalidField: "ref", + }, + ] as const; + + for (const attempt of attempts) { + const preparation = prepareClusterSessionCreation(workspace.id, undefined, attempt.draft); + expect(preparation).toEqual({ + args: null, + error: attempt.error, + invalidField: attempt.invalidField, + }); + if (preparation.args !== null) { + await createClusterSession(controller, TARGET, HOST, preparation.args); + } + } + expect(command).not.toHaveBeenCalled(); + }); + + it("sends only allowlisted workspace, session, and CI arguments", async () => { + const command = vi.fn(async (..._args: Parameters) => ({ accepted: true, result: {} })); + const controller = { command, getSnapshot: () => snapshot({ enabled: true, workspace, session }) } as unknown as DesktopRuntimeController; + + await createClusterWorkspace(controller, TARGET, HOST, { + displayName: "Release train", + retentionPolicy: "Retain", + capacity: "20Gi", + repository: { repositoryId: "repo-a", ref: "refs/heads/main", commit: "0123456789abcdef" }, + }); + await createClusterSession(controller, TARGET, HOST, { + workspaceId: workspace.id, + title: "Ship release", + runtimeProfile: "default", + guiEnabled: true, + ci: { provider: "woodpecker", repositoryId: "repo-a", ref: "refs/heads/main", commit: "0123456789abcdef" }, + }); + await runClusterCi(controller, TARGET, HOST, "session-a", revision("session-r3"), { + provider: "woodpecker", + action: "run", + repositoryId: "repo-a", + ref: "refs/heads/main", + commit: "0123456789abcdef", + }); + + expect(command.mock.calls.map(([targetId, intent]) => ({ targetId, intent }))).toEqual([ + { targetId: TARGET, intent: { hostId: HOST, command: "workspace.create", args: expect.objectContaining({ displayName: "Release train", capacity: "20Gi" }) } }, + { targetId: TARGET, intent: { hostId: HOST, command: "session.create", args: expect.objectContaining({ workspaceId: workspace.id, guiEnabled: true }) } }, + { targetId: TARGET, intent: { hostId: HOST, sessionId: "session-a", command: "ci.run", expectedRevision: "session-r3", args: expect.objectContaining({ provider: "woodpecker", action: "run", repositoryId: "repo-a" }) } }, + ]); + const serialized = JSON.stringify(command.mock.calls); + expect(serialized).not.toMatch(/token|secret|kubeconfig|namespace|image|url/iu); + }); +}); diff --git a/apps/web/test/live-settings-targets.test.ts b/apps/web/test/live-settings-targets.test.ts index 6cb4f0fd..9f4c66a7 100644 --- a/apps/web/test/live-settings-targets.test.ts +++ b/apps/web/test/live-settings-targets.test.ts @@ -32,6 +32,7 @@ import { capabilityDiff, EMPTY_TARGET_DRAFT, pairCommandForTarget, + selectTargetCapabilityGroups, validateTargetDraft, } from "../src/features/targets/model.ts"; import { @@ -655,6 +656,59 @@ describe("target add validation", () => { }); }); +describe("target capability groups", () => { + it("hides cluster sessions unless the app explicitly opts in", () => { + for (const enabled of [undefined, false]) { + expect(selectTargetCapabilityGroups(enabled).map((group) => group.id)).toEqual([ + "observe", + "control", + "shell", + "files", + "settings", + ]); + } + }); + + it("exposes the exact cluster-session capabilities for an explicit opt-in", () => { + const groups = selectTargetCapabilityGroups(true); + expect(groups.map((group) => group.id)).toEqual([ + "observe", + "control", + "shell", + "files", + "settings", + "cluster", + ]); + const cluster = groups.find((group) => group.id === "cluster"); + expect(cluster?.label).toBe("Cluster sessions"); + expect(cluster?.capabilities).toEqual([ + "ci.trigger", + "preview.read", + "preview.control", + "preview.input", + ]); + }); + + it("orders observe, control, and cluster capabilities without extras", () => { + const capabilities = capabilitiesForGroups(new Set(["cluster", "control", "observe"])); + expect(capabilities).toEqual([ + "sessions.read", + "catalog.read", + "config.read", + "audit.read", + "sessions.prompt", + "sessions.control", + "sessions.manage", + "agents.control", + "ci.trigger", + "preview.read", + "preview.control", + "preview.input", + ]); + expect(pairCommandForTarget(capabilities).capabilities).toEqual(capabilities); + }); +}); + describe("pair command", () => { it("emits one --capability flag per requested capability from the selected groups", () => { const requested = capabilitiesForGroups(new Set(["observe", "settings", "control"])); diff --git a/apps/web/test/panes-live.test.ts b/apps/web/test/panes-live.test.ts index d87e517c..e6e431cc 100644 --- a/apps/web/test/panes-live.test.ts +++ b/apps/web/test/panes-live.test.ts @@ -159,11 +159,11 @@ function fileFrame(path: string, content?: string, session = SESSION): FileFrame }; } -function snapshotFrame(revision: string): SessionSnapshotFrame { +function snapshotFrame(revision: string, seq = 0): SessionSnapshotFrame { return { v: PROTOCOL_VERSION, type: "snapshot", - cursor: { epoch: "epoch-1", seq: 0 }, + cursor: { epoch: "epoch-1", seq }, revision: brandRevision(revision), hostId: brandHostId(HOST), sessionId: brandSessionId(SESSION), @@ -1030,7 +1030,7 @@ describe("live pane actions", () => { fake.setProjection( project( - [snapshotFrame("rev-4"), fileFrame("src/app.ts", "const value = 2;\n")], + [snapshotFrame("rev-4", 1), fileFrame("src/app.ts", "const value = 2;\n")], base, ), ); diff --git a/apps/web/test/session-management.test.ts b/apps/web/test/session-management.test.ts index d5bd44bb..f65876ac 100644 --- a/apps/web/test/session-management.test.ts +++ b/apps/web/test/session-management.test.ts @@ -194,6 +194,9 @@ class FakeManagementController { ]) as DesktopRuntimeSnapshot["projection"]["sessionIndexMetadata"], sessionRefArrivalOrdinals: new Map(), sessionDeltaCursors: new Map(), + sessionInventoryCursors: new Map(), + workspaces: new Map(), + workspaceCursors: new Map(), lru: [], freshness: "fresh", arrivalOrdinal: 0, diff --git a/apps/web/test/session-navigation.test.ts b/apps/web/test/session-navigation.test.ts index 769bc355..92936967 100644 --- a/apps/web/test/session-navigation.test.ts +++ b/apps/web/test/session-navigation.test.ts @@ -1,6 +1,12 @@ +import type { DesktopRuntimeSnapshot } from "@t4-code/client"; + import { describe, expect, it } from "vite-plus/test"; -import { resolveSessionManagementNavigation } from "../src/features/session-runtime/session-navigation.ts"; +import { + previewSelectionForNavigation, + resolveSessionManagementNavigation, +} from "../src/features/session-runtime/session-navigation.ts"; +import { sessionViewId } from "../src/platform/live-workspace.ts"; import type { WorkspaceSession } from "../src/lib/workspace-data.ts"; function session(id: string, archived = false): WorkspaceSession { @@ -73,4 +79,47 @@ describe("session management navigation", () => { navigate: false, }); }); + + it("preserves the advertised preview id and only opts into negotiated authority", () => { + const hostId = "cluster/host"; + const sessionId = "session/gui"; + const viewId = sessionViewId(hostId, sessionId); + const snapshot = { + targetHosts: new Map([["cluster-target", hostId]]), + connections: new Map([["cluster-target", "connected"]]), + projection: { + sessions: new Map([ + [ + `${hostId}\u0000${sessionId}`, + { + previews: new Map([ + [ + "preview-a", + { + previewId: "preview-a", + authority: { + id: "omp-session", + kind: "isolated-session", + }, + }, + ], + ]), + }, + ], + ]), + }, + } as unknown as DesktopRuntimeSnapshot; + + expect(viewId).toBe("cluster%2Fhost/session%2Fgui"); + expect(previewSelectionForNavigation(snapshot, viewId, "preview-a")).toEqual({ + previewId: "preview-a", + optInKind: "isolated-session", + optInAuthorityId: "omp-session", + optIn: true, + }); + expect(previewSelectionForNavigation(snapshot, viewId, "preview-pending")).toEqual({ + previewId: "preview-pending", + optIn: false, + }); + }); }); diff --git a/apps/web/test/terminal-live-pty.test.ts b/apps/web/test/terminal-live-pty.test.ts index 3787615f..33a49305 100644 --- a/apps/web/test/terminal-live-pty.test.ts +++ b/apps/web/test/terminal-live-pty.test.ts @@ -131,11 +131,11 @@ function catalogFrame(items: readonly CatalogItemSpec[]): Record { it("refused mutation leases send nothing and do not replay after updates", async () => { const h = await harness(); const { session } = await openLive(h); - h.shell.emitFrame({ targetId: TARGET, frame: sessionSnapshotFrame("rev-2") }); + h.shell.emitFrame({ targetId: TARGET, frame: sessionSnapshotFrame("rev-2", 2) }); await settle(); h.shell.leaseRefused = true; session.write("blocked"); @@ -831,7 +831,7 @@ describe("input, resize, close", () => { expect(h.shell.inputs).toHaveLength(0); expect(h.shell.resizes).toHaveLength(0); expect(h.shell.closes).toHaveLength(0); - h.shell.emitFrame({ targetId: TARGET, frame: sessionSnapshotFrame("rev-3") }); + h.shell.emitFrame({ targetId: TARGET, frame: sessionSnapshotFrame("rev-3", 3) }); await settle(); expect(h.shell.inputs).toHaveLength(0); expect(h.shell.resizes).toHaveLength(0); diff --git a/cluster/images/cluster-server/Dockerfile b/cluster/images/cluster-server/Dockerfile new file mode 100644 index 00000000..b3334053 --- /dev/null +++ b/cluster/images/cluster-server/Dockerfile @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1.7 +FROM docker.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 AS dependencies +WORKDIR /opt/t4 +COPY . /opt/t4 +RUN corepack enable \ + && pnpm install --frozen-lockfile --ignore-scripts + +FROM docker.io/oven/bun:1.3.14-debian@sha256:9dba1a1b43ce28c9d7931bfc4eb00feb63b0114720a0277a8f939ae4dfc9db6f +USER root +RUN rm -f /etc/apt/sources.list.d/* \ + && printf '%s\n' \ + 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/20250721T000000Z bookworm main' \ + 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/20250721T000000Z bookworm-security main' \ + > /etc/apt/sources.list \ + && apt-get -o Acquire::Check-Valid-Until=false update \ + && apt-get install -y --no-install-recommends ca-certificates curl tini \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /opt/t4 +COPY --from=dependencies /opt/t4 /opt/t4 +ENV NODE_ENV=production \ + HOME=/tmp \ + T4_CLUSTER_SERVER_PORT=8080 \ + T4_CLUSTER_ADMIN_PORT=9090 +ARG SOURCE_REPOSITORY=https://github.com/LycaonLLC/t4-code +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.title="T4 stateless cluster server" +USER 10001:10001 +EXPOSE 8080 9090 +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/bun", "packages/cluster-server/src/main.ts"] diff --git a/cluster/images/controller/Dockerfile b/cluster/images/controller/Dockerfile new file mode 100644 index 00000000..9b974701 --- /dev/null +++ b/cluster/images/controller/Dockerfile @@ -0,0 +1,20 @@ +# syntax=docker/dockerfile:1.7 +FROM docker.io/library/golang:1.24.5-bookworm@sha256:ef8c5c733079ac219c77edab604c425d748c740d8699530ea6aced9de79aea40 AS build +WORKDIR /src/packages/cluster-operator +COPY packages/cluster-operator/go.mod ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY packages/cluster-operator/ ./ +ARG TARGETOS +ARG TARGETARCH +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -ldflags='-s -w -buildid=' -o /out/t4-cluster-operator ./cmd/manager + +FROM gcr.io/distroless/static-debian12:nonroot@sha256:7a2bd171a18bdd39a4729600d0dca5f16e779d41156a6908b4f8a9a289e76d92 +ARG SOURCE_REPOSITORY=https://github.com/LycaonLLC/t4-code +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.title="T4 cluster operator" +COPY --from=build --chown=65532:65532 /out/t4-cluster-operator /usr/local/bin/t4-cluster-operator +USER 65532:65532 +ENTRYPOINT ["/usr/local/bin/t4-cluster-operator"] diff --git a/cluster/images/session-runtime/Dockerfile b/cluster/images/session-runtime/Dockerfile new file mode 100644 index 00000000..6ce1a7e6 --- /dev/null +++ b/cluster/images/session-runtime/Dockerfile @@ -0,0 +1,72 @@ +# syntax=docker/dockerfile:1.7 + +FROM docker.io/library/rust:1.86-slim-bookworm@sha256:57d415bbd61ce11e2d5f73de068103c7bd9f3188dc132c97cef4a8f62989e944 AS omp-build +RUN rm -f /etc/apt/sources.list.d/* \ + && printf '%s\n' \ + 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/20250721T000000Z bookworm main' \ + 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/20250721T000000Z bookworm-security main' \ + > /etc/apt/sources.list \ + && apt-get -o Acquire::Check-Valid-Until=false update \ + && apt-get install -y --no-install-recommends ca-certificates git pkg-config libssl-dev \ + && rm -rf /var/lib/apt/lists/* +COPY --from=docker.io/oven/bun:1.3.14-debian@sha256:9dba1a1b43ce28c9d7931bfc4eb00feb63b0114720a0277a8f939ae4dfc9db6f /usr/local/bin/bun /usr/local/bin/bun +WORKDIR /opt/omp +RUN git init \ + && git remote add origin https://github.com/lyc-aon/oh-my-pi.git \ + && git fetch --depth=1 origin "refs/tags/t4code-17.0.5-appserver-10" \ + && test "$(git rev-parse FETCH_HEAD)" = "8476f4451ed95c5d5401785d279a93d3c659fac4" \ + && git checkout --detach "8476f4451ed95c5d5401785d279a93d3c659fac4" +RUN bun install --frozen-lockfile --ignore-scripts \ + && bun --cwd=packages/natives run build \ + && bun --cwd=packages/coding-agent run gen:tool-views \ + && grep -R -F -q 't4-omp-authority/1' packages/coding-agent \ + && rm -rf .git + +FROM docker.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 AS t4-build +WORKDIR /opt/t4 +COPY . /opt/t4 +RUN corepack enable \ + && pnpm install --frozen-lockfile --ignore-scripts + +FROM docker.io/oven/bun:1.3.14-debian@sha256:9dba1a1b43ce28c9d7931bfc4eb00feb63b0114720a0277a8f939ae4dfc9db6f +USER root +RUN rm -f /etc/apt/sources.list.d/* \ + && printf '%s\n' \ + 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/20250721T000000Z bookworm main' \ + 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/20250721T000000Z bookworm-security main' \ + > /etc/apt/sources.list \ + && apt-get -o Acquire::Check-Valid-Until=false update \ + && apt-get install -y --no-install-recommends \ + bash ca-certificates chromium fluxbox git openssh-client python3 ripgrep tini xvfb \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /opt/t4/bin /opt/omp /workspace /run/t4 \ + && chown -R 10001:10001 /workspace /run/t4 +COPY --from=omp-build /opt/omp /opt/omp +COPY --from=t4-build /opt/t4 /opt/t4 +COPY cluster/images/session-runtime/session-entrypoint.sh /opt/t4/bin/session-entrypoint +RUN printf '%s\n' \ + '#!/usr/bin/env bash' \ + 'set -euo pipefail' \ + 'export PI_ROOT=/opt/omp' \ + 'exec /usr/local/bin/bun /opt/omp/packages/coding-agent/src/cli.ts "$@"' \ + > /opt/t4/bin/omp \ + && chmod 0755 /opt/t4/bin/omp /opt/t4/bin/session-entrypoint \ + && chown -R 10001:10001 /opt/t4 /opt/omp +ENV PI_ROOT=/opt/omp \ + T4_OMP_EXECUTABLE=/opt/t4/bin/omp \ + T4_WORKSPACE_ROOT=/workspace \ + T4_HOST_PORT=8787 \ + DISPLAY=:99 \ + HOME=/workspace/.t4/runtime \ + XDG_RUNTIME_DIR=/run/t4 +ARG SOURCE_REPOSITORY=https://github.com/LycaonLLC/t4-code +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.title="T4 OMP session runtime" \ + io.t4.omp.version="17.0.5" \ + io.t4.omp.tag="t4code-17.0.5-appserver-10" \ + io.t4.omp.commit="8476f4451ed95c5d5401785d279a93d3c659fac4" \ + io.t4.omp.authority="t4-omp-authority/1" \ + io.t4.gui.display="Xvfb" +USER 10001:10001 +EXPOSE 8787 +ENTRYPOINT ["/usr/bin/tini", "--", "/opt/t4/bin/session-entrypoint"] diff --git a/cluster/images/session-runtime/session-entrypoint.sh b/cluster/images/session-runtime/session-entrypoint.sh new file mode 100755 index 00000000..43651f05 --- /dev/null +++ b/cluster/images/session-runtime/session-entrypoint.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +: "${T4_SESSION_STATE_ROOT:?T4_SESSION_STATE_ROOT is required}" +: "${T4_SESSION_NAME:?T4_SESSION_NAME is required}" +: "${T4_AUTHORITY_STATE_DIR:?T4_AUTHORITY_STATE_DIR is required}" +: "${T4_BROWSER_STATE_DIR:?T4_BROWSER_STATE_DIR is required}" +: "${T4_CLUSTER_SERVER_SERVICE_ACCOUNT:?T4_CLUSTER_SERVER_SERVICE_ACCOUNT is required}" +export T4_OMP_CONFIG_SOURCE_DIR="${T4_OMP_CONFIG_SOURCE_DIR:-/run/t4-omp-config-source}" +export T4_OMP_ALLOW_UNAUTHENTICATED="${T4_OMP_ALLOW_UNAUTHENTICATED:-false}" +export T4_KUBERNETES_TOKEN_PATH="${T4_KUBERNETES_TOKEN_PATH:-/var/run/secrets/kubernetes.io/serviceaccount/token}" +export T4_KUBERNETES_CA_PATH="${T4_KUBERNETES_CA_PATH:-/var/run/secrets/kubernetes.io/serviceaccount/ca.crt}" +export T4_KUBERNETES_NAMESPACE_PATH="${T4_KUBERNETES_NAMESPACE_PATH:-/var/run/secrets/kubernetes.io/serviceaccount/namespace}" +for projected_file in "${T4_KUBERNETES_TOKEN_PATH}" "${T4_KUBERNETES_CA_PATH}" "${T4_KUBERNETES_NAMESPACE_PATH}"; do + [[ -f "${projected_file}" && -r "${projected_file}" ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"kubernetes_api_projection"}' >&2; exit 64; } +done + +if [[ ! "${T4_SESSION_STATE_ROOT}" =~ ^/workspace/\.t4/sessions/[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ ]]; then + echo '{"component":"session-runtime","result":"invalid_config","condition":"session_state_path"}' >&2 + exit 64 +fi +[[ "${T4_SESSION_NAME}" =~ ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"session_name"}' >&2; exit 64; } +[[ "${T4_AUTHORITY_STATE_DIR}" == "${T4_SESSION_STATE_ROOT}/authority" ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"authority_state_path"}' >&2; exit 64; } +[[ "${T4_BROWSER_STATE_DIR}" == "${T4_SESSION_STATE_ROOT}/browser" ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"browser_state_path"}' >&2; exit 64; } + +case "${T4_OMP_ALLOW_UNAUTHENTICATED}" in + true|false) ;; + *) echo '{"component":"session-runtime","result":"invalid_config","condition":"omp_authentication_mode"}' >&2; exit 64 ;; +esac + +models_source="${T4_OMP_CONFIG_SOURCE_DIR}/models.yml" +settings_source="${T4_OMP_CONFIG_SOURCE_DIR}/config.yml" +[[ -f "${models_source}" && -r "${models_source}" && -s "${models_source}" ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"omp_models"}' >&2; exit 64; } +[[ -f "${settings_source}" && -r "${settings_source}" && -s "${settings_source}" ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"omp_settings"}' >&2; exit 64; } + +if [[ "${T4_OMP_ALLOW_UNAUTHENTICATED}" == "false" ]]; then + T4_OMP_CREDENTIAL_KEY="${1:-}" + [[ -n "${T4_OMP_CREDENTIAL_KEY}" ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"omp_credential_key"}' >&2; exit 64; } + [[ "${T4_OMP_CREDENTIAL_KEY}" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"omp_credential_key"}' >&2; exit 64; } + case "${T4_OMP_CREDENTIAL_KEY}" in + T4_*|OMP_*|PI_*|XDG_*|LD_*|HOME|DISPLAY|PATH|BASH_ENV|ENV|SHELLOPTS|NODE_OPTIONS|BUN_OPTIONS) + echo '{"component":"session-runtime","result":"invalid_config","condition":"omp_credential_key"}' >&2 + exit 64 + ;; + esac + [[ -v "${T4_OMP_CREDENTIAL_KEY}" ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"omp_credential"}' >&2; exit 64; } + credential_value="${!T4_OMP_CREDENTIAL_KEY}" + [[ -n "${credential_value}" ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"omp_credential"}' >&2; exit 64; } + unset credential_value +fi + +export HOME="${T4_SESSION_STATE_ROOT}/home" +export PI_CODING_AGENT_DIR="${HOME}/.omp/profiles/${T4_SESSION_NAME}/agent" +mkdir -p "${T4_AUTHORITY_STATE_DIR}" "${T4_BROWSER_STATE_DIR}" /run/t4 /tmp/t4 +install -d -m 0700 "${HOME}" "${PI_CODING_AGENT_DIR}" +models_private="${PI_CODING_AGENT_DIR}/.models.yml.new" +settings_private="${PI_CODING_AGENT_DIR}/.config.yml.new" +trap 'rm -f "${models_private}" "${settings_private}"' EXIT +install -m 0600 "${models_source}" "${models_private}" +install -m 0600 "${settings_source}" "${settings_private}" +mv -f "${models_private}" "${PI_CODING_AGENT_DIR}/models.yml" +mv -f "${settings_private}" "${PI_CODING_AGENT_DIR}/config.yml" +trap - EXIT +export DISPLAY="${DISPLAY:-:99}" +[[ "${DISPLAY}" =~ ^:([0-9]{1,3})$ ]] || { echo '{"component":"session-runtime","result":"invalid_config","condition":"display"}' >&2; exit 64; } +display_socket="/tmp/.X11-unix/X${BASH_REMATCH[1]}" +export T4_OMP_EXECUTABLE=/opt/t4/bin/omp +export T4_WORKSPACE_ROOT=/workspace + +children=() +stop_children() { + local pid + for pid in "${children[@]:-}"; do + kill -TERM "${pid}" 2>/dev/null || true + done +} +trap stop_children TERM INT EXIT + +Xvfb "${DISPLAY}" -screen 0 1920x1080x24 -nolisten tcp -ac & +children+=("$!") +for _ in $(seq 1 50); do + [[ -S "${display_socket}" ]] && break + kill -0 "${children[0]}" 2>/dev/null || { echo '{"component":"session-runtime","result":"startup_failed","condition":"xvfb"}' >&2; exit 70; } + sleep 0.1 +done +[[ -S "${display_socket}" ]] || { echo '{"component":"session-runtime","result":"startup_timeout","condition":"xvfb"}' >&2; exit 70; } +fluxbox -display "${DISPLAY}" & +children+=("$!") + +if [[ "${T4_GUI_ENABLED:-false}" == "true" ]]; then + chromium \ + --disable-setuid-sandbox \ + --disable-background-networking \ + --disable-breakpad \ + --disable-component-update \ + --disable-default-apps \ + --disable-sync \ + --metrics-recording-only \ + --no-first-run \ + --password-store=basic \ + --remote-debugging-address=127.0.0.1 \ + --remote-debugging-port=9222 \ + --user-data-dir="${T4_BROWSER_STATE_DIR}" \ + about:blank & + children+=("$!") +fi + +/usr/local/bin/bun /opt/t4/packages/cluster-server/src/session-host-main.ts & +host_pid=$! +children+=("${host_pid}") +wait "${host_pid}" diff --git a/deploy/charts/t4-cluster/Chart.yaml b/deploy/charts/t4-cluster/Chart.yaml new file mode 100644 index 00000000..550297cc --- /dev/null +++ b/deploy/charts/t4-cluster/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: t4-cluster +description: Default-off portable T4 Kubernetes operator and stateless cluster gateway +version: 0.1.0 +appVersion: "0.1.30" +type: application +kubeVersion: ">=1.30.0-0" diff --git a/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml b/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml new file mode 100644 index 00000000..fb95b1cd --- /dev/null +++ b/deploy/charts/t4-cluster/crds/t4clusterhosts.cluster.t4.dev.yaml @@ -0,0 +1,137 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: t4clusterhosts.cluster.t4.dev +spec: + group: cluster.t4.dev + scope: Namespaced + names: + plural: t4clusterhosts + singular: t4clusterhost + kind: T4ClusterHost + listKind: T4ClusterHostList + shortNames: [t4host] + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - name: Storage + type: string + jsonPath: .spec.storageClassName + - name: Available + type: string + jsonPath: .status.conditions[?(@.type=="Available")].status + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + required: [storageClassName, runtimeProfiles] + x-kubernetes-validations: + - rule: "!has(self.ciProvider) || (self.ciProvider.configMapRef.name != '' && (has(self.ciProvider.secretRef) != has(self.ciProvider.serviceAccountAudience)))" + message: CI provider requires a ConfigMap and exactly one Secret or projected ServiceAccount identity + - rule: "!has(self.allowedOrigins) || self.allowedOrigins.all(o, o.startsWith('https://') && !o.contains('@'))" + message: allowed origins must be credential-free HTTPS origins + properties: + storageClassName: + type: string + minLength: 1 + maxLength: 253 + pattern: '^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?(\.[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?)*$' + runtimeProfiles: + type: array + minItems: 1 + maxItems: 16 + uniqueItems: true + items: + type: string + minLength: 1 + maxLength: 64 + pattern: '^[a-z0-9]([-._a-z0-9]{0,62}[a-z0-9])?$' + ciProvider: + type: object + required: [configMapRef] + properties: + secretRef: + type: object + required: [name] + properties: + name: + type: string + minLength: 1 + maxLength: 253 + pattern: '^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$' + serviceAccountAudience: + type: string + minLength: 1 + maxLength: 253 + pattern: '^[A-Za-z0-9][A-Za-z0-9._:/-]*$' + configMapRef: + type: object + required: [name] + properties: + name: + type: string + minLength: 1 + maxLength: 253 + pattern: '^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$' + allowedOrigins: + type: array + maxItems: 32 + uniqueItems: true + items: + type: string + minLength: 9 + maxLength: 256 + pattern: '^https://[^/?#]+$' + status: + type: object + properties: + observedGeneration: + type: integer + format: int64 + minimum: 0 + conditions: + type: array + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [type] + items: + type: object + required: [type, status, reason, message, lastTransitionTime] + properties: + type: + type: string + minLength: 1 + maxLength: 32 + enum: [Available, StorageReady, CIReady] + status: + type: string + enum: ['True', 'False', Unknown] + observedGeneration: + type: integer + format: int64 + minimum: 0 + lastTransitionTime: + type: string + format: date-time + maxLength: 64 + reason: + type: string + minLength: 1 + maxLength: 64 + pattern: '^[A-Za-z][A-Za-z0-9]*$' + message: + type: string + maxLength: 512 diff --git a/deploy/charts/t4-cluster/crds/t4sessions.cluster.t4.dev.yaml b/deploy/charts/t4-cluster/crds/t4sessions.cluster.t4.dev.yaml new file mode 100644 index 00000000..107b0839 --- /dev/null +++ b/deploy/charts/t4-cluster/crds/t4sessions.cluster.t4.dev.yaml @@ -0,0 +1,147 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: t4sessions.cluster.t4.dev +spec: + group: cluster.t4.dev + scope: Namespaced + names: + plural: t4sessions + singular: t4session + kind: T4Session + listKind: T4SessionList + shortNames: [t4sess] + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - name: Phase + type: string + jsonPath: .status.phase + - name: Pod + type: string + jsonPath: .status.podName + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + required: [hostRef, workspaceRef, title, runtimeProfile] + x-kubernetes-validations: + - rule: "!has(self.ci) || (self.ci.repositoryId != '' && self.ci.ref != '' && self.ci.commit != '')" + message: CI metadata must contain an allowlisted repository id, ref, and commit + - rule: "!has(self.initialPromptSecretRef) || self.initialPromptSecretRef.name != ''" + message: initial prompt Secret reference must name a Secret + properties: + hostRef: + type: string + minLength: 1 + maxLength: 253 + pattern: '^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$' + x-kubernetes-validations: + - rule: self == oldSelf + message: hostRef is immutable + workspaceRef: + type: string + minLength: 1 + maxLength: 253 + pattern: '^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$' + title: + type: string + minLength: 1 + maxLength: 128 + runtimeProfile: + type: string + minLength: 1 + maxLength: 64 + pattern: '^[a-z0-9]([a-z0-9._-]*[a-z0-9])?$' + initialPromptSecretRef: + type: object + required: [name] + properties: + name: + type: string + minLength: 1 + maxLength: 253 + pattern: '^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$' + guiEnabled: + type: boolean + default: false + ci: + type: object + required: [repositoryId, ref, commit] + properties: + repositoryId: + type: string + minLength: 1 + maxLength: 128 + pattern: '^[A-Za-z0-9][A-Za-z0-9._:/-]*$' + ref: + type: string + minLength: 1 + maxLength: 256 + commit: + type: string + minLength: 7 + maxLength: 64 + pattern: '^[0-9a-fA-F]+$' + status: + type: object + properties: + observedGeneration: + type: integer + format: int64 + minimum: 0 + podName: + type: string + minLength: 1 + maxLength: 63 + serviceName: + type: string + minLength: 1 + maxLength: 63 + phase: + type: string + enum: [Pending, Running, Failed, Terminating, Unknown] + conditions: + type: array + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [type] + items: + type: object + required: [type, status, reason, message, lastTransitionTime] + properties: + type: + type: string + enum: [Available, WorkspaceReady, HostReady, RuntimeConfigured] + status: + type: string + enum: ['True', 'False', Unknown] + observedGeneration: + type: integer + format: int64 + minimum: 0 + lastTransitionTime: + type: string + format: date-time + maxLength: 64 + reason: + type: string + minLength: 1 + maxLength: 64 + pattern: '^[A-Za-z][A-Za-z0-9]*$' + message: + type: string + maxLength: 512 diff --git a/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml b/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml new file mode 100644 index 00000000..240f9fc3 --- /dev/null +++ b/deploy/charts/t4-cluster/crds/t4workspaces.cluster.t4.dev.yaml @@ -0,0 +1,142 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: t4workspaces.cluster.t4.dev +spec: + group: cluster.t4.dev + scope: Namespaced + names: + plural: t4workspaces + singular: t4workspace + kind: T4Workspace + listKind: T4WorkspaceList + shortNames: [t4ws] + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - name: Phase + type: string + jsonPath: .status.phase + - name: PVC + type: string + jsonPath: .status.pvcName + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + required: [hostRef, displayName, owner, size, retentionPolicy] + x-kubernetes-validations: + - rule: "!has(self.repository) || !has(self.repository.commit) || (has(self.repository.ref) && self.repository.ref != '')" + message: repository commit requires an explicit ref + properties: + hostRef: + type: string + minLength: 1 + maxLength: 253 + pattern: '^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$' + x-kubernetes-validations: + - rule: self == oldSelf + message: hostRef is immutable + displayName: + type: string + minLength: 1 + maxLength: 128 + owner: + type: string + minLength: 1 + maxLength: 256 + pattern: '^[A-Za-z0-9][A-Za-z0-9._:@/-]*$' + repository: + type: object + required: [repositoryId] + properties: + repositoryId: + type: string + minLength: 1 + maxLength: 128 + pattern: '^[A-Za-z0-9][A-Za-z0-9._:/-]*$' + ref: + type: string + minLength: 1 + maxLength: 256 + commit: + type: string + minLength: 7 + maxLength: 64 + pattern: '^[0-9a-fA-F]+$' + size: + type: string + minLength: 2 + maxLength: 32 + pattern: '^[1-9][0-9]*(Ei|Pi|Ti|Gi|Mi|Ki|E|P|T|G|M|K)$' + retentionPolicy: + type: string + enum: [Retain, Delete] + x-kubernetes-validations: + - rule: self == oldSelf + message: retentionPolicy is immutable + status: + type: object + properties: + observedGeneration: + type: integer + format: int64 + minimum: 0 + pvcName: + type: string + minLength: 1 + maxLength: 63 + pvcPhase: + type: string + enum: [Pending, Bound, Lost] + capacity: + type: string + minLength: 1 + maxLength: 32 + phase: + type: string + enum: [Pending, Ready, Failed, Terminating, Unknown] + conditions: + type: array + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [type] + items: + type: object + required: [type, status, reason, message, lastTransitionTime] + properties: + type: + type: string + enum: [StorageReady, Ready, HostReady] + status: + type: string + enum: ['True', 'False', Unknown] + observedGeneration: + type: integer + format: int64 + minimum: 0 + lastTransitionTime: + type: string + format: date-time + maxLength: 64 + reason: + type: string + minLength: 1 + maxLength: 64 + pattern: '^[A-Za-z][A-Za-z0-9]*$' + message: + type: string + maxLength: 512 diff --git a/deploy/charts/t4-cluster/templates/_helpers.tpl b/deploy/charts/t4-cluster/templates/_helpers.tpl new file mode 100644 index 00000000..60450013 --- /dev/null +++ b/deploy/charts/t4-cluster/templates/_helpers.tpl @@ -0,0 +1,35 @@ +{{- define "t4-cluster.name" -}} +t4-cluster +{{- end -}} + +{{- define "t4-cluster.fullname" -}} +{{- if contains "t4-cluster" .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-t4-cluster" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{- define "t4-cluster.suffixedName" -}} +{{- $suffix := .suffix -}} +{{- $maxBaseLength := sub 62 (len $suffix) | int -}} +{{- $base := include "t4-cluster.fullname" .context | trunc $maxBaseLength | trimSuffix "-" -}} +{{- printf "%s-%s" $base $suffix -}} +{{- end -}} + +{{- define "t4-cluster.labels" -}} +app.kubernetes.io/name: {{ include "t4-cluster.name" . | quote }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +app.kubernetes.io/part-of: "t4-cluster" +app.kubernetes.io/managed-by: {{ .Release.Service | quote }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | quote }} +{{- end -}} + +{{- define "t4-cluster.selectorLabels" -}} +app.kubernetes.io/name: {{ include "t4-cluster.name" . | quote }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +{{- end -}} + +{{- define "t4-cluster.image" -}} +{{- printf "%s@%s" .repository .digest -}} +{{- end -}} diff --git a/deploy/charts/t4-cluster/templates/clusterhost.yaml b/deploy/charts/t4-cluster/templates/clusterhost.yaml new file mode 100644 index 00000000..f03f05d6 --- /dev/null +++ b/deploy/charts/t4-cluster/templates/clusterhost.yaml @@ -0,0 +1,25 @@ +{{- if .Values.enabled }} +apiVersion: cluster.t4.dev/v1alpha1 +kind: T4ClusterHost +metadata: + name: {{ .Values.clusterHost.name | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +spec: + storageClassName: {{ .Values.storage.adminRWXStorageClass | quote }} + runtimeProfiles: + {{- toYaml .Values.clusterHost.runtimeProfiles | nindent 4 }} + allowedOrigins: + {{- toYaml .Values.clusterHost.allowedOrigins | nindent 4 }} + {{- if and .Values.woodpecker.configMap (or .Values.woodpecker.existingSecret .Values.woodpecker.serviceAccountAudience) }} + ciProvider: + {{- if .Values.woodpecker.existingSecret }} + secretRef: + name: {{ .Values.woodpecker.existingSecret | quote }} + {{- else }} + serviceAccountAudience: {{ .Values.woodpecker.serviceAccountAudience | quote }} + {{- end }} + configMapRef: + name: {{ .Values.woodpecker.configMap | quote }} + {{- end }} +{{- end }} diff --git a/deploy/charts/t4-cluster/templates/deployments.yaml b/deploy/charts/t4-cluster/templates/deployments.yaml new file mode 100644 index 00000000..923e0a58 --- /dev/null +++ b/deploy/charts/t4-cluster/templates/deployments.yaml @@ -0,0 +1,366 @@ +{{- if .Values.enabled }} +{{- if eq .Values.session.omp.modelsKey .Values.session.omp.settingsKey }} +{{- fail "session.omp.modelsKey and session.omp.settingsKey must be different" }} +{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "controller") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} + app.kubernetes.io/component: controller +spec: + replicas: {{ .Values.controller.replicas }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: controller + template: + metadata: + labels: + {{- include "t4-cluster.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: controller + app.kubernetes.io/part-of: t4-cluster + spec: + serviceAccountName: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "controller") | quote }} + automountServiceAccountToken: false + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: ["k3s-worker-02"] + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 20 }} + app.kubernetes.io/component: controller + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 14 }} + app.kubernetes.io/component: controller + containers: + - name: controller + image: {{ include "t4-cluster.image" .Values.images.controller | quote }} + imagePullPolicy: {{ .Values.images.controller.pullPolicy | quote }} + args: ["--zap-log-level=info"] + env: + - name: POD_NAMESPACE + valueFrom: {fieldRef: {fieldPath: metadata.namespace}} + - name: T4_SESSION_RUNTIME_IMAGE + value: {{ include "t4-cluster.image" .Values.images.sessionRuntime | quote }} + - name: T4_SESSION_OMP_CONFIG_MAP + value: {{ .Values.session.omp.configMap | quote }} + - name: T4_SESSION_OMP_MODELS_KEY + value: {{ .Values.session.omp.modelsKey | quote }} + - name: T4_SESSION_OMP_SETTINGS_KEY + value: {{ .Values.session.omp.settingsKey | quote }} + - name: T4_SESSION_OMP_CREDENTIAL_SECRET + value: {{ .Values.session.omp.credentialSecret | quote }} + - name: T4_SESSION_OMP_CREDENTIAL_KEY + value: {{ .Values.session.omp.credentialKey | quote }} + - name: T4_SESSION_OMP_ALLOW_UNAUTHENTICATED + value: {{ .Values.session.omp.allowUnauthenticated | quote }} + - name: T4_SESSION_SERVICE_ACCOUNT + value: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "session") | quote }} + - name: T4_CLUSTER_SERVER_SERVICE_ACCOUNT + value: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} + - name: T4_KUBERNETES_API_AUDIENCE + value: {{ .Values.kubernetes.apiAudience | quote }} + - name: T4_SESSION_EXCLUDED_NODES + value: {{ join "," .Values.session.nodeExclude | quote }} + - name: T4_SESSION_REQUEST_CPU + value: {{ .Values.session.resources.requests.cpu | quote }} + - name: T4_SESSION_REQUEST_MEMORY + value: {{ .Values.session.resources.requests.memory | quote }} + - name: T4_SESSION_LIMIT_CPU + value: {{ .Values.session.resources.limits.cpu | quote }} + - name: T4_SESSION_LIMIT_MEMORY + value: {{ .Values.session.resources.limits.memory | quote }} + - name: T4_SESSION_SHM_SIZE + value: {{ .Values.session.sharedMemorySize | quote }} + - name: T4_SESSION_TEMPORARY_SIZE + value: {{ .Values.session.temporarySize | quote }} + ports: + - name: metrics + containerPort: 8080 + - name: health + containerPort: 8081 + startupProbe: + httpGet: {path: /healthz, port: health} + failureThreshold: 30 + periodSeconds: 2 + readinessProbe: + httpGet: {path: /readyz, port: health} + failureThreshold: 3 + periodSeconds: 5 + livenessProbe: + httpGet: {path: /healthz, port: health} + failureThreshold: 3 + periodSeconds: 10 + resources: + {{- toYaml .Values.controller.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + volumeMounts: + - name: temporary + mountPath: /tmp + - name: kubernetes-api-access + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + volumes: + - name: temporary + emptyDir: + sizeLimit: 128Mi + - name: kubernetes-api-access + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: + audience: {{ .Values.kubernetes.apiAudience | quote }} + expirationSeconds: 3600 + path: token + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} + app.kubernetes.io/component: server +spec: + replicas: {{ .Values.server.replicas }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + minReadySeconds: 10 + selector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: server + template: + metadata: + annotations: + cluster.t4.dev/pre-stop-method: POST + cluster.t4.dev/pre-stop-path: /drainz + labels: + {{- include "t4-cluster.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: server + app.kubernetes.io/part-of: t4-cluster + spec: + serviceAccountName: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} + automountServiceAccountToken: false + terminationGracePeriodSeconds: {{ .Values.server.terminationGracePeriodSeconds }} + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: ["k3s-worker-02"] + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 20 }} + app.kubernetes.io/component: server + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 14 }} + app.kubernetes.io/component: server + containers: + - name: server + image: {{ include "t4-cluster.image" .Values.images.server | quote }} + imagePullPolicy: {{ .Values.images.server.pullPolicy | quote }} + env: + - name: T4_CLUSTER_SERVER_PORT + value: "8080" + - name: T4_CLUSTER_ADMIN_PORT + value: "9090" + - name: T4_CLUSTER_TRUSTED_PROXY_ADDRESSES + value: {{ join "," .Values.server.trustedProxyAddresses | quote }} + - name: T4_CLUSTER_TRUSTED_PROXY_CIDRS + value: {{ join "," .Values.server.trustedProxyCIDRs | quote }} + - name: T4_CLUSTER_HOST_NAME + value: {{ .Values.clusterHost.name | quote }} + - name: T4_CLUSTER_IDENTITY_TOKEN_FILE + value: /var/run/secrets/t4-cluster-identity/token + - name: T4_CLUSTER_SERVER_SERVICE_ACCOUNT + value: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} + - name: T4_KUBERNETES_API_AUDIENCE + value: {{ .Values.kubernetes.apiAudience | quote }} + - name: T4_KUBERNETES_TOKEN_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: T4_KUBERNETES_CA_PATH + value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - name: POD_NAMESPACE + valueFrom: {fieldRef: {fieldPath: metadata.namespace}} + - name: POD_NAME + valueFrom: {fieldRef: {fieldPath: metadata.name}} + - name: POD_UID + valueFrom: {fieldRef: {fieldPath: metadata.uid}} + {{- if .Values.woodpecker.existingSecret }} + - name: T4_WOODPECKER_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.woodpecker.existingSecret | quote }} + key: {{ .Values.woodpecker.tokenKey | quote }} + {{- else if .Values.woodpecker.serviceAccountAudience }} + - name: T4_WOODPECKER_TOKEN_FILE + value: /var/run/secrets/t4-ci/token + {{- end }} + {{- if .Values.woodpecker.configMap }} + - name: T4_WOODPECKER_BASE_URL + valueFrom: + configMapKeyRef: + name: {{ .Values.woodpecker.configMap | quote }} + key: {{ .Values.woodpecker.baseURLKey | quote }} + - name: T4_WOODPECKER_WEB_BASE_URL + valueFrom: + configMapKeyRef: + name: {{ .Values.woodpecker.configMap | quote }} + key: {{ .Values.woodpecker.webBaseURLKey | quote }} + - name: T4_WOODPECKER_REPOSITORIES + valueFrom: + configMapKeyRef: + name: {{ .Values.woodpecker.configMap | quote }} + key: {{ .Values.woodpecker.repositoriesKey | quote }} + {{- end }} + ports: + - name: websocket + containerPort: 8080 + - name: admin + containerPort: 9090 + lifecycle: + preStop: + exec: + command: [/usr/bin/curl, -fsS, -X, POST, http://127.0.0.1:9090/drainz] + startupProbe: + httpGet: {path: /healthz, port: admin} + failureThreshold: 30 + periodSeconds: 2 + readinessProbe: + httpGet: {path: /readyz, port: admin} + failureThreshold: 3 + periodSeconds: 5 + livenessProbe: + httpGet: {path: /healthz, port: admin} + failureThreshold: 3 + periodSeconds: 10 + resources: + {{- toYaml .Values.server.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + volumeMounts: + - name: temporary + mountPath: /tmp + - name: cluster-identity + mountPath: /var/run/secrets/t4-cluster-identity + readOnly: true + - name: kubernetes-api-access + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + {{- if .Values.woodpecker.serviceAccountAudience }} + - name: ci-identity + mountPath: /var/run/secrets/t4-ci + readOnly: true + {{- end }} + volumes: + - name: cluster-identity + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: + audience: "t4-cluster-internal" + expirationSeconds: 600 + path: token + - name: kubernetes-api-access + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: + audience: {{ .Values.kubernetes.apiAudience | quote }} + expirationSeconds: 3600 + path: token + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt + {{- if .Values.woodpecker.serviceAccountAudience }} + - name: ci-identity + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: + audience: {{ .Values.woodpecker.serviceAccountAudience | quote }} + expirationSeconds: {{ .Values.woodpecker.tokenExpirationSeconds }} + path: token + {{- end }} + - name: temporary + emptyDir: + sizeLimit: 256Mi +{{- end }} diff --git a/deploy/charts/t4-cluster/templates/ingress.yaml b/deploy/charts/t4-cluster/templates/ingress.yaml new file mode 100644 index 00000000..13945138 --- /dev/null +++ b/deploy/charts/t4-cluster/templates/ingress.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.enabled .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "t4-cluster.fullname" . | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.className | quote }} + {{- if .Values.ingress.tls.enabled }} + tls: + - hosts: [{{ .Values.ingress.host | quote }}] + {{- with .Values.ingress.tls.secretName }} + secretName: {{ . | quote }} + {{- end }} + {{- end }} + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} + port: + name: websocket +{{- end }} diff --git a/deploy/charts/t4-cluster/templates/networkpolicies.yaml b/deploy/charts/t4-cluster/templates/networkpolicies.yaml new file mode 100644 index 00000000..99837f80 --- /dev/null +++ b/deploy/charts/t4-cluster/templates/networkpolicies.yaml @@ -0,0 +1,207 @@ +{{- if and .Values.enabled .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "default-deny") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/part-of: t4-cluster + policyTypes: [Ingress, Egress] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "dns") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/part-of: t4-cluster + policyTypes: [Egress] + egress: + - to: + - namespaceSelector: + {{- toYaml .Values.networkPolicy.dns.namespaceSelector | nindent 12 }} + podSelector: + {{- toYaml .Values.networkPolicy.dns.podSelector | nindent 12 }} + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "controller-api") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: controller + policyTypes: [Egress] + egress: + {{- range .Values.networkPolicy.kubernetesApiCIDRs }} + - to: + - ipBlock: {cidr: {{ . | quote }}} + ports: + - {protocol: TCP, port: 443} + - {protocol: TCP, port: 6443} + {{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server-egress") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: server + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: t4-session-runtime + app.kubernetes.io/part-of: t4-cluster + ports: + - {protocol: TCP, port: 8787} + {{- range .Values.networkPolicy.kubernetesApiCIDRs }} + - to: + - ipBlock: {cidr: {{ . | quote }}} + ports: + - {protocol: TCP, port: 443} + - {protocol: TCP, port: 6443} + {{- end }} + {{- if .Values.networkPolicy.ciProviderPorts }} + {{- range .Values.networkPolicy.ciProviderCIDRs }} + - to: + - ipBlock: {cidr: {{ . | quote }}} + ports: + {{- range $.Values.networkPolicy.ciProviderPorts }} + - {protocol: TCP, port: {{ . }}} + {{- end }} + {{- end }} + {{- if and (not (empty .Values.networkPolicy.ciProvider.namespaceSelector)) (not (empty .Values.networkPolicy.ciProvider.podSelector)) }} + - to: + - namespaceSelector: + {{- toYaml .Values.networkPolicy.ciProvider.namespaceSelector | nindent 12 }} + podSelector: + {{- toYaml .Values.networkPolicy.ciProvider.podSelector | nindent 12 }} + ports: + {{- range .Values.networkPolicy.ciProviderPorts }} + - {protocol: TCP, port: {{ . }}} + {{- end }} + {{- end }} + {{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "session-host") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: t4-session-runtime + app.kubernetes.io/part-of: t4-cluster + policyTypes: [Ingress, Egress] + ingress: + - from: + - podSelector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 14 }} + app.kubernetes.io/component: server + ports: + - {protocol: TCP, port: 8787} + egress: + {{- range .Values.networkPolicy.kubernetesApiCIDRs }} + - to: + - ipBlock: {cidr: {{ . | quote }}} + ports: + - {protocol: TCP, port: 443} + - {protocol: TCP, port: 6443} + {{- end }} + {{- if .Values.networkPolicy.modelRoutePorts }} + {{- range .Values.networkPolicy.modelRouteCIDRs }} + - to: + - ipBlock: {cidr: {{ . | quote }}} + ports: + {{- range $.Values.networkPolicy.modelRoutePorts }} + - {protocol: TCP, port: {{ . }}} + {{- end }} + {{- end }} + {{- if and (not (empty .Values.networkPolicy.modelRoute.namespaceSelector)) (not (empty .Values.networkPolicy.modelRoute.podSelector)) }} + - to: + - namespaceSelector: + {{- toYaml .Values.networkPolicy.modelRoute.namespaceSelector | nindent 12 }} + podSelector: + {{- toYaml .Values.networkPolicy.modelRoute.podSelector | nindent 12 }} + ports: + {{- range .Values.networkPolicy.modelRoutePorts }} + - {protocol: TCP, port: {{ . }}} + {{- end }} + {{- end }} + {{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "gateway-ingress") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: server + policyTypes: [Ingress] + {{- if and (not (empty .Values.networkPolicy.gatewayIngress.namespaceSelector)) (not (empty .Values.networkPolicy.gatewayIngress.podSelector)) }} + ingress: + - from: + - namespaceSelector: + {{- toYaml .Values.networkPolicy.gatewayIngress.namespaceSelector | nindent 12 }} + podSelector: + {{- toYaml .Values.networkPolicy.gatewayIngress.podSelector | nindent 12 }} + ports: + - {protocol: TCP, port: 8080} + {{- else }} + ingress: [] + {{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "observability") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 6 }} + app.kubernetes.io/part-of: "t4-cluster" + matchExpressions: + - {key: app.kubernetes.io/component, operator: In, values: [controller, server]} + policyTypes: [Ingress] + {{- if and (not (empty .Values.networkPolicy.observability.namespaceSelector)) (not (empty .Values.networkPolicy.observability.podSelector)) }} + ingress: + - from: + - namespaceSelector: + {{- toYaml .Values.networkPolicy.observability.namespaceSelector | nindent 12 }} + podSelector: + {{- toYaml .Values.networkPolicy.observability.podSelector | nindent 12 }} + ports: + - {protocol: TCP, port: 8080} + - {protocol: TCP, port: 9090} + {{- else }} + ingress: [] + {{- end }} +{{- end }} diff --git a/deploy/charts/t4-cluster/templates/observability.yaml b/deploy/charts/t4-cluster/templates/observability.yaml new file mode 100644 index 00000000..21814c33 --- /dev/null +++ b/deploy/charts/t4-cluster/templates/observability.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.enabled .Values.observability.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} + {{- with .Values.observability.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + matchNames: [{{ .Release.Namespace | quote }}] + selector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: server + app.kubernetes.io/part-of: "t4-cluster" + endpoints: + - port: metrics + path: /metrics + interval: {{ .Values.observability.serviceMonitor.interval | quote }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "controller") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} + {{- with .Values.observability.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + matchNames: [{{ .Release.Namespace | quote }}] + selector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: controller + app.kubernetes.io/part-of: "t4-cluster" + endpoints: + - port: metrics + path: /metrics + interval: {{ .Values.observability.serviceMonitor.interval | quote }} +{{- end }} +{{- if and .Values.enabled .Values.observability.prometheusRule.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "t4-cluster.fullname" . | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} + {{- with .Values.observability.prometheusRule.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + groups: + - name: t4-cluster-ha + rules: + - alert: T4ClusterServerBelowQuorum + expr: kube_deployment_status_replicas_available{namespace={{ .Release.Namespace | quote }},deployment={{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }}} < 2 + for: 5m + labels: {severity: critical} + annotations: + summary: T4 cluster server has fewer than two available replicas + - alert: T4ClusterReconcileErrors + expr: increase(controller_runtime_reconcile_errors_total{namespace={{ .Release.Namespace | quote }},controller=~"t4clusterhost|t4workspace|t4session"}[10m]) > 0 + for: 10m + labels: {severity: warning} + annotations: + summary: T4 cluster operator reconciliation is failing + - alert: T4WorkspaceStoragePending + expr: max by (namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_status_phase{namespace={{ .Release.Namespace | quote }},phase=~"Pending|Lost"}) == 1 + for: 15m + labels: {severity: warning} + annotations: + summary: A PVC in the T4 namespace is pending or lost +{{- end }} diff --git a/deploy/charts/t4-cluster/templates/rbac.yaml b/deploy/charts/t4-cluster/templates/rbac.yaml new file mode 100644 index 00000000..dbaedab3 --- /dev/null +++ b/deploy/charts/t4-cluster/templates/rbac.yaml @@ -0,0 +1,163 @@ +{{- if .Values.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "controller") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: [cluster.t4.dev] + resources: [t4clusterhosts, t4workspaces, t4sessions] + verbs: [get, list, watch, update, patch] + - apiGroups: [cluster.t4.dev] + resources: [t4clusterhosts/status, t4workspaces/status, t4sessions/status] + verbs: [get, update, patch] + - apiGroups: [cluster.t4.dev] + resources: [t4workspaces/finalizers, t4sessions/finalizers] + verbs: [update] + - apiGroups: [''] + resources: [pods, services, persistentvolumeclaims] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: [''] + resources: [configmaps] + resourceNames: [{{ .Values.session.omp.configMap | quote }}] + verbs: [get] + {{- if not .Values.session.omp.allowUnauthenticated }} + - apiGroups: [''] + resources: [secrets] + resourceNames: [{{ .Values.session.omp.credentialSecret | quote }}] + verbs: [get] + {{- end }} + - apiGroups: [''] + resources: [events] + verbs: [create, patch] + - apiGroups: [coordination.k8s.io] + resources: [leases] + verbs: [get, list, watch, create, update, patch, delete] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: [cluster.t4.dev] + resources: [t4clusterhosts] + verbs: [get, list, watch] + - apiGroups: [cluster.t4.dev] + resources: [t4workspaces, t4sessions] + verbs: [get, list, watch, create, delete] + - apiGroups: [''] + resources: [pods, services, endpoints] + verbs: [get, list, watch] + - apiGroups: [discovery.k8s.io] + resources: [endpointslices] + verbs: [get, list, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "session-token-reviewer") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: [authentication.k8s.io] + resources: [tokenreviews] + verbs: [create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "storage-reader") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: [storage.k8s.io] + resources: [storageclasses] + verbs: [get, list, watch] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "controller") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "session") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "controller") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "controller") | quote }} + namespace: {{ .Release.Namespace | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "controller") | quote }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "storage-reader") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "controller") | quote }} + namespace: {{ .Release.Namespace | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "storage-reader") | quote }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} + namespace: {{ .Release.Namespace | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "session-token-reviewer") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "session") | quote }} + namespace: {{ .Release.Namespace | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "session-token-reviewer") | quote }} +{{- end }} diff --git a/deploy/charts/t4-cluster/templates/services.yaml b/deploy/charts/t4-cluster/templates/services.yaml new file mode 100644 index 00000000..de5a8bba --- /dev/null +++ b/deploy/charts/t4-cluster/templates/services.yaml @@ -0,0 +1,69 @@ +{{- if .Values.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} + app.kubernetes.io/component: server +spec: + type: ClusterIP + selector: + {{- include "t4-cluster.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: server + ports: + - name: websocket + port: 8080 + targetPort: websocket + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "metrics") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} + app.kubernetes.io/component: server +spec: + type: ClusterIP + selector: + {{- include "t4-cluster.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: server + ports: + - name: metrics + port: 9090 + targetPort: admin + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "controller-metrics") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} + app.kubernetes.io/component: controller +spec: + type: ClusterIP + selector: + {{- include "t4-cluster.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: controller + ports: + - name: metrics + port: 8080 + targetPort: metrics + protocol: TCP +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "t4-cluster.suffixedName" (dict "context" . "suffix" "server") | quote }} + labels: + {{- include "t4-cluster.labels" . | nindent 4 }} + app.kubernetes.io/component: server +spec: + minAvailable: 2 + selector: + matchLabels: + {{- include "t4-cluster.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: server +{{- end }} diff --git a/deploy/charts/t4-cluster/values.schema.json b/deploy/charts/t4-cluster/values.schema.json new file mode 100644 index 00000000..9869ccb5 --- /dev/null +++ b/deploy/charts/t4-cluster/values.schema.json @@ -0,0 +1,441 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "required": [ + "enabled", + "clusterHost", + "kubernetes", + "images", + "controller", + "server", + "session", + "storage", + "woodpecker", + "ingress", + "networkPolicy", + "observability" + ], + "properties": { + "enabled": { "type": "boolean", "default": false }, + "clusterHost": { + "type": "object", + "required": ["name", "runtimeProfiles", "allowedOrigins"], + "properties": { + "name": { "$ref": "#/definitions/dnsSubdomain" }, + "runtimeProfiles": { + "type": "array", + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "items": { "$ref": "#/definitions/runtimeProfile" } + }, + "allowedOrigins": { + "type": "array", + "maxItems": 32, + "uniqueItems": true, + "items": { "type": "string", "pattern": "^https://[^/?#]+$", "maxLength": 256 } + } + } + }, + "kubernetes": { + "type": "object", + "required": ["apiAudience"], + "properties": { + "apiAudience": { "$ref": "#/definitions/audience" } + } + }, + "images": { + "type": "object", + "required": ["controller", "server", "sessionRuntime"], + "properties": { + "controller": { "$ref": "#/definitions/image" }, + "server": { "$ref": "#/definitions/image" }, + "sessionRuntime": { "$ref": "#/definitions/image" } + } + }, + "imagePullSecrets": { + "type": "array", + "items": { + "type": "object", + "required": ["name"], + "properties": { "name": { "$ref": "#/definitions/dnsSubdomain" } } + } + }, + "controller": { + "type": "object", + "required": ["replicas", "resources"], + "properties": { + "replicas": { "type": "integer", "const": 2 }, + "resources": { "type": "object" } + } + }, + "server": { + "type": "object", + "required": ["replicas", "terminationGracePeriodSeconds", "trustedProxyAddresses", "trustedProxyCIDRs", "resources"], + "properties": { + "replicas": { "type": "integer", "minimum": 2, "maximum": 9 }, + "terminationGracePeriodSeconds": { "type": "integer", "minimum": 30, "maximum": 300 }, + "trustedProxyAddresses": { + "type": "array", + "maxItems": 64, + "uniqueItems": true, + "items": { "type": "string", "anyOf": [{ "format": "ipv4" }, { "format": "ipv6" }] } + }, + "trustedProxyCIDRs": { + "type": "array", + "maxItems": 64, + "uniqueItems": true, + "items": { "$ref": "#/definitions/nonDefaultRouteCIDR" } + }, + "resources": { "type": "object" } + } + }, + "session": { + "type": "object", + "required": ["nodeExclude", "resources", "sharedMemorySize", "temporarySize", "omp"], + "properties": { + "nodeExclude": { + "type": "array", + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "contains": { "const": "k3s-worker-02" }, + "items": { "$ref": "#/definitions/dnsLabel" } + }, + "resources": { "type": "object" }, + "sharedMemorySize": { "type": "string", "pattern": "^[1-9][0-9]*(Mi|Gi)$" }, + "temporarySize": { "type": "string", "pattern": "^[1-9][0-9]*(Mi|Gi)$" }, + "omp": { + "type": "object", + "required": ["configMap", "modelsKey", "settingsKey", "credentialSecret", "credentialKey", "allowUnauthenticated"], + "properties": { + "configMap": { "$ref": "#/definitions/optionalDNSSubdomain" }, + "modelsKey": { "$ref": "#/definitions/optionalConfigKey" }, + "settingsKey": { "$ref": "#/definitions/optionalConfigKey" }, + "credentialSecret": { "$ref": "#/definitions/optionalDNSSubdomain" }, + "credentialKey": { "$ref": "#/definitions/optionalCredentialEnvVarName" }, + "allowUnauthenticated": { "type": "boolean" } + } + } + } + }, + "storage": { + "type": "object", + "required": ["adminRWXStorageClass"], + "properties": { + "adminRWXStorageClass": { + "allOf": [{ "$ref": "#/definitions/optionalDNSSubdomain" }] + } + } + }, + "woodpecker": { + "type": "object", + "required": ["existingSecret", "tokenKey", "configMap", "baseURLKey", "webBaseURLKey", "repositoriesKey", "serviceAccountAudience", "tokenExpirationSeconds"], + "properties": { + "existingSecret": { "$ref": "#/definitions/optionalDNSSubdomain" }, + "tokenKey": { "$ref": "#/definitions/configKey" }, + "configMap": { "$ref": "#/definitions/optionalDNSSubdomain" }, + "baseURLKey": { "$ref": "#/definitions/configKey" }, + "repositoriesKey": { "$ref": "#/definitions/configKey" }, + "webBaseURLKey": { "$ref": "#/definitions/configKey" }, + "serviceAccountAudience": { + "type": "string", + "maxLength": 253, + "pattern": "^$|^[A-Za-z0-9][A-Za-z0-9._:/-]*$" + }, + "tokenExpirationSeconds": { "type": "integer", "minimum": 600, "maximum": 3600 } + }, + "oneOf": [ + { "properties": { "existingSecret": { "maxLength": 0 }, "configMap": { "maxLength": 0 }, "serviceAccountAudience": { "maxLength": 0 } } }, + { "properties": { "existingSecret": { "minLength": 1 }, "configMap": { "minLength": 1 }, "serviceAccountAudience": { "maxLength": 0 } } }, + { "properties": { "existingSecret": { "maxLength": 0 }, "configMap": { "minLength": 1 }, "serviceAccountAudience": { "minLength": 1 } } } + ] + }, + "ingress": { + "type": "object", + "required": ["enabled", "className", "host", "annotations", "tls"], + "properties": { + "enabled": { "type": "boolean" }, + "className": { "$ref": "#/definitions/optionalDNSLabel" }, + "host": { "$ref": "#/definitions/optionalDNSSubdomain" }, + "annotations": { "type": "object" }, + "tls": { + "type": "object", + "required": ["enabled", "secretName"], + "properties": { + "enabled": { "type": "boolean" }, + "secretName": { "$ref": "#/definitions/optionalDNSSubdomain" } + } + } + } + }, + "networkPolicy": { + "type": "object", + "required": ["enabled", "kubernetesApiCIDRs", "modelRouteCIDRs", "modelRoutePorts", "modelRoute", "ciProviderCIDRs", "ciProviderPorts", "ciProvider", "dns", "gatewayIngress", "observability"], + "properties": { + "enabled": { "type": "boolean" }, + "kubernetesApiCIDRs": { "$ref": "#/definitions/cidrList" }, + "modelRouteCIDRs": { "$ref": "#/definitions/cidrList" }, + "modelRoutePorts": { "$ref": "#/definitions/tcpPortList" }, + "modelRoute": { "$ref": "#/definitions/pairedSelectors" }, + "ciProviderCIDRs": { "$ref": "#/definitions/cidrList" }, + "ciProviderPorts": { "$ref": "#/definitions/tcpPortList" }, + "ciProvider": { "$ref": "#/definitions/pairedSelectors" }, + "dns": { + "type": "object", + "required": ["namespaceSelector", "podSelector"], + "properties": { + "namespaceSelector": { "allOf": [{ "$ref": "#/definitions/labelSelector" }], "minProperties": 1 }, + "podSelector": { "allOf": [{ "$ref": "#/definitions/labelSelector" }], "minProperties": 1 } + } + }, + "gatewayIngress": { "$ref": "#/definitions/pairedSelectors" }, + "observability": { "$ref": "#/definitions/pairedSelectors" } + } + }, + "observability": { + "type": "object", + "required": ["serviceMonitor", "prometheusRule"], + "properties": { + "serviceMonitor": { + "type": "object", + "required": ["enabled", "interval", "labels"], + "properties": { + "enabled": { "type": "boolean" }, + "interval": { "type": "string", "pattern": "^[1-9][0-9]*(ms|s|m|h)$" }, + "labels": { "type": "object" } + } + }, + "prometheusRule": { + "type": "object", + "required": ["enabled", "labels"], + "properties": { + "enabled": { "type": "boolean" }, + "labels": { "type": "object" } + } + } + } + } + }, + "definitions": { + "dnsLabel": { + "type": "string", + "minLength": 1, + "maxLength": 63, + "pattern": "^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$" + }, + "optionalDNSLabel": { + "type": "string", + "maxLength": 63, + "pattern": "^$|^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$" + }, + "dnsSubdomain": { + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?)*$" + }, + "optionalDNSSubdomain": { + "type": "string", + "maxLength": 253, + "pattern": "^$|^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?)*$" + }, + "runtimeProfile": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9]([-._a-z0-9]{0,62}[a-z0-9])?$" + }, + "audience": { + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:/-]*$" + }, + "configKey": { + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^[-._A-Za-z0-9]+$" + }, + "optionalConfigKey": { + "type": "string", + "maxLength": 253, + "pattern": "^$|^[-._A-Za-z0-9]+$" + }, + "optionalEnvVarName": { + "type": "string", + "maxLength": 253, + "pattern": "^$|^[A-Za-z_][A-Za-z0-9_]*$" + }, + "optionalCredentialEnvVarName": { + "allOf": [ + { "$ref": "#/definitions/optionalEnvVarName" }, + { + "not": { + "anyOf": [ + { "pattern": "^T4_" }, + { "pattern": "^OMP_" }, + { "pattern": "^PI_" }, + { "pattern": "^XDG_" }, + { "pattern": "^LD_" }, + { "enum": ["HOME", "DISPLAY", "PATH", "BASH_ENV", "ENV", "SHELLOPTS", "NODE_OPTIONS", "BUN_OPTIONS", "PI_CODING_AGENT_DIR", "PI_CONFIG_DIR"] } + ] + } + } + ] + }, + "nonDefaultRouteCIDR": { + "type": "string", + "maxLength": 64, + "pattern": "^[0-9A-Fa-f:.]+/([1-9]|[1-9][0-9]|1[01][0-9]|12[0-8])$" + }, + "cidrList": { + "type": "array", + "maxItems": 64, + "uniqueItems": true, + "items": { "$ref": "#/definitions/nonDefaultRouteCIDR" } + }, + "tcpPortList": { + "type": "array", + "maxItems": 16, + "uniqueItems": true, + "items": { "type": "integer", "minimum": 1, "maximum": 65535 } + }, + "labelSelector": { + "type": "object", + "maxProperties": 2, + "properties": { + "matchLabels": { + "type": "object", + "minProperties": 1, + "maxProperties": 32, + "additionalProperties": { "type": "string", "maxLength": 63 } + }, + "matchExpressions": { + "type": "array", + "minItems": 1, + "maxItems": 32, + "items": { + "type": "object", + "required": ["key", "operator"], + "properties": { + "key": { "type": "string", "minLength": 1, "maxLength": 253 }, + "operator": { "type": "string", "enum": ["In", "NotIn", "Exists", "DoesNotExist"] }, + "values": { "type": "array", "maxItems": 32, "items": { "type": "string", "maxLength": 63 } } + } + } + } + } + }, + "pairedSelectors": { + "type": "object", + "required": ["namespaceSelector", "podSelector"], + "properties": { + "namespaceSelector": { "$ref": "#/definitions/labelSelector" }, + "podSelector": { "$ref": "#/definitions/labelSelector" } + }, + "oneOf": [ + { + "properties": { + "namespaceSelector": { "maxProperties": 0 }, + "podSelector": { "maxProperties": 0 } + } + }, + { + "properties": { + "namespaceSelector": { "minProperties": 1 }, + "podSelector": { "minProperties": 1 } + } + } + ] + }, + "image": { + "type": "object", + "required": ["repository", "digest", "pullPolicy"], + "properties": { + "repository": { "type": "string", "minLength": 1, "maxLength": 256 }, + "digest": { "type": "string" }, + "pullPolicy": { "type": "string", "enum": ["Always", "IfNotPresent", "Never"] } + } + } + }, + "allOf": [ + { + "if": { "properties": { "enabled": { "const": true } } }, + "then": { + "properties": { + "server": { + "anyOf": [ + { "properties": { "trustedProxyAddresses": { "minItems": 1 } } }, + { "properties": { "trustedProxyCIDRs": { "minItems": 1 } } } + ] + }, + "storage": { "properties": { "adminRWXStorageClass": { "minLength": 1 } } }, + "session": { + "properties": { + "omp": { + "properties": { + "configMap": { "minLength": 1 }, + "modelsKey": { "minLength": 1 }, + "settingsKey": { "minLength": 1 } + }, + "oneOf": [ + { + "properties": { + "allowUnauthenticated": { "const": false }, + "credentialSecret": { "minLength": 1 }, + "credentialKey": { "minLength": 1 } + } + }, + { + "properties": { + "allowUnauthenticated": { "const": true }, + "credentialSecret": { "maxLength": 0 }, + "credentialKey": { "maxLength": 0 } + } + } + ] + } + } + }, + "images": { + "properties": { + "controller": { "properties": { "digest": { "pattern": "^sha256:[0-9a-f]{64}$" } } }, + "server": { "properties": { "digest": { "pattern": "^sha256:[0-9a-f]{64}$" } } }, + "sessionRuntime": { "properties": { "digest": { "pattern": "^sha256:[0-9a-f]{64}$" } } } + } + } + } + } + }, + { + "if": { "properties": { "ingress": { "properties": { "enabled": { "const": true } } } } }, + "then": { + "properties": { + "ingress": { + "properties": { + "className": { "minLength": 1 }, + "host": { "minLength": 1 }, + "tls": { "properties": { "enabled": { "const": true } } } + }, + "oneOf": [ + { + "properties": { + "className": { "const": "tailscale" }, + "tls": { "properties": { "secretName": { "maxLength": 0 } } } + } + }, + { + "properties": { + "className": { "not": { "const": "tailscale" } }, + "tls": { "properties": { "secretName": { "minLength": 1 } } } + } + } + ] + } + } + } + } + ] +} diff --git a/deploy/charts/t4-cluster/values.yaml b/deploy/charts/t4-cluster/values.yaml new file mode 100644 index 00000000..5224ced7 --- /dev/null +++ b/deploy/charts/t4-cluster/values.yaml @@ -0,0 +1,141 @@ +enabled: false + +clusterHost: + name: t4-cluster + runtimeProfiles: + - default + allowedOrigins: [] + +kubernetes: + # Audience accepted by this cluster's API server for projected API credentials. + apiAudience: https://kubernetes.default.svc + +images: + controller: + repository: ghcr.io/lycaonllc/t4-cluster-operator + digest: "" + pullPolicy: IfNotPresent + server: + repository: ghcr.io/lycaonllc/t4-cluster-server + digest: "" + pullPolicy: IfNotPresent + sessionRuntime: + repository: ghcr.io/lycaonllc/t4-session-runtime + digest: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] + +controller: + replicas: 2 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +server: + replicas: 3 + terminationGracePeriodSeconds: 60 + # At least one trusted Tailscale proxy address or canonical CIDR is required for client access. + trustedProxyAddresses: [] + trustedProxyCIDRs: [] + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "2" + memory: 2Gi + +session: + nodeExclude: + - k3s-worker-02 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "4" + memory: 8Gi + sharedMemorySize: 1Gi + temporarySize: 2Gi + + omp: + # Administrator-owned runtime configuration references. The chart creates neither object. + configMap: "" + modelsKey: "" + settingsKey: "" + credentialSecret: "" + credentialKey: "" + # Explicit opt-in for a private, policy-isolated models.yml provider using auth: none. + allowUnauthenticated: false + +storage: + # Required when enabled. The administrator creates this backend-neutral RWX StorageClass. + adminRWXStorageClass: "" + +woodpecker: + existingSecret: "" + tokenKey: token + # Alternative to existingSecret: a short-lived token accepted by an administrator-owned CI adapter. + serviceAccountAudience: "" + tokenExpirationSeconds: 600 + configMap: "" + baseURLKey: baseURL + webBaseURLKey: webBaseURL + repositoriesKey: repositories + +ingress: + enabled: false + className: "" + host: "" + annotations: {} + tls: + enabled: true + # Leave empty for the Tailscale ingress class; its operator provisions TLS. + secretName: "" + +networkPolicy: + enabled: true + # Empty destination lists fail closed. Supply narrow destinations for enabled components. + kubernetesApiCIDRs: [] + modelRouteCIDRs: [] + modelRoutePorts: [] + modelRoute: + # Optional in-cluster model endpoint identity. Set both selectors or neither. + namespaceSelector: {} + podSelector: {} + ciProviderCIDRs: [] + # This port is inert until a CI destination CIDR or complete selector pair is supplied. + ciProviderPorts: + - 443 + ciProvider: + # Optional in-cluster CI endpoint identity. Set both selectors or neither. + namespaceSelector: {} + podSelector: {} + dns: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + gatewayIngress: + # Empty selectors fail closed. Set both to the ingress controller identity. + namespaceSelector: {} + podSelector: {} + observability: + namespaceSelector: {} + podSelector: {} + +observability: + serviceMonitor: + enabled: false + interval: 30s + labels: {} + prometheusRule: + enabled: false + labels: {} diff --git a/docs/CLUSTER_OPERATOR.md b/docs/CLUSTER_OPERATOR.md new file mode 100644 index 00000000..c942e997 --- /dev/null +++ b/docs/CLUSTER_OPERATOR.md @@ -0,0 +1,183 @@ +# T4 Kubernetes cluster operator + +The portable `t4-cluster` chart runs the infrastructure control plane for T4 workspaces and one-session runtime pods. It is disabled by default. Kubernetes owns only infrastructure desired state, placement, PVCs, pods, Services, retention, and infrastructure conditions. OMP remains the sole authority for sessions, agent ids and parentage, lifecycle, turns, prompts, approvals, jobs, IRC, artifacts, terminals, browser commands, cancellation, and takeover through `t4-omp-authority/1`. + +## Prerequisites + +- Kubernetes 1.30 or newer. +- An administrator-managed StorageClass that actually provisions `ReadWriteMany` volumes. +- Three immutable image digests: `t4-cluster-operator`, `t4-cluster-server`, and `t4-session-runtime`. +- An administrator-owned same-namespace ConfigMap containing the OMP `models.yml` and `config.yml` inputs, plus either a same-namespace credential Secret or an explicit private unauthenticated-provider opt-in. +- Narrow Kubernetes API, model-route, CI-provider, ingress-controller, and metrics-scraper network identities and ports for the enabled integrations. +- A local T4 client explicitly configured to request the default-false `cluster.operator` feature. Installing this chart does not enable the client feature. + +The chart does not install NFS, CSI drivers, a StorageClass, host paths, or backend-specific storage configuration. The cluster administrator must create and validate the RWX StorageClass. Mark a class as reviewed for this controller: + +```yaml +metadata: + annotations: + cluster.t4.dev/access-modes: ReadWriteMany +``` + +That declaration does not make a non-RWX backend safe. Missing classes, classes without this exact declaration, unbound claims, and claims without `ReadWriteMany` fail closed. A `T4Session` pod is never created before its workspace claim is both Bound and RWX. + +## Install + +Installing with defaults creates no controller, gateway, session workload, RBAC, Secret, or network policy: + +```sh +helm install t4-cluster deploy/charts/t4-cluster --namespace t4-system --create-namespace +``` + +Helm processes files in `crds/` separately. Use `--skip-crds` if CRD lifecycle is administered independently. To enable the control plane, provide a private values file or a deployment controller values source: + +```yaml +enabled: true +storage: + adminRWXStorageClass: portable-rwx +images: + controller: + repository: registry.example/t4-cluster-operator + digest: sha256:0000000000000000000000000000000000000000000000000000000000000000 + pullPolicy: IfNotPresent + server: + repository: registry.example/t4-cluster-server + digest: sha256:0000000000000000000000000000000000000000000000000000000000000000 + pullPolicy: IfNotPresent + sessionRuntime: + repository: registry.example/t4-session-runtime + digest: sha256:0000000000000000000000000000000000000000000000000000000000000000 + pullPolicy: IfNotPresent +kubernetes: + # Set this to an audience accepted by this cluster's API server. + apiAudience: https://kubernetes.default.svc +session: + nodeExclude: [k3s-worker-02] + omp: + # Existing same-namespace objects; the chart creates neither one. + configMap: omp-runtime-config + modelsKey: models.yml + settingsKey: config.yml + credentialSecret: omp-runtime-credential + # This existing Secret key is also the environment variable name seen by OMP. + credentialKey: PI_MODEL_API_KEY + allowUnauthenticated: false +networkPolicy: + kubernetesApiCIDRs: [192.0.2.10/32] + modelRouteCIDRs: [198.51.100.20/32] + modelRoutePorts: [19481] + modelRoute: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: private-model-service + podSelector: + matchLabels: + app.kubernetes.io/name: private-model-endpoint + ciProviderCIDRs: [203.0.113.30/32] + ciProviderPorts: [8080] + ciProvider: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: private-ci-service + podSelector: + matchLabels: + app.kubernetes.io/name: ci-trigger + dns: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + gatewayIngress: + namespaceSelector: + matchLabels: + ingress.example/namespace: gateway + podSelector: + matchLabels: + ingress.example/component: proxy + observability: + namespaceSelector: + matchLabels: + monitoring.example/namespace: metrics + podSelector: + matchLabels: + monitoring.example/component: prometheus +``` + +The sample destination CIDRs, TCP ports, model/CI endpoint selectors, and gateway/observability labels are documentation values, not usable defaults. Keep the chart backend-neutral and set destinations and integration sources to the actual cluster endpoints. Model and CI route CIDRs and their optional paired namespace/pod selectors are allowed only on the corresponding exact port lists. An empty model port list renders no model route; the default CI port remains inert until a CI CIDR or complete selector pair is supplied. Empty paired selectors render no selector rule, and other empty destination lists likewise deny those flows. + +The chart never creates the referenced OMP ConfigMap or Secret and never accepts their contents. The administrator must create them in the chart namespace. `modelsKey` is projected as read-only `models.yml`, `settingsKey` as read-only `config.yml`, and the configured `credentialKey` selects the same Secret key and OMP environment-variable name. The entrypoint validates both projected files and the nonempty credential without logging values, then atomically installs private copies under the OMP authority child's actual named-profile directory `${T4_SESSION_STATE_ROOT}/home/.omp/profiles/${T4_SESSION_NAME}/agent` before starting Xvfb or OMP. This must match the `OMP_PROFILE=${T4_SESSION_NAME}` passed by the session host to the pinned authority bridge child. + +The controller uses uncached, exact-name `get` permissions for only those two administrator-owned objects; it cannot list or watch namespace configuration. It validates the selected keys without logging or hashing their contents, incorporates each object's Kubernetes `resourceVersion` into the session Pod hash, and rechecks ready sessions every 30 seconds. Updating either object therefore recreates the session Pod and reloads OMP configuration from the durable workspace state. Deleting a required object or emptying a selected key deletes the owned authority Pod and clears the advertised Pod and Service route before reporting the exact fail-closed condition. In unauthenticated mode the controller has no Secret permission. + +Credential mode is the default and requires both `credentialSecret` and `credentialKey`. The projected models and settings keys must be distinct. Credential keys cannot overlap the session runtime's `T4_*`, `OMP_*`, `PI_*`, `XDG_*`, loader, shell, display, or path environment; collisions fail chart validation, controller reconciliation, and entrypoint preflight instead of silently replacing runtime authority. An administrator may instead set `allowUnauthenticated: true` only when both credential fields are empty and the private, identity- and NetworkPolicy-isolated model endpoint is intentionally unauthenticated. In that mode the referenced `models.yml` must declare `auth: none` and must not declare `apiKey` or `authHeader`; never use this opt-in for an Internet-reachable or shared endpoint. The chart does not select or hardcode a provider, model, or prompt policy in either mode. + +Install or enable with immutable digests: + +```sh +helm upgrade --install t4-cluster deploy/charts/t4-cluster --namespace t4-system --values operator-values.yaml +``` + +The controller always has two replicas and uses a Kubernetes Lease named from `t4-cluster-operator.cluster.t4.dev`; one replica reconciles at a time. The server defaults to three stateless replicas and supports a minimum of two. Its Deployment uses `maxUnavailable: 0`, a `minAvailable: 2` PDB, topology spread, anti-affinity, readiness draining, and an explicit `k3s-worker-02` exclusion. Session pods also exclude that node by default. Additional cluster-specific exclusions belong in deployment values, not this portable chart. + +The chart creates dedicated controller, server, and session ServiceAccounts instead of a shared credential Secret. Controller and server pods disable automatic token mounting and receive explicit short-lived Kubernetes API projections. Each server pod also receives a separate 10-minute projected token with the fixed `t4-cluster-internal` audience and presents it only in the existing `omp-app/1` upstream authentication field when dialing a session host. Session pods use `automountServiceAccountToken: false`; their only Kubernetes identity mount is an explicit short-lived API-audience token plus the cluster CA and namespace. The session ServiceAccount may only create `authentication.k8s.io/tokenreviews`, and the host accepts a connection only when TokenReview confirms the expected server ServiceAccount username and audience. + +The Kubernetes API audience is configurable because managed clusters can reject the conventional `https://kubernetes.default.svc` audience. The same value is used for the controller API token, server API token, and session-host TokenReview credential. The internal server identity audience remains the fixed `t4-cluster-internal` boundary. DNS selectors default to the conventional `kube-system`/`k8s-app: kube-dns` pair and can be replaced together for clusters using a different DNS deployment. + +When chart-managed ingress is enabled, a host, ingress class, and TLS stanza are mandatory. For `ingressClassName: tailscale`, the chart renders the TLS host without a Secret reference so the Tailscale operator can provision and manage the certificate. Other ingress classes must reference an administrator-managed TLS Secret. + +## API configuration + +All APIs are namespaced under `cluster.t4.dev/v1alpha1`: + +- `T4ClusterHost` selects the reviewed RWX StorageClass, allowed runtime profile names, bounded projection policy, exact HTTPS origins, and optional CI Secret/ConfigMap references. +- `T4Workspace` selects a host, bounded repository metadata, size, and `Retain` or `Delete` storage retention. The controller always requests `ReadWriteMany`. +- `T4Session` selects a host, workspace, allowlisted runtime profile, optional initial-prompt Secret reference, GUI policy, and optional allowlisted CI repository/ref/commit metadata. + +Images, provider endpoints, resource policy, model routes, shell text, raw prompts, tokens, and secret values are not accepted in CRs. Status contains only `observedGeneration`, Kubernetes object references, infrastructure phases, PVC capacity/phase, and bounded conditions. It never reports OMP ids, agent trees, or runtime lifecycle truth. + +The optional initial prompt is referenced by Secret name and key `prompt`; the Secret is mounted only into that session pod. Do not place prompt content in the CR or Helm values. + +`T4Workspace` has `cluster.t4.dev/workspace-protection`. With `retentionPolicy: Delete`, deletion waits for its PVC to disappear. With `Retain`, the controller first removes its owner reference and marks it retained, then permits workspace deletion. `T4Session` has `cluster.t4.dev/session-cleanup` and waits for its pod and Service to disappear. + +## Images and runtime + +`cluster/images/controller/Dockerfile`, `cluster/images/cluster-server/Dockerfile`, and `cluster/images/session-runtime/Dockerfile` use digest-pinned multi-platform build bases. BuildKit selects the requested target platform; the Dockerfiles do not hardcode or label an architecture that was not built. Debian packages are resolved through a dated snapshot. Published chart values still require per-image immutable digests. + +The session runtime verifies and builds the exact OMP tag `t4code-17.0.5-appserver-10` at commit `8476f4451ed95c5d5401785d279a93d3c659fac4`. It preserves `t4-omp-authority/1`, starts the existing T4 session-host entrypoint, and provides Xvfb, a minimal window manager, and Chromium without privileged mode or host display access. The shared claim is mounted at `/workspace`; authority and browser state live in controller-selected per-session subdirectories. OMP configuration is copied from the read-only administrator ConfigMap projection into the private authority child home before launch. `/dev/shm` is an explicit memory-backed volume. Browser Preview remains the existing GUI stream and control surface. + +Session pods do not receive an automatically mounted ServiceAccount token. The explicit projected reviewer token can only create TokenReviews and is not resource-authorized. ConfigMap and credential inputs are namespace-local references; credential mode uses an explicit non-optional `SecretKeyRef`, while explicit unauthenticated mode adds no Secret reference. All containers drop capabilities, disallow privilege escalation, use RuntimeDefault seccomp, and use read-only root filesystems. No per-session NodePort, LoadBalancer, host network, host PID, host display, or hostPath is created. + +## Upgrade and rollback + +CRD changes must remain structural and additive. Helm installs files under `crds/` on first install but does not upgrade or delete them. Before every `helm upgrade`, an administrator must review and apply each newer CRD independently, wait for API discovery to serve the updated schemas, and only then upgrade workloads with the new immutable image digests: + +```sh +kubectl apply --server-side -f deploy/charts/t4-cluster/crds/ +kubectl wait --for=condition=Established crd/t4clusterhosts.cluster.t4.dev crd/t4workspaces.cluster.t4.dev crd/t4sessions.cluster.t4.dev +``` + +Do not rely on `helm upgrade` to change CRDs. If a CRD pre-upgrade apply fails validation, stop the upgrade and keep the currently deployed controller/server images; do not force-replace the CRD or remove stored versions. + +```sh +helm upgrade t4-cluster deploy/charts/t4-cluster --namespace t4-system --values operator-values.yaml +``` + +For an application rollback, retain the additive CRDs and use the previous known-compatible values and image digest set: + +```sh +helm rollback t4-cluster REVISION --namespace t4-system +``` + +Do not roll OMP independently of the T4 session runtime. Roll back the known-compatible T4/OMP image set together. The pinned authority boundary is not negotiated down. + +## Uninstall + +1. Stop accepting new workspace/session mutations at the gateway. +2. Delete `T4Session` resources and wait for their pods and Services to be removed. +3. Review every `T4Workspace` retention policy. `Delete` removes its PVC; `Retain` deliberately leaves an orphaned PVC for administrator recovery. +4. Run `helm uninstall t4-cluster --namespace t4-system`. +5. Remove retained PVCs only after their contents have been recovered or confirmed disposable. + +CRDs are not removed by `helm uninstall`. This preserves custom resources and retained storage across rollback/reinstall. Remove the three CRDs only as a separate, explicit administrative operation after confirming no instances remain; CRD deletion removes all instances regardless of their retention intent. diff --git a/e2e/built-web-server.ts b/e2e/built-web-server.ts index d4661d1f..0df03bf9 100644 --- a/e2e/built-web-server.ts +++ b/e2e/built-web-server.ts @@ -17,11 +17,19 @@ const MIME_TYPES: Readonly> = { ".woff2": "font/woff2", }; -function injectBackend(html: string): string { - const payload = JSON.stringify({ - wsUrl: "ws://127.0.0.1:1/v1/ws", - label: "PWA fixture backend", - }); +interface BuiltWebBackend { + readonly wsUrl: string; + readonly label: string; + readonly clusterOperatorEnabled?: true; +} + +const DEFAULT_BACKEND: BuiltWebBackend = { + wsUrl: "ws://127.0.0.1:1/v1/ws", + label: "PWA fixture backend", +}; + +function injectBackend(html: string, backend: BuiltWebBackend): string { + const payload = JSON.stringify(backend); const tag = ``; if (!html.includes("")) throw new Error("web dist index is missing "); return html.replace("", `${tag}`); @@ -32,7 +40,7 @@ export class BuiltWebServer { private readonly server: Server; private port = 0; - constructor() { + constructor(private readonly backend: BuiltWebBackend = DEFAULT_BACKEND) { this.server = createServer((request, response) => { void this.handle(request.url ?? "/", request.method ?? "GET") .then(({ body, contentType, status }) => { @@ -96,7 +104,7 @@ export class BuiltWebServer { } if (pathname === "/" || pathname === "/index.html") { return { - body: injectBackend(await readFile(resolve(WEB_DIST, "index.html"), "utf8")), + body: injectBackend(await readFile(resolve(WEB_DIST, "index.html"), "utf8"), this.backend), contentType: MIME_TYPES[".html"]!, status: 200, }; diff --git a/e2e/cluster-operator.spec.ts b/e2e/cluster-operator.spec.ts new file mode 100644 index 00000000..793a5bd3 --- /dev/null +++ b/e2e/cluster-operator.spec.ts @@ -0,0 +1,591 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { type AddressInfo } from "node:net"; +import { expect, test, type Page } from "@playwright/test"; +import { decodeServerFrame } from "../packages/protocol/src/index.ts"; +import WebSocket, { WebSocketServer } from "ws"; + +import { BuiltWebServer } from "./built-web-server.ts"; + +const PUBLIC_WSS = "wss://operator-fixture.tailnet.ts.net/v1/ws"; +const HOST = "cluster-host"; +const SESSION = "cluster-session"; +const VIEW = `${HOST}/${SESSION}`; +const PROOF_ROOT = "artifacts/cluster-proof"; +type ScenarioId = + | "gui-auth-isolation" + | "desktop-viewport" + | "mobile-viewport" + | "wire-reconnect-idempotency"; + +async function recordScenario( + page: Page, + id: ScenarioId, + assertions: readonly string[], + viewport: "desktop" | "mobile", +): Promise { + await mkdir(`${PROOF_ROOT}/scenarios`, { recursive: true }); + await mkdir(`${PROOF_ROOT}/screenshots`, { recursive: true }); + await page.screenshot({ + animations: "disabled", + path: `${PROOF_ROOT}/screenshots/${id}-${viewport}-redacted.png`, + }); + await writeFile( + `${PROOF_ROOT}/scenarios/${id}.json`, + `${JSON.stringify({ + schemaVersion: "t4-cluster-scenario/1", + id, + status: "passed", + observedAt: new Date().toISOString(), + assertions, + })}\n`, + ); +} +const WORKSPACE = { + id: "workspace-a", + displayName: "Release train", + phase: "Ready", + retentionPolicy: "Retain", + storageClass: "t4-workspaces-rwx", + capacity: "20Gi", + accessMode: "ReadWriteMany", + revision: "workspace-r2", + condition: { + type: "StorageReady", + status: "True", + reason: "Bound", + message: "The RWX claim is bound.", + observedGeneration: 2, + }, +} as const; + +const GRANTED_CAPABILITIES = [ + "sessions.read", + "sessions.manage", + "sessions.prompt", + "sessions.control", + "preview.read", + "preview.control", + "preview.input", + "ci.trigger", +] as const; +const GRANTED_FEATURES = ["resume", "host.watch", "cluster.operator", "preview.control"] as const; +const GUI_PREVIEW = { + previewId: "preview-a", + state: "ready", + url: "https://127.0.0.1:4173/", + revision: "preview-r1", + cursor: { epoch: "preview-1", seq: 1 }, + title: "Session GUI", + canGoBack: false, + canGoForward: false, + viewport: { width: 1280, height: 720, deviceScaleFactor: 1 }, + availableActions: ["navigate", "capture", "click", "fill", "type", "press", "scroll"], + authority: { + id: "omp-session", + label: "Session", + kind: "isolated-session", + requiresExplicitOptIn: false, + }, +} as const; +const DECOY_PREVIEW = { + ...GUI_PREVIEW, + previewId: "preview-0", + url: "https://127.0.0.1:4173/decoy", + revision: "preview-r2", + cursor: { epoch: "preview-1", seq: 2 }, + title: "Unrelated preview", +} as const; + +function sessionRef(ciStatus: "queued" | "running" | "success" = "running") { + return { + hostId: HOST, + sessionId: SESSION, + project: { projectId: "cluster/workspace-a", name: "Release train" }, + revision: "session-r3", + title: "Ship release", + status: "active", + updatedAt: "2026-07-20T12:00:00.000Z", + model: "gpt-proxy/gpt-5.6-sol", + liveState: { + phase: "running", + cluster: { + workspaceId: WORKSPACE.id, + phase: "Running", + gui: { state: "Ready", previewId: "preview-a" }, + }, + ci: { + provider: "woodpecker", + correlation: "exact", + repositoryId: "repo-a", + branch: "main", + ref: "refs/heads/main", + commit: "0123456789abcdef", + pipelineNumber: 42, + status: ciStatus, + currentStage: ciStatus === "success" ? "complete" : "verify", + startedAt: "2026-07-20T12:01:00.000Z", + link: "https://ci.tailnet.ts.net/repos/repo-a/pipeline/42", + }, + }, + }; +} + +function requestedPreviewId(frame: Record): string { + const args = frame.args; + if ( + args === null || + typeof args !== "object" || + Array.isArray(args) || + !("previewId" in args) || + typeof args.previewId !== "string" + ) { + throw new Error("preview command requires a preview id"); + } + return args.previewId; +} + +class OperatorWireFixture { + readonly commands: Array> = []; + readonly hellos: Array> = []; + readonly welcomes: Array<{ + readonly grantedCapabilities: readonly string[]; + readonly grantedFeatures: readonly string[]; + }> = []; + private readonly server = new WebSocketServer({ host: "127.0.0.1", port: 0, path: "/v1/ws" }); + private workspaceSeq = 2; + private ciStatus: "queued" | "running" | "success" = "running"; + + get url(): string { + const address = this.server.address() as AddressInfo | null; + if (address === null) throw new Error("operator fixture is not listening"); + return `ws://127.0.0.1:${address.port}/v1/ws`; + } + + async start(): Promise { + if (this.server.address() !== null) return; + await new Promise((resolve, reject) => { + this.server.once("listening", resolve); + this.server.once("error", reject); + }); + this.server.on("connection", (socket) => { + socket.on("message", (data) => this.receive(socket, JSON.parse(data.toString()) as Record)); + }); + } + + async stop(): Promise { + for (const socket of this.server.clients) socket.terminate(); + await new Promise((resolve) => this.server.close(() => resolve())); + } + + disconnectClients(): void { + for (const socket of this.server.clients) socket.close(1012, "fixture restart"); + } + + private send(socket: WebSocket, frame: unknown): void { + const decoded = decodeServerFrame(frame); + if (decoded.type === "welcome") this.welcomes.push(decoded); + socket.send(JSON.stringify(decoded)); + } + + private response(socket: WebSocket, frame: Record, result: unknown): void { + this.send(socket, { + v: "omp-app/1", + type: "response", + requestId: frame.requestId, + commandId: frame.commandId, + hostId: HOST, + ...(frame.sessionId === undefined ? {} : { sessionId: frame.sessionId }), + command: frame.command, + ok: true, + result, + }); + } + + private receive(socket: WebSocket, frame: Record): void { + if (frame.type === "hello") { + this.hellos.push(frame); + this.send(socket, { + v: "omp-app/1", + type: "welcome", + selectedProtocol: "omp-app/1", + hostId: HOST, + ompVersion: "17.0.5", + ompBuild: "8476f4451ed95c5d5401785d279a93d3c659fac4", + appserverVersion: "cluster-fixture", + appserverBuild: "redacted", + epoch: "cluster-epoch-1", + grantedCapabilities: GRANTED_CAPABILITIES, + grantedFeatures: GRANTED_FEATURES, + negotiatedLimits: {}, + authentication: "paired", + resumed: this.hellos.length > 1, + }); + return; + } + if (frame.type !== "command") return; + this.commands.push(frame); + switch (frame.command) { + case "session.list": + this.response(socket, frame, { + cursor: { epoch: "session-index-1", seq: 3 }, + sessions: [sessionRef(this.ciStatus)], + totalCount: 1, + truncated: false, + }); + return; + case "host.watch": + this.response(socket, frame, { + watchId: "cluster-watch", + cursor: { epoch: "session-index-1", seq: 3 }, + }); + return; + case "workspace.list": + this.response(socket, frame, { + cursor: { epoch: "workspace-index-1", seq: this.workspaceSeq }, + workspaces: [WORKSPACE], + }); + this.send(socket, { + v: "omp-app/1", + type: "workspace.state", + hostId: HOST, + workspaceId: WORKSPACE.id, + cursor: { epoch: "workspace-index-1", seq: this.workspaceSeq }, + revision: WORKSPACE.revision, + upsert: WORKSPACE, + }); + return; + case "session.attach": + this.response(socket, frame, { attached: true, cursor: { epoch: "transcript-1", seq: 1 } }); + this.send(socket, { + v: "omp-app/1", + type: "snapshot", + hostId: HOST, + sessionId: SESSION, + cursor: { epoch: "transcript-1", seq: 1 }, + revision: "session-r3", + entries: [], + }); + this.send(socket, { + v: "omp-app/1", + type: "agent", + hostId: HOST, + sessionId: SESSION, + agentId: "Main", + state: "running", + progress: 0.4, + detail: { kind: "main", title: "Main", startedAt: "2026-07-20T12:00:00.000Z" }, + }); + this.send(socket, { + v: "omp-app/1", + type: "agent", + hostId: HOST, + sessionId: SESSION, + agentId: "WorkerA", + state: "running", + progress: 0.7, + detail: { + parentId: "Main", + title: "WorkerA", + description: "Verify CI and wait for peer reply", + currentTool: "irc.wait", + evidence: "Parked peer revived; owner-scoped job still running", + startedAt: "2026-07-20T12:00:10.000Z", + }, + }); + this.send(socket, { + v: "omp-app/1", + type: "preview.state", + hostId: HOST, + sessionId: SESSION, + ...GUI_PREVIEW, + }); + this.send(socket, { + v: "omp-app/1", + type: "preview.state", + hostId: HOST, + sessionId: SESSION, + ...DECOY_PREVIEW, + }); + return; + case "session.steer": + case "session.prompt": + this.response(socket, frame, { accepted: true }); + return; + case "preview.state": + this.response(socket, frame, { + previews: [structuredClone(GUI_PREVIEW), structuredClone(DECOY_PREVIEW)], + }); + return; + case "preview.policy.check": + this.response(socket, frame, { allowed: true, confirmationRequired: false }); + return; + case "preview.lease.acquire": + this.response(socket, frame, { + previewId: requestedPreviewId(frame), + leaseId: "preview-lease-a", + expiresAt: Date.now() + 30_000, + }); + return; + case "preview.lease.release": + this.response(socket, frame, { + previewId: requestedPreviewId(frame), + released: true, + }); + return; + case "preview.type": + case "preview.fill": + case "preview.press": + case "preview.scroll": + this.response(socket, frame, { + preview: + requestedPreviewId(frame) === DECOY_PREVIEW.previewId + ? DECOY_PREVIEW + : GUI_PREVIEW, + }); + return; + case "ci.run": + this.ciStatus = "queued"; + this.response(socket, frame, { + triggered: true, + pipelineNumber: 43, + status: "queued", + }); + return; + default: + this.response(socket, frame, {}); + } + } +} + +let wire: OperatorWireFixture; +let web: BuiltWebServer; + +async function routeFixtureWss(page: Page): Promise { + await page.addInitScript( + ({ publicUrl, routedUrl }) => { + const NativeWebSocket = window.WebSocket; + class RoutedWebSocket extends NativeWebSocket { + constructor(url: string | URL, protocols?: string | string[]) { + super(String(url) === publicUrl ? routedUrl : url, protocols); + } + } + Object.defineProperties(RoutedWebSocket, { + CONNECTING: { value: NativeWebSocket.CONNECTING }, + OPEN: { value: NativeWebSocket.OPEN }, + CLOSING: { value: NativeWebSocket.CLOSING }, + CLOSED: { value: NativeWebSocket.CLOSED }, + }); + window.WebSocket = RoutedWebSocket; + }, + { publicUrl: PUBLIC_WSS, routedUrl: wire.url }, + ); +} + +async function openSession(page: Page, mobile = false): Promise { + await routeFixtureWss(page); + await page.goto(web.url, { waitUntil: "domcontentloaded" }); + if (mobile) await page.getByRole("button", { name: "Show session list" }).click(); + const row = page.locator(`[data-session-row="${VIEW}"]`); + await expect(row).toBeVisible(); + await row.click(); + await expect(page).toHaveURL(/#\/sessions\/cluster-host%2Fcluster-session|#\/sessions\/cluster-host\/cluster-session/u); +} + +test.describe("OMP/T4 cluster GUI boundaries", () => { + test.describe.configure({ mode: "serial" }); + + test.beforeAll(async () => { + wire = new OperatorWireFixture(); + await wire.start(); + web = new BuiltWebServer({ + wsUrl: PUBLIC_WSS, + label: "Redacted cluster fixture", + clusterOperatorEnabled: true, + }); + await web.start(); + }); + + test.afterAll(async () => { + await web?.stop(); + await wire?.stop(); + }); + + test("desktop attach, steer, and reconnect preserve canonical cursors without duplicate rows", async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await openSession(page); + const composer = page.getByRole("textbox", { name: "Message the session" }); + await composer.fill("Keep the current approach"); + await page.getByRole("button", { name: "Steer", exact: true }).click(); + await expect.poll(() => wire.commands.some((frame) => frame.command === "session.steer")).toBe(true); + + const helloCount = wire.hellos.length; + wire.disconnectClients(); + await expect.poll(() => wire.hellos.length).toBeGreaterThan(helloCount); + await expect(page.locator(`[data-session-row="${VIEW}"]`)).toHaveCount(1); + expect(wire.hellos.at(-1)?.requestedFeatures).toContain("cluster.operator"); + const latestHello = wire.hellos.at(-1); + const requestedCapabilities = ( + latestHello?.capabilities as { readonly client?: readonly string[] } | undefined + )?.client; + expect(latestHello?.requestedFeatures).toEqual( + expect.arrayContaining(["cluster.operator", "preview.control"]), + ); + expect(requestedCapabilities).toEqual( + expect.arrayContaining([ + "sessions.read", + "sessions.manage", + "ci.trigger", + "preview.read", + "preview.control", + "preview.input", + ]), + ); + const latestWelcome = wire.welcomes.at(-1); + expect(latestWelcome?.grantedFeatures).toEqual( + expect.arrayContaining(["cluster.operator", "preview.control"]), + ); + expect(latestWelcome?.grantedCapabilities).toEqual( + expect.arrayContaining(["sessions.read", "sessions.manage", "ci.trigger", "preview.read", "preview.control", "preview.input"]), + ); + await recordScenario( + page, + "wire-reconnect-idempotency", + ["hello.feature-granted", "reconnect.completed", "session.row-unique"], + "desktop", + ); + }); + + test("Agent View keeps exact OMP ids, parentage, attention, and progress alongside exact CI state", async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 900 }); + await openSession(page); + await page.goto(`${web.url}#/agents`); + await expect(page.getByRole("heading", { name: "Agent View" })).toBeVisible(); + await expect(page.getByRole("heading", { name: "Main", exact: true })).toBeVisible(); + await expect(page.getByRole("heading", { name: "WorkerA", exact: true })).toBeVisible(); + await expect(page.getByText("Parent: Main", { exact: false })).toBeVisible(); + await expect(page.getByRole("progressbar", { name: "70% done" })).toBeVisible(); + await expect(page.getByText("Parked peer revived; owner-scoped job still running", { exact: true })).toBeVisible(); + await expect(page.getByText("verify", { exact: true }).first()).toBeVisible(); + await expect(page.getByText("running", { exact: true }).first()).toBeVisible(); + await recordScenario( + page, + "desktop-viewport", + ["agent.parentage", "agent.progress", "ci.stage-exact"], + "desktop", + ); + }); + + test("phone explicitly chooses a cluster host and keeps CI and GUI workflows reachable", async ({ page }) => { + await page.emulateMedia({ reducedMotion: "reduce" }); + await page.setViewportSize({ width: 390, height: 844 }); + await openSession(page, true); + expect(wire.hellos.at(-1)?.requestedFeatures).toContain("cluster.operator"); + const helloCount = wire.hellos.length; + wire.disconnectClients(); + await expect.poll(() => wire.hellos.length).toBeGreaterThan(helloCount); + await page.getByRole("button", { name: "Show session list" }).click(); + await expect(page.locator(`[data-session-row="${VIEW}"]`)).toHaveCount(1); + await page + .getByRole("dialog", { name: "Working folders and sessions" }) + .getByRole("button", { name: "Close", exact: true }) + .click(); + await page.goto(`${web.url}#/hosts`); + const workspaceRow = page.locator('[data-cluster-workspace-id="workspace-a"]'); + await expect(workspaceRow).toHaveCount(1); + await expect(workspaceRow).toHaveAttribute("data-cluster-host-id", HOST); + await expect(workspaceRow).toContainText("t4-workspaces-rwx"); + await expect(page.getByRole("heading", { name: "Cluster workspaces" })).toBeVisible(); + + const creationHost = page.getByLabel("Creation host"); + const createWorkspace = page.getByRole("button", { name: "Create cluster workspace" }); + await expect(creationHost).toHaveValue(""); + await expect(createWorkspace).toBeDisabled(); + await creationHost.selectOption(HOST); + await expect(creationHost).toHaveValue(HOST); + await expect(createWorkspace).toBeEnabled(); + + const ciLink = page.getByRole("link", { name: "Open CI pipeline" }); + await expect(ciLink).toHaveAttribute( + "href", + "https://ci.tailnet.ts.net/repos/repo-a/pipeline/42", + ); + const openGui = page.getByRole("button", { name: "Open GUI" }); + const runCi = page.getByRole("button", { name: "Run CI" }); + await expect(openGui).toBeEnabled(); + await expect(runCi).toBeEnabled(); + for (const control of [creationHost, createWorkspace, openGui, runCi]) { + const box = await control.boundingBox(); + expect(box).not.toBeNull(); + expect(box!.height).toBeGreaterThanOrEqual(43.99); + } + expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true); + + await runCi.click(); + await expect.poll(() => wire.commands.some((frame) => frame.command === "ci.run")).toBe(true); + expect(wire.commands.findLast((frame) => frame.command === "ci.run")).toMatchObject({ + hostId: HOST, + sessionId: SESSION, + expectedRevision: "session-r3", + args: { + provider: "woodpecker", + action: "run", + repositoryId: "repo-a", + ref: "refs/heads/main", + commit: "0123456789abcdef", + }, + }); + + await openGui.click(); + await expect(page.getByRole("heading", { name: "Browser preview" })).toBeVisible(); + await expect( + page.getByRole("combobox", { name: "Preview", exact: true }), + ).toHaveValue("preview-a"); + await expect(page.getByRole("textbox", { name: "URL" })).toHaveValue(GUI_PREVIEW.url); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true); + expect( + Number.parseFloat( + await page.evaluate(() => + getComputedStyle(document.documentElement).getPropertyValue("--motion-duration-fast").trim(), + ), + ), + ).toBe(0); + await recordScenario( + page, + "mobile-viewport", + [ + "mobile.wss-only", + "workspace.host-selected", + "ci.route-qualified", + "gui.preview-selected", + "motion.reduced", + "touch.target", + ], + "mobile", + ); + }); + + test("phone Browser Preview gates input through lease/revision and denies cross-session GUI routes", async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await openSession(page, true); + await page.goto(`${web.url}#/hosts`); + await page.getByRole("button", { name: "Open GUI" }).click(); + await expect(page.getByRole("heading", { name: "Browser preview" })).toBeVisible(); + await page.getByRole("textbox", { name: "Text" }).fill("hello from phone"); + await page.getByRole("button", { name: "Type", exact: true }).click(); + await expect.poll(() => wire.commands.some((frame) => frame.command === "preview.lease.acquire")).toBe(true); + await expect.poll(() => wire.commands.some((frame) => frame.command === "preview.type")).toBe(true); + + await page.setViewportSize({ width: 844, height: 390 }); + await expect(page.getByRole("heading", { name: "Browser preview" })).toBeVisible(); + await page.goto(`${web.url}#/sessions/${encodeURIComponent(`${HOST}/other-session`)}/preview`); + await expect(page.getByRole("heading", { name: "Browser preview" })).toHaveCount(0); + await page.goBack(); + await expect(page.getByRole("heading", { name: "Browser preview" })).toBeVisible(); + await recordScenario( + page, + "gui-auth-isolation", + ["preview.lease-gated", "preview.input-forwarded", "cross-session.denied"], + "mobile", + ); + }); +}); diff --git a/ops/t4-maintainer/deploy-local.sh b/ops/t4-maintainer/deploy-local.sh index 23b17346..9d92c039 100755 --- a/ops/t4-maintainer/deploy-local.sh +++ b/ops/t4-maintainer/deploy-local.sh @@ -418,7 +418,7 @@ download_release_asset() { verify_release_checksum() { local sums=$1 asset=$2 name expected actual matches name=$(basename -- "$asset") - matches=$(awk -v name="$name" '$2 == name && $1 ~ /^[0-9a-f]{64}$/ {print $1}' "$sums") + matches=$(awk -v name="$name" '$2 == name && length($1) == 64 && $1 ~ /^[0-9a-f]+$/ {print $1}' "$sums") [[ $matches != *$'\n'* && $matches =~ ^[0-9a-f]{64}$ ]] || fail "release checksum is missing or ambiguous for $name" expected=$matches actual=$($SHA256SUM "$asset" | awk '{print $1}') @@ -1324,7 +1324,7 @@ $JQ -n \ allowedOrigin: $gateway_origin, port: $gateway_port, appSocket: $gateway_socket, - label: $gateway_label, + "label": $gateway_label, deploymentIdentity: $deployment_identity, nodeExecutable: $gateway_node_executable, runtimeSourceRoot: $runtime_root, diff --git a/ops/t4-maintainer/run.sh b/ops/t4-maintainer/run.sh index 54efd07f..81b06d3f 100755 --- a/ops/t4-maintainer/run.sh +++ b/ops/t4-maintainer/run.sh @@ -1019,7 +1019,7 @@ release_assets_are_public() { return 1 } if [[ $name != SHA256SUMS.txt ]]; then - expected_digest=$(awk -v name="$name" '$2 == name && $1 ~ /^[0-9a-f]{64}$/ {print $1}' "$manifest_file") + expected_digest=$(awk -v name="$name" '$2 == name && length($1) == 64 && $1 ~ /^[0-9a-f]+$/ {print $1}' "$manifest_file") [[ $expected_digest != *$'\n'* && $expected_digest =~ ^[0-9a-f]{64}$ ]] || { rm -f -- "$manifest_file" return 1 @@ -1037,7 +1037,7 @@ release_assets_are_public() { } fi done - manifest_entries=$(awk '$1 ~ /^[0-9a-f]{64}$/ && NF == 2 {count += 1} END {print count + 0}' "$manifest_file") + manifest_entries=$(awk 'length($1) == 64 && $1 ~ /^[0-9a-f]+$/ && NF == 2 {count += 1} END {print count + 0}' "$manifest_file") rm -f -- "$manifest_file" [[ $manifest_entries == 6 ]] } diff --git a/package.json b/package.json index f6f0387b..35184fa1 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,8 @@ "test:soak": "pnpm --filter @t4-code/client exec vp test run test/projection.test.ts -t \"retains a 10k snapshot\" && pnpm build:web && playwright test e2e/remote-app.spec.ts --grep \"@soak\"", "test:protocol": "vp run --filter @t4-code/protocol --filter @t4-code/fixture-server test", "test:legacy-bridge-continuity": "vp test run scripts/legacy-bridge-continuity.test.ts --reporter=verbose", + "test:cluster:ci": "node --test scripts/cluster-ci/*.test.mjs", + "test:cluster:e2e": "playwright test e2e/cluster-operator.spec.ts", "perf:core": "vp test run scripts/perf/core.test.ts", "perf:ui": "pnpm build:web && node scripts/perf/ui.mjs", "perf:electron": "pnpm build:web && pnpm build:desktop && node scripts/perf/electron.mjs", diff --git a/packages/client/src/desktop-runtime-bootstrap.ts b/packages/client/src/desktop-runtime-bootstrap.ts index 2b956369..3f0fe4fb 100644 --- a/packages/client/src/desktop-runtime-bootstrap.ts +++ b/packages/client/src/desktop-runtime-bootstrap.ts @@ -1,4 +1,4 @@ -import { decodeSessionListResult } from "@t4-code/protocol"; +import { CLUSTER_OPERATOR_FEATURE, decodeSessionListResult } from "@t4-code/protocol"; import type { CommandRequest, CommandResult } from "@t4-code/protocol/desktop-ipc"; import { DesktopRuntimeError } from "./desktop-runtime-contracts.ts"; import type { DesktopWelcomePayload } from "./desktop-runtime-contracts.ts"; @@ -12,6 +12,7 @@ export type DesktopBootstrapErrorReporter = (error: unknown, code: DesktopBootst export interface DesktopHostBootstrapOptions { readonly targetId: string; readonly frame: DesktopWelcomePayload; + readonly clusterOperatorEnabled?: boolean; readonly issue: DesktopBootstrapCommand; readonly onError?: DesktopBootstrapErrorReporter; readonly onSessionList?: (result: CommandResult) => void; @@ -46,6 +47,13 @@ export async function bootstrapDesktopHost(options: DesktopHostBootstrapOptions) } } } + if ( + options.clusterOperatorEnabled === true && + capability("sessions.read") && + feature(CLUSTER_OPERATOR_FEATURE) + ) { + await issueSafely({ hostId: host, command: "workspace.list", args: {} }); + } if (capability("catalog.read") && feature("catalog.metadata")) { await issueSafely({ hostId: host, command: "catalog.get", args: {} }); } diff --git a/packages/client/src/desktop-runtime-contracts.ts b/packages/client/src/desktop-runtime-contracts.ts index 80cc11e2..c208d2e4 100644 --- a/packages/client/src/desktop-runtime-contracts.ts +++ b/packages/client/src/desktop-runtime-contracts.ts @@ -50,6 +50,8 @@ import type { RuntimeIntegrationDescriptor } from "./runtime-integration.ts"; export interface DesktopShellPort { readonly kind: "desktop"; readonly platform: "linux" | "darwin"; + /** Local source option; absent means cluster.operator is not requested or projected. */ + readonly clusterOperatorEnabled?: boolean; readonly bootstrap: () => Promise; readonly connect: (request: TargetRequest) => Promise; readonly disconnect: (request: TargetRequest) => Promise; @@ -151,6 +153,8 @@ export interface DesktopRuntimeSnapshot { readonly catalogs: ReadonlyMap; readonly settings: ReadonlyMap; readonly projection: ProjectionSnapshot; + /** Local opt-in. Missing remains false for older renderer snapshots. */ + readonly clusterOperatorEnabled?: boolean; readonly runtimeErrors: readonly DesktopRuntimeErrorEntry[]; } @@ -163,6 +167,8 @@ export interface DesktopRuntimeTimerScheduler { export interface DesktopRuntimeOptions { readonly shell: DesktopShellPort; + /** Requests and projects cluster.operator only when explicitly true. */ + readonly clusterOperatorEnabled?: boolean; readonly projection?: ProjectionStore; readonly projectionOptions?: ProjectionOptions; readonly clock?: { now(): number }; diff --git a/packages/client/src/desktop-runtime.ts b/packages/client/src/desktop-runtime.ts index 9c095f2f..b1d2be6d 100644 --- a/packages/client/src/desktop-runtime.ts +++ b/packages/client/src/desktop-runtime.ts @@ -22,7 +22,7 @@ import { type TerminalResizeRequest, type TerminalResult, } from "@t4-code/protocol/desktop-ipc"; -import { decodeCatalog, decodeSessions, hostId, revision, sessionId, type CatalogFrame, type Cursor, type SettingsFrame } from "@t4-code/protocol"; +import { CLUSTER_OPERATOR_FEATURE, decodeCatalog, decodeCommandResult, decodeSessions, decodeWorkspaceInfrastructureProjection, hostId, revision, sessionId, type CatalogFrame, type Cursor, type SessionRef, type SettingsFrame, type WorkspaceInfrastructureProjection, type WorkspaceListResult } from "@t4-code/protocol"; import type { Unsubscribe } from "./index.ts"; import { ProjectionStore } from "./projection.ts"; import { @@ -97,6 +97,7 @@ export class DesktopRuntimeController { private readonly projection: ProjectionStore; private readonly ownsProjection: boolean; private readonly clock: { now(): number }; + private readonly clusterOperatorEnabled: boolean; private readonly timers: DesktopRuntimeTimerScheduler; private readonly sessionRefreshTimers = new Map(); private readonly sessionRefreshInFlight = new Map(); @@ -124,8 +125,12 @@ export class DesktopRuntimeController { private readonly controllerLeaseFallbackTtlMs = 20_000; constructor(options: DesktopRuntimeOptions) { this.shell = options.shell; + this.clusterOperatorEnabled = options.clusterOperatorEnabled === true; this.projection = options.projection ?? new ProjectionStore(options.projectionOptions); this.ownsProjection = options.projection === undefined; + // Cached cluster truth has no current welcome grant. It must be refreshed + // only after this runtime observes both authority gates for a host. + this.projection.clearWorkspaceInventory(); this.clock = options.clock ?? { now: () => Date.now() }; this.timers = options.timers ?? defaultTimerScheduler; this.promptLeases = new PromptLeaseStore({ clock: this.clock, issue: (request) => this.shell.command(request), invalidateTarget: async (targetId) => { try { await this.disconnect(targetId); } catch { /* uncertain acquire cleanup is best effort */ } } }); @@ -143,11 +148,12 @@ export class DesktopRuntimeController { catalogs: mapValue([]), settings: mapValue([]), projection: this.projection.getSnapshot(), + clusterOperatorEnabled: this.clusterOperatorEnabled, runtimeErrors: Object.freeze([]), }); this.projection.subscribe((snapshot) => { if (this.stopped) return; - this.replace({ projection: snapshot }); + this.replace({ projection: this.clearUnauthorizedWorkspaceInventories(snapshot) }); }); } getSnapshot(): DesktopRuntimeSnapshot { return this.current; } @@ -293,11 +299,14 @@ export class DesktopRuntimeController { private async issueCommand(targetId: string, intent: CommandRequest["intent"], capturedIdentity?: DesktopTargetIdentity): Promise { const identity = capturedIdentity ?? this.captureTargetIdentity(targetId); const result = freezeClone(await this.shell.command(freezeClone({ targetId, intent }))); - const isHostProductCommand = intent.command === "session.list" || intent.command === "host.list" || intent.command === "catalog.get" || intent.command === "settings.read"; + const isHostProductCommand = intent.command === "session.list" || intent.command === "host.list" || intent.command === "workspace.list" || intent.command === "catalog.get" || intent.command === "settings.read"; if (result.accepted === true && isHostProductCommand) { if (identity === undefined || !this.isCurrentTargetIdentity(identity) || !this.isCurrentHostProjection(identity) || identity.hostId !== String(intent.hostId)) { throw new DesktopRuntimeError("stale", "host product command completed for a stale target binding"); } + if (intent.command === "workspace.list" && !this.clusterProjectionGrantedForTarget(targetId, String(intent.hostId))) { + throw new DesktopRuntimeError("stale", "workspace inventory completed without current cluster operator authority"); + } this.applyHostCommandResult(targetId, String(intent.hostId), intent.command, result, identity); this.notifyValidatedHostCommand(targetId, String(intent.hostId), intent.command, result); if (intent.command === "catalog.get") this.catalogReadyByTarget.add(targetId); @@ -532,6 +541,85 @@ export class DesktopRuntimeController { const metadata = this.hostState.metadataForTarget(targetId); return metadata?.hostId === hostIdValue && metadata.grantedFeatures.includes("controller.lease"); } + private clusterProjectionGrantedForTarget(targetId: string, hostIdValue?: string): boolean { + if (!this.clusterOperatorEnabled) return false; + const metadata = this.hostState.metadataForTarget(targetId); + return metadata !== undefined && + (hostIdValue === undefined || metadata.hostId === hostIdValue) && + metadata.grantedCapabilities.includes("sessions.read") && + metadata.grantedFeatures.includes(CLUSTER_OPERATOR_FEATURE); + } + private clusterProjectionGrantedForHost( + hostIdValue: string, + targetHosts: ReadonlyMap = this.current.targetHosts, + ): boolean { + for (const [targetId, boundHostId] of targetHosts) { + if ( + boundHostId === hostIdValue && + this.clusterProjectionGrantedForTarget(targetId, hostIdValue) + ) return true; + } + return false; + } + private normalizeSessionInfrastructureEvent( + targetId: string, + event: RendererServerEvent, + ): RendererServerEvent { + if (event.kind !== "sessions" && event.kind !== "session.delta") return event; + const sourceHostId = typeof event.payload.hostId === "string" + ? event.payload.hostId + : this.current.targetHosts.get(targetId); + if ( + sourceHostId !== undefined && + this.clusterProjectionGrantedForTarget(targetId, sourceHostId) + ) return event; + + const withoutInfrastructure = (session: SessionRef): SessionRef => { + if (session.liveState?.cluster === undefined && session.liveState?.ci === undefined) return session; + const liveState = { ...session.liveState }; + delete liveState.cluster; + delete liveState.ci; + return Object.freeze({ ...session, liveState: Object.freeze(liveState) }); + }; + if (event.kind === "session.delta") { + if (event.payload.upsert === undefined) return event; + const upsert = withoutInfrastructure(event.payload.upsert); + if (upsert === event.payload.upsert) return event; + return Object.freeze({ + ...event, + payload: Object.freeze({ ...event.payload, upsert }), + }) as RendererServerEvent; + } + + let sessions: SessionRef[] | undefined; + for (const [index, session] of event.payload.sessions.entries()) { + const normalized = withoutInfrastructure(session); + if (normalized === session) continue; + sessions ??= [...event.payload.sessions]; + sessions[index] = normalized; + } + if (sessions === undefined) return event; + return Object.freeze({ + ...event, + payload: Object.freeze({ ...event.payload, sessions: Object.freeze(sessions) }), + }) as RendererServerEvent; + } + private clearUnauthorizedWorkspaceInventories( + snapshot: DesktopRuntimeSnapshot["projection"], + targetHosts: ReadonlyMap = this.current.targetHosts, + ): DesktopRuntimeSnapshot["projection"] { + const hosts = new Set(snapshot.workspaceCursors.keys()); + for (const itemKey of snapshot.workspaces.keys()) { + const separator = itemKey.indexOf("\u0000"); + if (separator > 0) hosts.add(itemKey.slice(0, separator)); + } + for (const hostIdValue of hosts) { + if (!this.clusterProjectionGrantedForHost(hostIdValue, targetHosts)) { + this.projection.clearWorkspaceInventory(hostIdValue); + } + } + return this.projection.getSnapshot(); + } private async acquireControllerLeaseNow( targetId: string, hostIdValue: string, @@ -695,20 +783,14 @@ export class DesktopRuntimeController { } private handleServerEvent(event: RendererServerEventEnvelope): void { const incomingEvent = event.event; - const transcriptEvent = this.isRetainedTranscriptEvent(incomingEvent); - // Do not deep-clone a potentially large transcript payload before applying - // retention. The shared projection consumes the decoded event directly; - // renderer subscribers receive only the bounded immutable copy. - const rendererEvent: RendererServerEventEnvelope = { - targetId: event.targetId, - event: transcriptEvent - ? sanitizeRetainedTranscriptEvent(incomingEvent) - : freezeClone(incomingEvent), - }; + if ( + incomingEvent.kind === "workspace.state" && + !this.clusterProjectionGrantedForTarget(event.targetId, String(incomingEvent.payload.hostId)) + ) return; if (incomingEvent.kind === "welcome") { if (!this.handleWelcome(event.targetId, incomingEvent.payload)) return; this.applyProjectionEvent(event.targetId, incomingEvent); - this.notifyServerEvents(rendererEvent); + this.notifyServerEvents({ targetId: event.targetId, event: freezeClone(incomingEvent) }); } else { const payload = asRecord(incomingEvent.payload) ?? {}; const hostIdValue = typeof payload.hostId === "string" ? payload.hostId : undefined; @@ -720,7 +802,7 @@ export class DesktopRuntimeController { const targetMetadata = this.hostState.metadataForTarget(event.targetId); const hostMetadata = hostIdValue === undefined ? undefined : this.current.hosts.get(hostIdValue); const hostProjectionCurrent = targetMetadata !== undefined && hostMetadata !== undefined && hostMetadata.targetId === event.targetId && targetMetadata.epoch === hostMetadata.epoch; - const hostProductResponse = incomingEvent.kind === "response" && incomingEvent.payload.ok && (incomingEvent.payload.command === "session.list" || incomingEvent.payload.command === "host.list" || incomingEvent.payload.command === "catalog.get" || incomingEvent.payload.command === "settings.read"); + const hostProductResponse = incomingEvent.kind === "response" && incomingEvent.payload.ok && (incomingEvent.payload.command === "session.list" || incomingEvent.payload.command === "host.list" || incomingEvent.payload.command === "workspace.list" || incomingEvent.payload.command === "catalog.get" || incomingEvent.payload.command === "settings.read"); const modernInventory = targetMetadata?.grantedCapabilities.includes("sessions.read") === true; if (hostIdValue !== undefined && !hostProjectionCurrent) return; if (incomingEvent.kind === "sessions" && modernInventory && (this.sessionInventoryEpochByTarget.get(event.targetId) !== incomingEvent.payload.cursor.epoch)) return; @@ -739,10 +821,18 @@ export class DesktopRuntimeController { this.replace({ settings: mapValue(new Map(this.current.settings).set(hostIdValue, settings)) }); } } - if (hostIdValue === undefined || hostProjectionCurrent || incomingEvent.kind === "response") { - this.applyProjectionEvent(event.targetId, incomingEvent); + const normalizedEvent = this.normalizeSessionInfrastructureEvent(event.targetId, incomingEvent); + if (hostIdValue === undefined || hostProjectionCurrent || normalizedEvent.kind === "response") { + this.applyProjectionEvent(event.targetId, normalizedEvent); + } + if (!hostProductResponse) { + this.notifyServerEvents({ + targetId: event.targetId, + event: this.isRetainedTranscriptEvent(normalizedEvent) + ? sanitizeRetainedTranscriptEvent(normalizedEvent) + : freezeClone(normalizedEvent), + }); } - if (!hostProductResponse) this.notifyServerEvents(rendererEvent); return; } } @@ -768,6 +858,10 @@ export class DesktopRuntimeController { } } private applyHostCommandResult(targetId: string, hostValue: string, command: string, response: CommandResult, expectedIdentity?: DesktopTargetIdentity): void { + if ( + command === "workspace.list" && + !this.clusterProjectionGrantedForTarget(targetId, hostValue) + ) return; const result = asRecord(response.result); if (result === undefined) throw new DesktopRuntimeError("protocol", "host response result is not an object"); if (command === "session.list" || command === "host.list") { @@ -783,6 +877,17 @@ export class DesktopRuntimeController { }); if (catalog.type !== "catalog") throw new DesktopRuntimeError("protocol", "catalog response decoded as settings"); this.replace({ catalogs: mapValue(new Map(this.current.catalogs).set(hostValue, catalog)) }); + } else if (command === "workspace.list") { + const decoded = decodeCommandResult(command, result) as unknown as WorkspaceListResult; + const workspaces: WorkspaceInfrastructureProjection[] = []; + for (const item of decoded.workspaces) { + try { + workspaces.push(decodeWorkspaceInfrastructureProjection(item)); + } catch { + // Legacy local workspace rows are not cluster infrastructure projections. + } + } + this.projection.replaceWorkspaceInventory(hostValue, workspaces, decoded.cursor); } else if (command === "settings.read") { const settings = decodeCatalog({ v: "omp-app/1", type: "settings", hostId: hostValue, revision: result.revision, settings: result.settings }); if (settings.type !== "settings") throw new DesktopRuntimeError("protocol", "settings response decoded as catalog"); @@ -820,6 +925,11 @@ export class DesktopRuntimeController { // the previous transport generation, even for the brief notification // before the welcome frame itself reaches the shared projection. this.projection.invalidateSessionInventory(String(frame.hostId)); + if (this.clusterProjectionGrantedForHost(String(frame.hostId), reconciled.targetHosts)) { + this.projection.invalidateWorkspaceInventory(String(frame.hostId)); + } else { + this.projection.clearWorkspaceInventory(String(frame.hostId)); + } if (this.current.connections.get(targetId) !== "connected") { this.welcomedBeforeConnected.add(targetId); } @@ -839,6 +949,7 @@ export class DesktopRuntimeController { if (identity === undefined) return; void bootstrapDesktopHost({ targetId, + clusterOperatorEnabled: this.clusterOperatorEnabled, frame, issue: (intent) => this.stopped ? Promise.resolve(undefined) : this.issueCommand(targetId, intent, identity), onError: (error, code) => { @@ -968,6 +1079,10 @@ export class DesktopRuntimeController { connections: reconciled.connections, targetHosts: reconciled.targetHosts, hosts: reconciled.hosts, + projection: this.clearUnauthorizedWorkspaceInventories( + this.projection.getSnapshot(), + reconciled.targetHosts, + ), }); for (const targetId of refreshAfterReconcile) this.requestSessionRefresh(targetId); } diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index e1b0d196..58d45f41 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -1,10 +1,13 @@ export { OmpClient, createOmpClient } from "./omp-client-runtime.ts"; export { + DEFAULT_CLUSTER_OPERATOR_ENABLED, OMP_RUNTIME_INTEGRATION, OMP_RUNTIME_KIND, T4_RUNTIME_FEATURES, availableRuntimeFeature, runtimeIdentityKey, + clusterOperatorRequestedCapabilities, + clusterOperatorRequestedFeatures, unavailableRuntimeFeature, } from "./runtime-integration.ts"; export type { @@ -96,6 +99,7 @@ export { applyPublicFrame, ProjectionStore, createProjectionStore, + MAX_INDEXED_WORKSPACES, MAX_RETAINED_TERMINALS, MAX_RETAINED_TERMINAL_BYTES, MAX_RETAINED_TERMINAL_BYTES_PER_TERMINAL, @@ -139,6 +143,7 @@ export type { ProjectionOptions, ProjectionSubscription, } from "./projection.ts"; +export type { WorkspaceInfrastructureProjection } from "@t4-code/protocol"; export { MAX_RETAINED_TRANSCRIPT_ENTRIES, MAX_RETAINED_TRANSCRIPT_BYTES, diff --git a/packages/client/src/omp-client-cursor.ts b/packages/client/src/omp-client-cursor.ts index 46ec95bd..85a4eed3 100644 --- a/packages/client/src/omp-client-cursor.ts +++ b/packages/client/src/omp-client-cursor.ts @@ -46,11 +46,18 @@ export class CursorJournal { const session = sessionId(record.sessionId); const key = sessionKey(String(host), String(session)); const normalized = { hostId: String(host), sessionId: String(session), cursor }; + // Map order is the journal's LRU order. Advancing an existing session must + // protect it from the next bounded eviction just like adding a new one. + this.records.delete(key); this.records.set(key, normalized); this.bySession.set(key, cursor); if (this.records.size > MAX_SAVED) { const oldest = this.records.keys().next().value; - if (typeof oldest === "string") this.records.delete(oldest); + if (typeof oldest === "string") { + this.records.delete(oldest); + this.bySession.delete(oldest); + this.saved.delete(oldest); + } } if (this.store === undefined) return; const queue = this.queues.get(key) ?? { latest: undefined, running: false }; diff --git a/packages/client/src/omp-client-runtime.ts b/packages/client/src/omp-client-runtime.ts index 33d6e4e3..c728a26b 100644 --- a/packages/client/src/omp-client-runtime.ts +++ b/packages/client/src/omp-client-runtime.ts @@ -998,6 +998,8 @@ export class OmpClient { private acceptSnapshot(event: PublicEvent<"snapshot">): void { const frame = event.payload; const currentKey = sessionKey(String(frame.hostId), String(frame.sessionId)); + const previous = this.cursorJournal.bySession.get(currentKey); + if (previous?.epoch === frame.cursor.epoch && frame.cursor.seq <= previous.seq) return; this.desyncedSessions.delete(currentKey); this.epochValue = frame.cursor.epoch; this.cursorValue = frame.cursor; diff --git a/packages/client/src/projection-cache.ts b/packages/client/src/projection-cache.ts index 7dac6269..d2950102 100644 --- a/packages/client/src/projection-cache.ts +++ b/packages/client/src/projection-cache.ts @@ -1,4 +1,4 @@ -import type { Cursor, DurableEntry, SessionRef } from "@t4-code/protocol"; +import { decodeWorkspaceInfrastructureProjection, type Cursor, type DurableEntry, type SessionRef, type WorkspaceInfrastructureProjection } from "@t4-code/protocol"; import { MAX_PROJECTION_CACHE_BYTES } from "@t4-code/protocol/desktop-ipc"; import { MAX_INDEXED_SESSION_REFS } from "./projection.ts"; import type { @@ -44,6 +44,9 @@ interface ProjectionCacheData { readonly sessionIndex: Array<[string, SessionRef]>; readonly sessionIndexMetadata?: Array<[string, SessionIndexMetadata]>; readonly sessionDeltaCursors?: Array<[string, Cursor]>; + readonly sessionInventoryCursors?: Array<[string, Cursor]>; + readonly workspaces?: Array<[string, WorkspaceInfrastructureProjection]>; + readonly workspaceCursors?: Array<[string, Cursor]>; readonly activeSessionKey?: string; readonly lru: string[]; readonly cursor?: Cursor; @@ -271,6 +274,15 @@ export function encodeProjectionCache(snapshot: ProjectionSnapshot, savedAt = Da sessionDeltaCursors: [...snapshot.sessionDeltaCursors.entries()] .slice(0, MAX_INDEXED_SESSION_REFS) .map(([key, value]) => [key, safeJson(value) as Cursor]), + sessionInventoryCursors: [...snapshot.sessionInventoryCursors.entries()] + .slice(0, MAX_INDEXED_SESSION_REFS) + .map(([key, value]) => [key, safeJson(value) as Cursor]), + workspaces: [...snapshot.workspaces.entries()] + .slice(0, MAX_INDEXED_SESSION_REFS) + .map(([key, value]) => [key, safeJson(value) as WorkspaceInfrastructureProjection]), + workspaceCursors: [...snapshot.workspaceCursors.entries()] + .slice(0, MAX_INDEXED_SESSION_REFS) + .map(([key, value]) => [key, safeJson(value) as Cursor]), ...(snapshot.activeSessionKey === undefined ? {} : { activeSessionKey: snapshot.activeSessionKey }), @@ -801,6 +813,51 @@ export function decodeProjectionCache( sessionDeltaCursors.set(item[0], Object.freeze({ epoch: item[1].epoch, seq: item[1].seq })); } } + const sessionInventoryCursors = new Map(); + if (Array.isArray(data.sessionInventoryCursors)) { + for (const item of data.sessionInventoryCursors.slice(0, MAX_INDEXED_SESSION_REFS)) { + if ( + !Array.isArray(item) || + typeof item[0] !== "string" || + !isRecord(item[1]) || + typeof item[1].epoch !== "string" || + typeof item[1].seq !== "number" || + !Number.isSafeInteger(item[1].seq) || + item[1].seq < 0 + ) + continue; + sessionInventoryCursors.set( + item[0], + Object.freeze({ epoch: item[1].epoch, seq: item[1].seq }), + ); + } + } + const workspaces = new Map(); + if (Array.isArray(data.workspaces)) { + for (const item of data.workspaces.slice(0, MAX_INDEXED_SESSION_REFS)) { + if (!Array.isArray(item) || typeof item[0] !== "string") continue; + try { + workspaces.set(item[0], decodeWorkspaceInfrastructureProjection(item[1])); + } catch { + // One malformed cached projection cannot revive operator state. + } + } + } + const workspaceCursors = new Map(); + if (Array.isArray(data.workspaceCursors)) { + for (const item of data.workspaceCursors.slice(0, MAX_INDEXED_SESSION_REFS)) { + if ( + !Array.isArray(item) || + typeof item[0] !== "string" || + !isRecord(item[1]) || + typeof item[1].epoch !== "string" || + typeof item[1].seq !== "number" || + !Number.isSafeInteger(item[1].seq) || + item[1].seq < 0 + ) continue; + workspaceCursors.set(item[0], Object.freeze({ epoch: item[1].epoch, seq: item[1].seq })); + } + } const lru = Array.isArray(data.lru) ? data.lru .filter((item): item is string => typeof item === "string" && sessions.has(item)) @@ -819,6 +876,9 @@ export function decodeProjectionCache( sessionIndexMetadata: new ImmutableMap(sessionIndexMetadata), sessionRefArrivalOrdinals: new ImmutableMap(), sessionDeltaCursors: new ImmutableMap(sessionDeltaCursors), + sessionInventoryCursors: new ImmutableMap(sessionInventoryCursors), + workspaces: new ImmutableMap(workspaces), + workspaceCursors: new ImmutableMap(workspaceCursors), lru: Object.freeze(lru), ...(activeSessionKey === undefined ? {} : { activeSessionKey }), ...(isRecord(data.cursor) && diff --git a/packages/client/src/projection.ts b/packages/client/src/projection.ts index 4528aec7..edc4055f 100644 --- a/packages/client/src/projection.ts +++ b/packages/client/src/projection.ts @@ -1,10 +1,11 @@ -import { isCursor } from "@t4-code/protocol"; +import { decodeWorkspaceInfrastructureProjection, isCursor } from "@t4-code/protocol"; import type { Cursor, DurableEntry, OmpServerFrame, SessionEvent, SessionRef, + WorkspaceInfrastructureProjection, } from "@t4-code/protocol"; import { ImmutableSet } from "./immutable-set.ts"; import { ImmutableMap } from "./immutable-map.ts"; @@ -199,6 +200,12 @@ export interface ProjectionSnapshot { readonly sessionRefArrivalOrdinals: ReadonlyMap; /** Session-list delta cursors are independent from transcript cursors. */ readonly sessionDeltaCursors: ReadonlyMap; + /** Ordered complete session-list cursors, independently retained per host. */ + readonly sessionInventoryCursors: ReadonlyMap; + /** Cluster workspace lifecycle, bounded and keyed by host + workspace id. */ + readonly workspaces: ReadonlyMap; + /** Workspace lifecycle cursors are independent from session and transcript cursors. */ + readonly workspaceCursors: ReadonlyMap; readonly lru: readonly string[]; readonly cursor?: Cursor; readonly epoch?: string; @@ -210,6 +217,7 @@ export interface ProjectionSnapshot { export interface ProjectionOptions { readonly maxWarmSessions?: number; readonly maxIndexedSessions?: number; + readonly maxWorkspaces?: number; readonly maxEntries?: number; readonly maxTranscriptBytes?: number; readonly maxEntryBytes?: number; @@ -243,6 +251,7 @@ export interface ProjectionSubscription { } export const MAX_INDEXED_SESSION_REFS = 1000; +export const MAX_INDEXED_WORKSPACES = 1000; export const MAX_RETAINED_TERMINALS = 64; export const MAX_RETAINED_TERMINAL_BYTES = 1024 * 1024; export const MAX_RETAINED_TERMINAL_BYTES_PER_TERMINAL = 256 * 1024; @@ -254,6 +263,7 @@ export const MAX_RETAINED_PREVIEW_EVENTS = 128; const DEFAULT_OPTIONS: Required = { maxWarmSessions: 8, maxIndexedSessions: MAX_INDEXED_SESSION_REFS, + maxWorkspaces: MAX_INDEXED_WORKSPACES, maxEntries: MAX_RETAINED_TRANSCRIPT_ENTRIES, maxTranscriptBytes: MAX_RETAINED_TRANSCRIPT_BYTES, maxEntryBytes: MAX_RETAINED_TRANSCRIPT_ENTRY_BYTES, @@ -631,12 +641,55 @@ export function createProjectionSnapshot(): ProjectionSnapshot { sessionIndexMetadata: immutableMap(), sessionRefArrivalOrdinals: immutableMap(), sessionDeltaCursors: immutableMap(), + sessionInventoryCursors: immutableMap(), + workspaces: immutableMap(), + workspaceCursors: immutableMap(), lru: freezeArray([]), freshness: "fresh" as const, arrivalOrdinal: 0, }); } + +function applyWorkspaceInventory( + snapshot: ProjectionSnapshot, + host: string, + workspaces: readonly WorkspaceInfrastructureProjection[], + cursor: Cursor | undefined, + maxWorkspaces: number, +): ProjectionSnapshot { + if (cursor !== undefined) { + const previousCursor = snapshot.workspaceCursors.get(host); + if ( + previousCursor !== undefined && + previousCursor.epoch === cursor.epoch && + cursor.seq < previousCursor.seq + ) + return snapshot; + } + let nextWorkspaces = snapshot.workspaces; + for (const [itemKey] of nextWorkspaces) { + if (itemKey.startsWith(`${host}\u0000`)) nextWorkspaces = mapWithout(nextWorkspaces, itemKey); + } + for (const raw of workspaces.slice(0, maxWorkspaces)) { + const workspace = decodeWorkspaceInfrastructureProjection(raw); + nextWorkspaces = mapWith( + nextWorkspaces, + key(host, workspace.id), + Object.freeze(workspace), + maxWorkspaces, + ); + } + const workspaceCursors = + cursor === undefined + ? snapshot.workspaceCursors + : mapWith(snapshot.workspaceCursors, host, Object.freeze({ ...cursor }), maxWorkspaces); + if (nextWorkspaces === snapshot.workspaces && workspaceCursors === snapshot.workspaceCursors) { + return snapshot; + } + return Object.freeze({ ...snapshot, workspaces: nextWorkspaces, workspaceCursors }); +} + function nextArrivalOrdinal(snapshot: ProjectionSnapshot): number { return snapshot.arrivalOrdinal < Number.MAX_SAFE_INTEGER ? snapshot.arrivalOrdinal + 1 @@ -1012,6 +1065,44 @@ function applyProjectionInput( ): ProjectionSnapshot { const config = resolveProjectionOptions(options); switch (frame.type) { + case "workspace.state": { + const host = boundedIdentity(frame.hostId); + const workspaceId = boundedIdentity(frame.workspaceId); + if (host === undefined || workspaceId === undefined) return snapshot; + const previousCursor = snapshot.workspaceCursors.get(host); + if ( + previousCursor !== undefined && + previousCursor.epoch === frame.cursor.epoch && + frame.cursor.seq <= previousCursor.seq + ) + return snapshot; + let workspaces = snapshot.workspaces; + if (frame.upsert !== undefined) { + const workspace = decodeWorkspaceInfrastructureProjection(frame.upsert); + if (workspace.id !== workspaceId) return snapshot; + workspaces = mapWith( + workspaces, + key(host, workspaceId), + Object.freeze(workspace), + config.maxWorkspaces, + ); + } else if (frame.remove !== undefined) { + if (String(frame.remove) !== workspaceId) return snapshot; + workspaces = mapWithout(workspaces, key(host, workspaceId)); + } else { + return snapshot; + } + return Object.freeze({ + ...snapshot, + workspaces, + workspaceCursors: mapWith( + snapshot.workspaceCursors, + host, + Object.freeze({ ...frame.cursor }), + config.maxWorkspaces, + ), + }); + } case "sessions": { const arrivalOrdinal = nextArrivalOrdinal(snapshot); const refs = frame.sessions @@ -1019,6 +1110,15 @@ function applyProjectionInput( .map((ref) => sanitizeSessionRef(ref)) .filter((ref): ref is SessionRef => ref !== undefined); const authoritativeHosts = authoritativeSessionHosts(frame, refs); + for (const host of authoritativeHosts) { + const previousCursor = snapshot.sessionInventoryCursors.get(host); + if ( + previousCursor !== undefined && + previousCursor.epoch === frame.cursor.epoch && + frame.cursor.seq < previousCursor.seq + ) + return snapshot; + } const incomingKeys = new Set( refs.map((ref) => key(String(ref.hostId), String(ref.sessionId))), ); @@ -1076,12 +1176,22 @@ function applyProjectionInput( config.maxIndexedSessions, ); } + let sessionInventoryCursors = snapshot.sessionInventoryCursors; + for (const hostId of authoritativeHosts) { + sessionInventoryCursors = mapWith( + sessionInventoryCursors, + hostId, + Object.freeze({ ...frame.cursor }), + config.maxIndexedSessions, + ); + } let next = Object.freeze({ ...snapshot, sessionIndex, sessionIndexMetadata, sessionRefArrivalOrdinals, sessionDeltaCursors, + sessionInventoryCursors, sessions, lru, activeSessionKey, @@ -1202,6 +1312,14 @@ function applyProjectionInput( } case "snapshot": { const sessionKey = key(String(frame.hostId), String(frame.sessionId)); + const current = snapshot.sessions.get(sessionKey); + if ( + current?.cursor !== undefined && + current.cursor.epoch === frame.cursor.epoch && + (frame.cursor.seq < current.cursor.seq || + (frame.cursor.seq === current.cursor.seq && current.freshness !== "cached")) + ) + return snapshot; const retained = retainDurableEntries(frame.entries, { maxEntries: config.maxEntries, maxBytes: config.maxTranscriptBytes, @@ -1634,6 +1752,7 @@ function applyProjectionInput( // but do not let their old completeness metadata prove that a route is // gone until the host sends the next authoritative sessions frame. const sessionIndexMetadata = mapWithout(snapshot.sessionIndexMetadata, String(frame.hostId)); + const sessionInventoryCursors = mapWithout(snapshot.sessionInventoryCursors, String(frame.hostId)); const sessions = immutableMap( [...snapshot.sessions.entries()].map( ([sessionKey, session]) => @@ -1662,7 +1781,7 @@ function applyProjectionInput( ), ); if (snapshot.epoch === undefined || snapshot.epoch === frame.epoch) { - return updateRoot(Object.freeze({ ...snapshot, sessionIndexMetadata, sessions }), { + return updateRoot(Object.freeze({ ...snapshot, sessionIndexMetadata, sessionInventoryCursors, sessions }), { epoch: frame.epoch, freshness: "fresh", }); @@ -1670,6 +1789,7 @@ function applyProjectionInput( return Object.freeze({ ...snapshot, sessionIndexMetadata, + sessionInventoryCursors, sessions, epoch: frame.epoch, freshness: "catching-up", @@ -1762,6 +1882,81 @@ export class ProjectionStore { } return next; } + replaceWorkspaceInventory( + hostId: string, + workspaces: readonly WorkspaceInfrastructureProjection[], + cursor?: Cursor, + ): ProjectionSnapshot { + if (this.disposed) return this.current; + const host = boundedIdentity(hostId); + if (host === undefined) return this.current; + const next = applyWorkspaceInventory( + this.current, + host, + workspaces, + cursor, + this.options.maxWorkspaces, + ); + if (next === this.current) return next; + this.mutationGeneration += 1; + this.current = next; + this.queueCacheSave(); + for (const listener of Array.from(this.listeners)) { + try { + listener(next); + } catch { + /* listener isolation */ + } + } + return next; + } + invalidateWorkspaceInventory(hostId: string): ProjectionSnapshot { + if (this.disposed || !this.current.workspaceCursors.has(hostId)) return this.current; + const next = Object.freeze({ + ...this.current, + workspaceCursors: mapWithout(this.current.workspaceCursors, hostId), + }); + this.mutationGeneration += 1; + this.current = next; + this.queueCacheSave(); + return next; + } + clearWorkspaceInventory(hostId?: string): ProjectionSnapshot { + if (this.disposed) return this.current; + const host = hostId === undefined ? undefined : boundedIdentity(hostId); + if (hostId !== undefined && host === undefined) return this.current; + let workspaces = this.current.workspaces; + let workspaceCursors = this.current.workspaceCursors; + if (host === undefined) { + if (workspaces.size === 0 && workspaceCursors.size === 0) return this.current; + workspaces = immutableMap(); + workspaceCursors = immutableMap(); + } else { + const prefix = `${host}\u0000`; + let retained: Map | undefined; + for (const itemKey of workspaces.keys()) { + if (!itemKey.startsWith(prefix)) continue; + retained ??= new Map(workspaces); + retained.delete(itemKey); + } + const hasCursor = workspaceCursors.has(host); + if (retained === undefined && !hasCursor) return this.current; + if (retained !== undefined) workspaces = immutableMap(retained); + if (hasCursor) workspaceCursors = mapWithout(workspaceCursors, host); + } + const next = Object.freeze({ ...this.current, workspaces, workspaceCursors }); + this.mutationGeneration += 1; + this.current = next; + this.queueCacheSave(); + for (const listener of Array.from(this.listeners)) { + try { + listener(next); + } catch { + /* listener isolation */ + } + } + return next; + } activateSession(hostId: string, sessionId: string): ProjectionSnapshot { if (this.disposed) return this.current; const next = Object.freeze({ @@ -1781,14 +1976,27 @@ export class ProjectionStore { * list whose host-wide cursor may legitimately restart at sequence zero. */ invalidateSessionInventory(hostId?: string): ProjectionSnapshot { - if (this.disposed || this.current.sessionIndexMetadata.size === 0) return this.current; - if (hostId !== undefined && !this.current.sessionIndexMetadata.has(hostId)) return this.current; + if (this.disposed) return this.current; + if ( + hostId !== undefined && + !this.current.sessionIndexMetadata.has(hostId) && + !this.current.sessionInventoryCursors.has(hostId) + ) + return this.current; const sessionIndexMetadata = hostId === undefined ? immutableMap() : mapWithout(this.current.sessionIndexMetadata, hostId); - if (sessionIndexMetadata === this.current.sessionIndexMetadata) return this.current; - const next = Object.freeze({ ...this.current, sessionIndexMetadata }); + const sessionInventoryCursors = + hostId === undefined + ? immutableMap() + : mapWithout(this.current.sessionInventoryCursors, hostId); + if ( + sessionIndexMetadata === this.current.sessionIndexMetadata && + sessionInventoryCursors === this.current.sessionInventoryCursors + ) + return this.current; + const next = Object.freeze({ ...this.current, sessionIndexMetadata, sessionInventoryCursors }); this.mutationGeneration += 1; this.current = next; this.queueCacheSave(); diff --git a/packages/client/src/runtime-integration.ts b/packages/client/src/runtime-integration.ts index c2d92da4..7ba9dcb8 100644 --- a/packages/client/src/runtime-integration.ts +++ b/packages/client/src/runtime-integration.ts @@ -6,6 +6,34 @@ * can actually support. */ +import { CI_TRIGGER_CAPABILITY, CLUSTER_OPERATOR_FEATURE } from "@t4-code/protocol"; + +export const DEFAULT_CLUSTER_OPERATOR_ENABLED = false as const; + +/** + * Cluster operation is a local product opt-in, not an automatic consequence + * of a new wire version. Keep the caller's stable array when no filtering is + * needed and never synthesize a feature the endpoint did not publish. + */ +export function clusterOperatorRequestedFeatures( + features: readonly string[], + enabled: boolean = DEFAULT_CLUSTER_OPERATOR_ENABLED, +): readonly string[] { + const contains = features.includes(CLUSTER_OPERATOR_FEATURE); + if (enabled || !contains) return features; + return Object.freeze(features.filter((feature) => feature !== CLUSTER_OPERATOR_FEATURE)); +} + +export function clusterOperatorRequestedCapabilities( + capabilities: readonly string[], + enabled: boolean = DEFAULT_CLUSTER_OPERATOR_ENABLED, +): readonly string[] { + const contains = capabilities.includes(CI_TRIGGER_CAPABILITY); + if (enabled || !contains) return capabilities; + return Object.freeze( + capabilities.filter((capability) => capability !== CI_TRIGGER_CAPABILITY), + ); +} export const OMP_RUNTIME_KIND = "omp" as const; /** Additions to this union are deliberate product decisions, not wire changes. */ diff --git a/packages/client/test/client.test.ts b/packages/client/test/client.test.ts index 643ce2d9..4d471ca4 100644 --- a/packages/client/test/client.test.ts +++ b/packages/client/test/client.test.ts @@ -23,6 +23,8 @@ import { type PublicOmpServerEvent, type TimerScheduler, } from "../src/index.ts"; +import { CursorJournal } from "../src/omp-client-cursor.ts"; +import { MAX_SAVED, sessionKey as cursorSessionKey } from "../src/omp-client-contracts.ts"; const HOST = "host-fixture"; const SESSION = "session-fixture"; @@ -639,6 +641,36 @@ describe("OmpClient protocol state machine", () => { expect(loadErrors).toContain("storage"); await failing.close(); }); + it("evicts the least recently advanced cursor instead of an active session", () => { + const journal = new CursorJournal( + undefined, + () => undefined, + () => { throw new Error("unexpected storage error"); }, + ); + for (let index = 0; index < MAX_SAVED; index += 1) { + journal.remember({ + hostId: HOST, + sessionId: `session-${index}`, + cursor: { epoch: "epoch-a", seq: index }, + }); + } + journal.remember({ + hostId: HOST, + sessionId: "session-0", + cursor: { epoch: "epoch-a", seq: MAX_SAVED }, + }); + journal.remember({ + hostId: HOST, + sessionId: `session-${MAX_SAVED}`, + cursor: { epoch: "epoch-a", seq: MAX_SAVED }, + }); + + expect(journal.records.size).toBe(MAX_SAVED); + expect(journal.records.has(cursorSessionKey(HOST, "session-0"))).toBe(true); + expect(journal.bySession.has(cursorSessionKey(HOST, "session-0"))).toBe(true); + expect(journal.records.has(cursorSessionKey(HOST, "session-1"))).toBe(false); + expect(journal.bySession.has(cursorSessionKey(HOST, "session-1"))).toBe(false); + }); it("serializes deferred cursor saves, coalesces latest, and continues after failure", async () => { const store = new DeferredStore(); const transport = new FakeTransport({ welcome: welcome() }); diff --git a/packages/client/test/cluster-operator.test.ts b/packages/client/test/cluster-operator.test.ts new file mode 100644 index 00000000..219e94bd --- /dev/null +++ b/packages/client/test/cluster-operator.test.ts @@ -0,0 +1,179 @@ +import { + CI_TRIGGER_CAPABILITY, + CLUSTER_OPERATOR_FEATURE, + hostId, + revision, + type WorkspaceStateFrame, +} from "@t4-code/protocol"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { bootstrapDesktopHost, type DesktopBootstrapCommand } from "../src/desktop-runtime-bootstrap.ts"; +import { + DEFAULT_CLUSTER_OPERATOR_ENABLED, + applyPublicFrame, + clusterOperatorRequestedFeatures, + createProjectionSnapshot, + ProjectionStore, + type ProjectionFrame, +} from "../src/index.ts"; + +const HOST = hostId("cluster-host"); +const welcome = { + hostId: HOST, + grantedCapabilities: ["sessions.read", "sessions.manage", CI_TRIGGER_CAPABILITY], + grantedFeatures: [CLUSTER_OPERATOR_FEATURE, "host.watch"], +} as never; + +function workspaceState( + seq: number, + phase: "Pending" | "Ready" | "Failed" | "Terminating" | "Unknown", + workspaceId = "workspace-a", +): WorkspaceStateFrame { + return { + v: "omp-app/1", + type: "workspace.state", + hostId: HOST, + workspaceId, + cursor: { epoch: "workspace-epoch", seq }, + revision: revision(`workspace-r${seq}`), + upsert: { + id: workspaceId, + displayName: "Operator workspace", + phase, + retentionPolicy: "Retain", + capacity: "20Gi", + storageClass: "rwx", + accessMode: "ReadWriteMany", + revision: revision(`workspace-r${seq}`), + }, + } as WorkspaceStateFrame; +} + +describe("cluster operator client contract", () => { + it("is source-default-off and requests the one feature only after explicit opt-in", () => { + expect(DEFAULT_CLUSTER_OPERATOR_ENABLED).toBe(false); + expect(clusterOperatorRequestedFeatures(["resume", CLUSTER_OPERATOR_FEATURE])).toEqual([ + "resume", + ]); + expect( + clusterOperatorRequestedFeatures(["resume", CLUSTER_OPERATOR_FEATURE], true), + ).toEqual(["resume", CLUSTER_OPERATOR_FEATURE]); + }); + + it("does not issue cluster bootstrap commands while the local option is off", async () => { + const issue = vi.fn(async (_intent: Parameters[0]) => ({ + targetId: "cluster", + requestId: "bootstrap-request", + commandId: "bootstrap-command", + accepted: true as const, + result: { cursor: { epoch: "session-epoch", seq: 1 }, sessions: [] }, + })); + + await bootstrapDesktopHost({ targetId: "cluster", frame: welcome, issue }); + + expect(issue).toHaveBeenCalledTimes(2); + expect(issue.mock.calls.map(([intent]) => intent.command)).toEqual([ + "session.list", + "host.watch", + ]); + expect(issue).not.toHaveBeenCalledWith( + expect.objectContaining({ command: "workspace.list" }), + ); + }); + + it("bootstraps cluster workspace state only when enabled and advertised", async () => { + const commands: string[] = []; + const issue = vi.fn(async (intent: { readonly command: string }) => { + commands.push(intent.command); + return { + accepted: true, + result: + intent.command === "session.list" + ? { cursor: { epoch: "session-epoch", seq: 1 }, sessions: [] } + : intent.command === "workspace.list" + ? { + cursor: { epoch: "workspace-epoch", seq: 4 }, + workspaces: [workspaceState(4, "Ready").upsert], + } + : {}, + }; + }) as never; + + await bootstrapDesktopHost({ + targetId: "cluster", + frame: welcome, + issue, + clusterOperatorEnabled: true, + }); + + expect(commands).toEqual(["session.list", "host.watch", "workspace.list"]); + }); + + it("keeps workspace cursors independent and drops duplicate replay", () => { + let state = createProjectionSnapshot(); + state = applyPublicFrame(state, workspaceState(1, "Pending") as ProjectionFrame); + const first = state.workspaces.get(`${String(HOST)}\u0000workspace-a`); + expect(first?.phase).toBe("Pending"); + expect(state.workspaceCursors.get(String(HOST))).toEqual({ + epoch: "workspace-epoch", + seq: 1, + }); + + state = applyPublicFrame(state, workspaceState(1, "Failed") as ProjectionFrame); + expect(state.workspaces.get(`${String(HOST)}\u0000workspace-a`)).toBe(first); + + state = applyPublicFrame( + state, + { + v: "omp-app/1", + type: "sessions", + hostId: HOST, + cursor: { epoch: "session-epoch", seq: 900 }, + sessions: [], + totalCount: 0, + truncated: false, + } as ProjectionFrame, + ); + state = applyPublicFrame(state, workspaceState(2, "Ready") as ProjectionFrame); + expect(state.workspaces.get(`${String(HOST)}\u0000workspace-a`)?.phase).toBe("Ready"); + expect(state.workspaceCursors.get(String(HOST))?.seq).toBe(2); + }); + + it("accepts a complete workspace inventory at the same cursor as the latest delta", () => { + const store = new ProjectionStore(); + store.applyPublicFrame(workspaceState(2, "Ready") as ProjectionFrame); + store.replaceWorkspaceInventory( + String(HOST), + [workspaceState(2, "Failed", "workspace-b").upsert!], + { epoch: "workspace-epoch", seq: 2 }, + ); + + expect(store.snapshot.workspaces.has(`${String(HOST)}\u0000workspace-a`)).toBe(false); + expect(store.snapshot.workspaces.get(`${String(HOST)}\u0000workspace-b`)?.phase).toBe("Failed"); + expect(store.snapshot.workspaceCursors.get(String(HOST))?.seq).toBe(2); + }); + + it("applies exact workspace removals without disturbing another host", () => { + const hostB = hostId("cluster-host-b"); + let state = applyPublicFrame(createProjectionSnapshot(), workspaceState(1, "Ready") as ProjectionFrame); + state = applyPublicFrame( + state, + { ...workspaceState(1, "Ready", "workspace-b"), hostId: hostB } as ProjectionFrame, + ); + state = applyPublicFrame( + state, + { + v: "omp-app/1", + type: "workspace.state", + hostId: HOST, + workspaceId: "workspace-a", + cursor: { epoch: "workspace-epoch", seq: 2 }, + revision: revision("workspace-r2"), + remove: "workspace-a", + } as WorkspaceStateFrame as ProjectionFrame, + ); + + expect(state.workspaces.has(`${String(HOST)}\u0000workspace-a`)).toBe(false); + expect(state.workspaces.has(`${String(hostB)}\u0000workspace-b`)).toBe(true); + }); +}); diff --git a/packages/client/test/desktop-runtime.test.ts b/packages/client/test/desktop-runtime.test.ts index e45f080b..6dce78db 100644 --- a/packages/client/test/desktop-runtime.test.ts +++ b/packages/client/test/desktop-runtime.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vite-plus/test"; -import { hostId, operationId, revision, sessionId, type WelcomeFrame } from "@t4-code/protocol"; +import { CLUSTER_OPERATOR_FEATURE, hostId, operationId, revision, sessionId, type SessionRef, type WelcomeFrame, type WorkspaceInfrastructureProjection, type WorkspaceStateFrame } from "@t4-code/protocol"; import { rendererServerEventFromFrame } from "@t4-code/protocol/desktop-ipc"; import type { BootstrapResult, @@ -40,6 +40,8 @@ interface TestServerFrameEnvelope { } import { createDesktopRuntimeController, type DesktopRuntimeController, type DesktopShellPort } from "../src/desktop-runtime.ts"; import { redactedMessage } from "../src/desktop-runtime-contracts.ts"; +import { ProjectionStore } from "../src/projection.ts"; +import { decodeProjectionCacheValue } from "../src/projection-cache.ts"; import { MAX_RETAINED_SESSION_EVENT_BYTES, retainedJsonBytes, @@ -69,6 +71,60 @@ const remoteTargetRequest = (targetId: string): TargetAddRequest => ({ const welcome = (host: string, capabilities: readonly string[], features: readonly string[], epoch = "epoch-1"): WelcomeFrame => ({ v: "omp-app/1", type: "welcome", selectedProtocol: "omp-app/1", hostId: hostId(host), ompVersion: "omp", ompBuild: "test", appserverVersion: "app", appserverBuild: "test", epoch, grantedCapabilities: [...capabilities], grantedFeatures: [...features], negotiatedLimits: {}, authentication: "local", resumed: false, }); +const workspaceInfrastructure = (workspaceId = "workspace-a"): WorkspaceInfrastructureProjection => ({ + id: workspaceId, + displayName: "Operator workspace", + phase: "Ready", + retentionPolicy: "Retain", + capacity: "20Gi", + storageClass: "rwx", + accessMode: "ReadWriteMany", + revision: revision("workspace-r1"), +}); +const workspaceState = (workspaceId = "workspace-event"): WorkspaceStateFrame => ({ + v: "omp-app/1", + type: "workspace.state", + hostId: hostId("host-a"), + workspaceId, + cursor: { epoch: "workspace-event-epoch", seq: 1 }, + revision: revision("workspace-event-r1"), + upsert: { ...workspaceInfrastructure(workspaceId), revision: revision("workspace-event-r1") }, +}); +const sessionClusterState = Object.freeze({ + workspaceId: "workspace-session-a", + phase: "Running" as const, + gui: Object.freeze({ state: "Ready" as const, previewId: "preview-session-a" }), +}); +const sessionCiState = Object.freeze({ + provider: "woodpecker" as const, + correlation: "exact" as const, + repositoryId: "repository-a", + branch: "main", + ref: "refs/heads/main", + commit: "deadbeef", + pipelineNumber: 42, + status: "running" as const, + currentStage: "test", +}); +const sessionWithInfrastructure = ( + revisionValue: string, + title: string, + phase: string, +): SessionRef => Object.freeze({ + hostId: hostId("host-a"), + project: Object.freeze({ projectId: "project-a" as never, name: "Project A" }), + sessionId: sessionId("session-a"), + revision: revision(revisionValue), + title, + status: "active", + updatedAt: "2026-07-21T00:00:00Z", + liveState: Object.freeze({ + phase, + cluster: sessionClusterState, + ci: sessionCiState, + }), + model: "model-a", +}); class FakeTimerScheduler { readonly timers = new Map void; readonly delayMs: number }>(); private nextHandle = 1; @@ -112,9 +168,15 @@ class FakeShell implements DesktopShellPort { sessionListError: Error | undefined; sessionListResultMissing = false; sessionListGate: Promise | undefined; + workspaceListGate: Promise | undefined; catalogResult: unknown = { revision: "catalog-1", items: [] }; settingsResult: unknown = { revision: "settings-1", settings: {} }; + workspaceListResult: unknown = { + cursor: { epoch: "workspace-epoch", seq: 1 }, + workspaces: [workspaceInfrastructure()], + }; private sessionListGateResolve: (() => void) | undefined; + private workspaceListGateResolve: (() => void) | undefined; deferNextSessionList(): void { this.sessionListGate = new Promise((resolve) => { this.sessionListGateResolve = resolve; }); } @@ -123,6 +185,14 @@ class FakeShell implements DesktopShellPort { this.sessionListGateResolve = undefined; resolve?.(); } + deferNextWorkspaceList(): void { + this.workspaceListGate = new Promise((resolve) => { this.workspaceListGateResolve = resolve; }); + } + resolveWorkspaceList(): void { + const resolve = this.workspaceListGateResolve; + this.workspaceListGateResolve = undefined; + resolve?.(); + } async bootstrap(): Promise { this.bootstrapCalls += 1; if (this.emitWelcomeOnBootstrap !== undefined) this.emitFrame(this.emitWelcomeOnBootstrap); return { platform: "linux", version: "omp-app/1", connected: false }; } async listTargets(): Promise { return { targets: Object.freeze([...this.listedTargets]) }; } async connectTarget(request: TargetRequest): Promise { this.connectCalls += 1; if (this.rejectConnect) throw new Error("appserver unavailable"); this.emitState({ targetId: request.targetId, state: "connected" }); return { targetId: request.targetId, state: "connected" }; } @@ -168,6 +238,12 @@ class FakeShell implements DesktopShellPort { } if (request.intent.command === "catalog.get") return { ...base, result: this.catalogResult }; if (request.intent.command === "settings.read") return { ...base, result: this.settingsResult }; + if (request.intent.command === "workspace.list") { + const gate = this.workspaceListGate; + this.workspaceListGate = undefined; + if (gate !== undefined) await gate; + return { ...base, result: this.workspaceListResult }; + } return base; } async pair(request: PairRequest): Promise { return { targetId: request.targetId, paired: true }; } @@ -209,10 +285,14 @@ class FakeShell implements DesktopShellPort { // eslint-disable-next-line unicorn/no-useless-spread -- preserve listener snapshot when callbacks may unsubscribe during dispatch. for (const listener of [...this.serverEvents]) listener(envelope); } - // eslint-disable-next-line unicorn/no-useless-spread -- preserve listener snapshot when callbacks may unsubscribe during dispatch. - emitState(event: ConnectionStateEvent): void { for (const listener of [...this.states]) listener(event); } - // eslint-disable-next-line unicorn/no-useless-spread -- preserve listener snapshot when callbacks may unsubscribe during dispatch. - emitWake(): void { for (const listener of [...this.wakes]) listener(); } + emitState(event: ConnectionStateEvent): void { + // eslint-disable-next-line unicorn/no-useless-spread -- preserve listener snapshot when callbacks may unsubscribe during dispatch. + for (const listener of [...this.states]) listener(event); + } + emitWake(): void { + // eslint-disable-next-line unicorn/no-useless-spread -- preserve listener snapshot when callbacks may unsubscribe during dispatch. + for (const listener of [...this.wakes]) listener(); + } async confirm(request: ConfirmRequest): Promise { return { targetId: request.targetId, requestId: "confirm-request", confirmationId: request.confirmationId, commandId: request.commandId, accepted: true }; } async terminalInput(request: TerminalInputRequest): Promise { return { targetId: request.targetId, accepted: true }; } async terminalResize(request: TerminalResizeRequest): Promise { return { targetId: request.targetId, accepted: true }; } @@ -222,8 +302,10 @@ class FakeShell implements DesktopShellPort { if (this.stopSpeakingGate !== undefined) await this.stopSpeakingGate; return { accepted: true }; } - // eslint-disable-next-line unicorn/no-useless-spread -- preserve listener snapshot when callbacks may unsubscribe during dispatch. - emitError(event: RuntimeErrorEvent): void { for (const listener of [...this.errors]) listener(event); } + emitError(event: RuntimeErrorEvent): void { + // eslint-disable-next-line unicorn/no-useless-spread -- preserve listener snapshot when callbacks may unsubscribe during dispatch. + for (const listener of [...this.errors]) listener(event); + } } const leaseIntent = (args: Record = {}): CommandRequest["intent"] => ({ @@ -340,6 +422,238 @@ describe("desktop runtime projection", () => { }); expect(runtime.getSnapshot().targetHosts.get("local")).toBe("host-a"); }); + it.each([ + { authority: "the feature is default-off and unnegotiated", enabled: false, capabilities: [], features: [], granted: false }, + { authority: "only the host claim is present", enabled: false, capabilities: ["sessions.read"], features: [CLUSTER_OPERATOR_FEATURE], granted: false }, + { authority: "cluster.operator was not negotiated", enabled: true, capabilities: ["sessions.read"], features: [], granted: false }, + { authority: "sessions.read was not negotiated", enabled: true, capabilities: [], features: [CLUSTER_OPERATOR_FEATURE], granted: false }, + { authority: "the effective cluster projection grant is present", enabled: true, capabilities: ["sessions.read"], features: [CLUSTER_OPERATOR_FEATURE], granted: true }, + ])("normalizes sessions and session.delta infrastructure metadata when $authority", async ({ enabled, capabilities, features, granted }) => { + const shell = new FakeShell(); + const runtime = createDesktopRuntimeController({ shell, clusterOperatorEnabled: enabled }); + await runtime.start(); + shell.emitFrame({ targetId: "local", frame: welcome("host-a", capabilities, features) }); + for (let index = 0; index < 8; index += 1) await Promise.resolve(); + const delivered: RendererServerEventEnvelope[] = []; + runtime.subscribeEvents( + { targetId: "local", kinds: ["sessions", "session.delta"] }, + (event) => delivered.push(event), + ); + + const listed = sessionWithInfrastructure("revision-listed", "Listed session", "idle"); + const sessionsFrame = Object.freeze({ + v: "omp-app/1" as const, + type: "sessions" as const, + hostId: hostId("host-a"), + cursor: Object.freeze({ epoch: "epoch-1", seq: 8 }), + sessions: Object.freeze([listed]), + totalCount: 1, + truncated: false, + }); + shell.emitFrame({ targetId: "local", frame: sessionsFrame as unknown as RendererServerFrame }); + + const listedProjection = runtime.getSnapshot().projection.sessionIndex.get("host-a\u0000session-a"); + expect(listedProjection).toMatchObject({ + hostId: hostId("host-a"), + sessionId: sessionId("session-a"), + project: { projectId: "project-a", name: "Project A" }, + revision: revision("revision-listed"), + title: "Listed session", + status: "active", + liveState: { phase: "idle" }, + model: "model-a", + }); + expect(listedProjection?.liveState).toEqual(granted ? listed.liveState : { phase: "idle" }); + + const changed = sessionWithInfrastructure("revision-delta", "Updated session", "running"); + const deltaFrame = Object.freeze({ + v: "omp-app/1" as const, + type: "session.delta" as const, + hostId: hostId("host-a"), + sessionId: sessionId("session-a"), + cursor: Object.freeze({ epoch: "delta-epoch", seq: 1 }), + revision: revision("revision-delta"), + upsert: changed, + }); + shell.emitFrame({ targetId: "local", frame: deltaFrame as unknown as RendererServerFrame }); + + const changedProjection = runtime.getSnapshot().projection.sessionIndex.get("host-a\u0000session-a"); + expect(changedProjection).toMatchObject({ + hostId: hostId("host-a"), + sessionId: sessionId("session-a"), + revision: revision("revision-delta"), + title: "Updated session", + status: "active", + liveState: { phase: "running" }, + model: "model-a", + }); + expect(changedProjection?.liveState).toEqual(granted ? changed.liveState : { phase: "running" }); + expect(delivered.map((event) => event.event.kind)).toEqual(["sessions", "session.delta"]); + const listedEvent = delivered[0]?.event; + const deltaEvent = delivered[1]?.event; + if (listedEvent?.kind !== "sessions" || deltaEvent?.kind !== "session.delta") throw new Error("expected session inventory events"); + expect(listedEvent.payload.sessions[0]).toMatchObject({ + hostId: hostId("host-a"), + sessionId: sessionId("session-a"), + title: "Listed session", + liveState: { phase: "idle" }, + }); + expect(listedEvent.payload.sessions[0]?.liveState).toEqual(granted ? listed.liveState : { phase: "idle" }); + expect(deltaEvent.payload.upsert).toMatchObject({ + hostId: hostId("host-a"), + sessionId: sessionId("session-a"), + title: "Updated session", + liveState: { phase: "running" }, + }); + expect(deltaEvent.payload.upsert?.liveState).toEqual(granted ? changed.liveState : { phase: "running" }); + expect(listed.liveState).toEqual({ phase: "idle", cluster: sessionClusterState, ci: sessionCiState }); + expect(changed.liveState).toEqual({ phase: "running", cluster: sessionClusterState, ci: sessionCiState }); + }); + it.each([ + { enabled: false, capabilities: ["sessions.read"], features: [CLUSTER_OPERATOR_FEATURE] }, + { enabled: true, capabilities: ["sessions.read"], features: [] }, + { enabled: true, capabilities: [], features: [CLUSTER_OPERATOR_FEATURE] }, + ])("clears retained workspaces and rejects workspace.list projection without every authority gate", async ({ enabled, capabilities, features }) => { + const projection = new ProjectionStore(); + projection.replaceWorkspaceInventory( + "host-a", + [workspaceInfrastructure()], + { epoch: "retained-workspace", seq: 4 }, + ); + const shell = new FakeShell(); + const runtime = createDesktopRuntimeController({ + shell, + projection, + clusterOperatorEnabled: enabled, + }); + + expect(runtime.getSnapshot().projection.workspaces.size).toBe(0); + expect(runtime.getSnapshot().projection.workspaceCursors.size).toBe(0); + await runtime.start(); + shell.emitFrame({ targetId: "local", frame: welcome("host-a", capabilities, features) }); + for (let index = 0; index < 8; index += 1) await Promise.resolve(); + await expect(runtime.command("local", { + hostId: hostId("host-a"), + command: "workspace.list", + args: {}, + })).rejects.toMatchObject({ code: "stale" }); + + expect(runtime.getSnapshot().projection.workspaces.size).toBe(0); + expect(runtime.getSnapshot().projection.workspaceCursors.size).toBe(0); + }); + it.each([ + { authority: "the feature flag is off", enabled: false, capabilities: ["sessions.read"], features: [CLUSTER_OPERATOR_FEATURE], granted: false }, + { authority: "cluster.operator was not negotiated", enabled: true, capabilities: ["sessions.read"], features: [], granted: false }, + { authority: "the effective cluster projection grant is present", enabled: true, capabilities: ["sessions.read"], features: [CLUSTER_OPERATOR_FEATURE], granted: true }, + ])("gates workspace.state projection and renderer delivery when $authority", async ({ enabled, capabilities, features, granted }) => { + const shell = new FakeShell(); + const runtime = createDesktopRuntimeController({ shell, clusterOperatorEnabled: enabled }); + await runtime.start(); + shell.emitFrame({ targetId: "local", frame: welcome("host-a", capabilities, features) }); + const deliveredKinds: string[] = []; + runtime.subscribeEvents((event) => deliveredKinds.push(event.event.kind)); + + shell.emitFrame({ targetId: "local", frame: workspaceState() }); + shell.emitFrame({ targetId: "local", frame: { v: "omp-app/1", type: "catalog", hostId: hostId("host-a"), revision: revision("catalog-event-r1"), items: [] } }); + + expect(runtime.getSnapshot().projection.workspaces.has("host-a\u0000workspace-event")).toBe(granted); + expect(deliveredKinds).toEqual(granted ? ["workspace.state", "catalog"] : ["catalog"]); + }); + it("rejects an in-flight workspace.list result after same-epoch authority withdrawal", async () => { + const shell = new FakeShell(); + const runtime = createDesktopRuntimeController({ shell, clusterOperatorEnabled: true }); + await runtime.start(); + shell.emitFrame({ + targetId: "local", + frame: welcome("host-a", ["sessions.read"], [CLUSTER_OPERATOR_FEATURE]), + }); + for (let index = 0; index < 12; index += 1) await Promise.resolve(); + + const responses: RendererServerEventEnvelope[] = []; + runtime.subscribeEvents((event) => { if (event.event.kind === "response") responses.push(event); }); + const authorized = await runtime.command("local", { + hostId: hostId("host-a"), + command: "workspace.list", + args: {}, + }); + expect(authorized.result).toEqual(shell.workspaceListResult); + expect(responses).toHaveLength(1); + + shell.deferNextWorkspaceList(); + const pending = runtime.command("local", { + hostId: hostId("host-a"), + command: "workspace.list", + args: {}, + }); + await Promise.resolve(); + shell.emitFrame({ + targetId: "local", + frame: welcome("host-a", ["sessions.read"], [], "epoch-1"), + }); + shell.resolveWorkspaceList(); + + await expect(pending).rejects.toMatchObject({ code: "stale" }); + expect(responses.filter((event) => event.event.kind === "response" && event.event.payload.command === "workspace.list")).toHaveLength(1); + expect(runtime.getSnapshot().projection.workspaces.size).toBe(0); + const ordinary = await runtime.command("local", { + hostId: hostId("host-a"), + command: "session.prompt", + args: { prompt: "hello" }, + }); + expect(ordinary.accepted).toBe(true); + }); + it("purges retained workspace cache before any current host grant", async () => { + const saves: string[] = []; + const projection = new ProjectionStore({ + cacheStore: { + load: () => undefined, + save: (serialized) => { saves.push(serialized); }, + }, + }); + await projection.ready(); + projection.replaceWorkspaceInventory( + "host-a", + [workspaceInfrastructure()], + { epoch: "retained-workspace", seq: 4 }, + ); + await projection.flush(); + expect(decodeProjectionCacheValue(saves.at(-1))?.workspaces.size).toBe(1); + + const runtime = createDesktopRuntimeController({ + shell: new FakeShell(), + projection, + clusterOperatorEnabled: true, + }); + await projection.flush(); + + const clearedCache = decodeProjectionCacheValue(saves.at(-1)); + expect(runtime.getSnapshot().projection.workspaces.size).toBe(0); + expect(clearedCache?.workspaces.size).toBe(0); + expect(clearedCache?.workspaceCursors.size).toBe(0); + }); + it("clears retained workspaces when a host withdraws the cluster operator grant", async () => { + const shell = new FakeShell(); + const runtime = createDesktopRuntimeController({ shell, clusterOperatorEnabled: true }); + await runtime.start(); + shell.emitFrame({ + targetId: "local", + frame: welcome("host-a", ["sessions.read"], [CLUSTER_OPERATOR_FEATURE]), + }); + for (let index = 0; index < 12; index += 1) await Promise.resolve(); + + expect(runtime.getSnapshot().projection.workspaces.get("host-a\u0000workspace-a")?.phase).toBe("Ready"); + expect(runtime.getSnapshot().projection.workspaceCursors.get("host-a")).toEqual({ + epoch: "workspace-epoch", + seq: 1, + }); + + shell.emitFrame({ + targetId: "local", + frame: welcome("host-a", ["sessions.read"], [], "epoch-2"), + }); + expect(runtime.getSnapshot().projection.workspaces.size).toBe(0); + expect(runtime.getSnapshot().projection.workspaceCursors.size).toBe(0); + }); it("preserves operation capabilities from catalog responses and live catalog frames", async () => { const shell = new FakeShell(); shell.catalogResult = { @@ -617,7 +931,7 @@ describe("desktop runtime projection", () => { shell.emitFrame({ targetId: "local", frame: welcome("host-a", ["sessions.read"], []) }); for (let index = 0; index < 8; index += 1) await Promise.resolve(); shell.sessionListResult = { - cursor: { epoch: "epoch-1", seq: 2 }, + cursor: { epoch: "epoch-1", seq: 8 }, sessions: [], totalCount: 4, truncated: true, @@ -1209,6 +1523,18 @@ describe("desktop runtime projection", () => { result: sessionInventory("host-remote", "epoch-1", "old-frame"), } as unknown as RendererServerFrame), }); + shell.emitFrame({ + targetId: "remote", + frame: ({ + v: "omp-app/1", + type: "response", + requestId: "old-workspace-request", + hostId: hostId("host-remote"), + ok: true, + command: "workspace.list", + result: shell.workspaceListResult, + } as unknown as RendererServerFrame), + }); expect(responses).toHaveLength(0); await runtime.command("remote", { hostId: hostId("host-remote"), command: "session.list", args: {} }); expect(responses).toHaveLength(1); diff --git a/packages/client/test/projection.test.ts b/packages/client/test/projection.test.ts index 4e396668..5a57557c 100644 --- a/packages/client/test/projection.test.ts +++ b/packages/client/test/projection.test.ts @@ -234,6 +234,36 @@ describe("client projections", () => { }); }); + it("rejects a delayed lower same-epoch session inventory without regressing the index", () => { + const currentRef = ref(String(HOST), "ordered", { title: "Current" }); + const staleRef = ref(String(HOST), "ordered", { title: "Stale" }); + const current = applyPublicFrame(createProjectionSnapshot(), { + v: V, + type: "sessions", + hostId: HOST, + cursor: { epoch: "ordered-inventory", seq: 5 }, + sessions: [currentRef], + totalCount: 1, + truncated: false, + } as ProjectionFrame); + const staleFrame = { + v: V, + type: "sessions", + hostId: HOST, + cursor: { epoch: "ordered-inventory", seq: 4 }, + sessions: [staleRef], + totalCount: 1, + truncated: false, + } as ProjectionFrame; + + expect(applyPublicFrame(current, staleFrame)).toBe(current); + expect(current.sessionIndex.get(sessionKey("ordered"))?.title).toBe("Current"); + expect(current.sessionInventoryCursors.get(String(HOST))?.seq).toBe(5); + + const restored = decodeProjectionCacheValue(encodeProjectionCache(current))!; + expect(applyPublicFrame(restored, staleFrame)).toBe(restored); + }); + it("invalidates inventory completeness on welcome until the next sessions frame", () => { const listed = ref(String(HOST), "cached"); let state = applyPublicFrame(createProjectionSnapshot(), { @@ -251,6 +281,7 @@ describe("client projections", () => { expect(state.sessionIndex.has(sessionKey("cached"))).toBe(true); expect(state.sessionIndexMetadata.has(String(HOST))).toBe(false); + expect(state.sessionInventoryCursors.has(String(HOST))).toBe(false); state = applyPublicFrame(state, { v: V, type: "sessions", @@ -548,6 +579,48 @@ describe("client projections", () => { }); expect(state.sessions.get(sessionKey("session-a"))?.events).toEqual([]); }); + it("rejects a stale same-epoch snapshot without replacing newer projection state", () => { + const freshEntry = childEntry("fresh-entry", "fresh"); + const staleEntry = childEntry("stale-entry", "stale"); + const state = applyPublicFrame(createProjectionSnapshot(), { + ...frame("snapshot"), + cursor: { epoch: "e1", seq: 10 }, + revision: revision("fresh-revision"), + entries: [freshEntry], + }); + const stale = applyPublicFrame(state, { + ...frame("snapshot"), + cursor: { epoch: "e1", seq: 9 }, + revision: revision("stale-revision"), + entries: [staleEntry], + }); + + expect(stale).toBe(state); + expect(stale.sessions.get(sessionKey("session-a"))?.cursor).toEqual({ epoch: "e1", seq: 10 }); + expect(stale.sessions.get(sessionKey("session-a"))?.entries).toEqual([freshEntry]); + }); + + it("accepts an equal-cursor live snapshot to refresh a cached baseline", () => { + const cached = decodeProjectionCacheValue( + encodeProjectionCache( + applyPublicFrame(createProjectionSnapshot(), { + ...frame("snapshot"), + cursor: { epoch: "e1", seq: 10 }, + entries: [childEntry("cached-entry", "cached")], + }), + ), + )!; + const refreshed = applyPublicFrame(cached, { + ...frame("snapshot"), + cursor: { epoch: "e1", seq: 10 }, + revision: revision("live-revision"), + entries: [childEntry("live-entry", "live")], + }); + + expect(refreshed.sessions.get(sessionKey("session-a"))?.freshness).toBe("fresh"); + expect(refreshed.sessions.get(sessionKey("session-a"))?.revision).toBe("live-revision"); + expect(refreshed.sessions.get(sessionKey("session-a"))?.entries[0]?.id).toBe("live-entry"); + }); it("uses the emitting owner cursor for remove-other deltas without touching transcript state", () => { let state = applyPublicFrame(createProjectionSnapshot(), frame("snapshot")); state = applyPublicFrame(state, { diff --git a/packages/cluster-operator/api/v1alpha1/groupversion_info.go b/packages/cluster-operator/api/v1alpha1/groupversion_info.go new file mode 100644 index 00000000..7a84a3fe --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/groupversion_info.go @@ -0,0 +1,14 @@ +// Package v1alpha1 defines the namespaced Kubernetes infrastructure API for T4 clusters. +// It intentionally contains no OMP session, agent, prompt, transcript, or lifecycle truth. +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + GroupVersion = schema.GroupVersion{Group: "cluster.t4.dev", Version: "v1alpha1"} + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/packages/cluster-operator/api/v1alpha1/types.go b/packages/cluster-operator/api/v1alpha1/types.go new file mode 100644 index 00000000..2132e115 --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/types.go @@ -0,0 +1,182 @@ +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + WorkspaceFinalizer = "cluster.t4.dev/workspace-protection" + SessionFinalizer = "cluster.t4.dev/session-cleanup" + RWXStorageClassAnnotation = "cluster.t4.dev/access-modes" + RetainedPVCAnnotation = "cluster.t4.dev/retained" + WorkspaceUIDAnnotation = "cluster.t4.dev/workspace-uid" + SessionPodSpecHashAnnotation = "cluster.t4.dev/pod-spec-hash" +) + +// +kubebuilder:validation:Enum=Retain;Delete +type RetentionPolicy string + +const ( + RetentionPolicyRetain RetentionPolicy = "Retain" + RetentionPolicyDelete RetentionPolicy = "Delete" +) + +func ValidRetentionPolicy(value RetentionPolicy) bool { + return value == RetentionPolicyRetain || value == RetentionPolicyDelete +} + +type InfrastructurePhase string + +const ( + InfrastructurePending InfrastructurePhase = "Pending" + InfrastructureReady InfrastructurePhase = "Ready" + InfrastructureRunning InfrastructurePhase = "Running" + InfrastructureFailed InfrastructurePhase = "Failed" + InfrastructureTerminating InfrastructurePhase = "Terminating" + InfrastructureUnknown InfrastructurePhase = "Unknown" +) + +func ValidInfrastructurePhase(value InfrastructurePhase) bool { + switch value { + case InfrastructurePending, InfrastructureReady, InfrastructureRunning, InfrastructureFailed, InfrastructureTerminating, InfrastructureUnknown: + return true + default: + return false + } +} + +type CIProviderReferences struct { + SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"` + ConfigMapRef corev1.LocalObjectReference `json:"configMapRef"` + ServiceAccountAudience string `json:"serviceAccountAudience,omitempty"` +} +type T4ClusterHostSpec struct { + StorageClassName string `json:"storageClassName"` + RuntimeProfiles []string `json:"runtimeProfiles"` + CIProvider *CIProviderReferences `json:"ciProvider,omitempty"` + AllowedOrigins []string `json:"allowedOrigins,omitempty"` +} + +type T4ClusterHostStatus struct { + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced,shortName=t4host +// +kubebuilder:printcolumn:name="Storage",type=string,JSONPath=`.spec.storageClassName` +// +kubebuilder:printcolumn:name="Available",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].status` +type T4ClusterHost struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec T4ClusterHostSpec `json:"spec,omitempty"` + Status T4ClusterHostStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +type T4ClusterHostList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []T4ClusterHost `json:"items"` +} + +type RepositoryMetadata struct { + RepositoryID string `json:"repositoryId"` + Ref string `json:"ref,omitempty"` + Commit string `json:"commit,omitempty"` +} + +type T4WorkspaceSpec struct { + HostRef string `json:"hostRef"` + DisplayName string `json:"displayName"` + Owner string `json:"owner"` + Repository *RepositoryMetadata `json:"repository,omitempty"` + Size resource.Quantity `json:"size"` + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="retentionPolicy is immutable" + RetentionPolicy RetentionPolicy `json:"retentionPolicy"` +} + +type T4WorkspaceStatus struct { + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + PVCName string `json:"pvcName,omitempty"` + PVCPhase corev1.PersistentVolumeClaimPhase `json:"pvcPhase,omitempty"` + Capacity resource.Quantity `json:"capacity,omitempty"` + Phase InfrastructurePhase `json:"phase,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced,shortName=t4ws +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="PVC",type=string,JSONPath=`.status.pvcName` +type T4Workspace struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec T4WorkspaceSpec `json:"spec,omitempty"` + Status T4WorkspaceStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +type T4WorkspaceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []T4Workspace `json:"items"` +} + +type SessionCIMetadata struct { + RepositoryID string `json:"repositoryId"` + Ref string `json:"ref"` + Commit string `json:"commit"` +} + +type T4SessionSpec struct { + HostRef string `json:"hostRef"` + WorkspaceRef string `json:"workspaceRef"` + Title string `json:"title"` + RuntimeProfile string `json:"runtimeProfile"` + InitialPromptSecretRef *corev1.LocalObjectReference `json:"initialPromptSecretRef,omitempty"` + GUIEnabled bool `json:"guiEnabled,omitempty"` + CI *SessionCIMetadata `json:"ci,omitempty"` +} + +type T4SessionStatus struct { + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + PodName string `json:"podName,omitempty"` + ServiceName string `json:"serviceName,omitempty"` + Phase InfrastructurePhase `json:"phase,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced,shortName=t4sess +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Pod",type=string,JSONPath=`.status.podName` +type T4Session struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec T4SessionSpec `json:"spec,omitempty"` + Status T4SessionStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +type T4SessionList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []T4Session `json:"items"` +} + +func init() { + SchemeBuilder.Register( + &T4ClusterHost{}, &T4ClusterHostList{}, + &T4Workspace{}, &T4WorkspaceList{}, + &T4Session{}, &T4SessionList{}, + ) +} diff --git a/packages/cluster-operator/api/v1alpha1/types_test.go b/packages/cluster-operator/api/v1alpha1/types_test.go new file mode 100644 index 00000000..fde778fb --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/types_test.go @@ -0,0 +1,178 @@ +package v1alpha1_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" +) + +func TestKindsAreNamespacedAndRegistered(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + for _, kind := range []string{"T4ClusterHost", "T4Workspace", "T4Session"} { + gvk := clusterv1alpha1.GroupVersion.WithKind(kind) + if _, err := scheme.New(gvk); err != nil { + t.Fatalf("%s is not registered: %v", gvk, err) + } + } +} + +func TestStatusIsInfrastructureOnlyAndBounded(t *testing.T) { + statuses := []struct { + name string + generation int64 + conditions []metav1.Condition + }{ + {"host", clusterv1alpha1.T4ClusterHostStatus{}.ObservedGeneration, clusterv1alpha1.T4ClusterHostStatus{}.Conditions}, + {"workspace", clusterv1alpha1.T4WorkspaceStatus{}.ObservedGeneration, clusterv1alpha1.T4WorkspaceStatus{}.Conditions}, + {"session", clusterv1alpha1.T4SessionStatus{}.ObservedGeneration, clusterv1alpha1.T4SessionStatus{}.Conditions}, + } + for _, status := range statuses { + if status.generation != 0 || status.conditions != nil { + t.Fatalf("zero %s status must be empty", status.name) + } + } + + // Compile-time API guards: infrastructure references are explicit; no OMP ids, + // prompts, transcript, agent tree, or lifecycle ownership is represented here. + _ = clusterv1alpha1.T4WorkspaceStatus{PVCName: "pvc", Phase: clusterv1alpha1.InfrastructurePending} + _ = clusterv1alpha1.T4SessionStatus{PodName: "pod", ServiceName: "service", Phase: clusterv1alpha1.InfrastructurePending} +} + +func TestEnumsRejectUnboundedValuesAtTheGoBoundary(t *testing.T) { + if !clusterv1alpha1.ValidRetentionPolicy(clusterv1alpha1.RetentionPolicyRetain) || + !clusterv1alpha1.ValidRetentionPolicy(clusterv1alpha1.RetentionPolicyDelete) || + clusterv1alpha1.ValidRetentionPolicy("Archive") { + t.Fatal("retention policy allowlist is not exact") + } + if !clusterv1alpha1.ValidInfrastructurePhase(clusterv1alpha1.InfrastructureRunning) || + clusterv1alpha1.ValidInfrastructurePhase("OMPRunning") { + t.Fatal("infrastructure phase allowlist accepts non-infrastructure state") + } +} + +func TestCRDContractConstants(t *testing.T) { + if got, want := clusterv1alpha1.GroupVersion.String(), "cluster.t4.dev/v1alpha1"; got != want { + t.Fatalf("group version = %q, want %q", got, want) + } + if got, want := clusterv1alpha1.WorkspaceFinalizer, "cluster.t4.dev/workspace-protection"; got != want { + t.Fatalf("workspace finalizer = %q", got) + } + if got, want := clusterv1alpha1.SessionFinalizer, "cluster.t4.dev/session-cleanup"; got != want { + t.Fatalf("session finalizer = %q", got) + } + if got, want := clusterv1alpha1.RWXStorageClassAnnotation, "cluster.t4.dev/access-modes"; got != want { + t.Fatalf("RWX storage annotation = %q", got) + } +} + +func TestCRDSchemasAreStructuralBoundedAndValidated(t *testing.T) { + paths := []string{ + "t4clusterhosts.cluster.t4.dev.yaml", + "t4workspaces.cluster.t4.dev.yaml", + "t4sessions.cluster.t4.dev.yaml", + } + for _, name := range paths { + raw, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "deploy", "charts", "t4-cluster", "crds", name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(raw, &crd); err != nil { + t.Fatalf("decode %s: %v", name, err) + } + if crd.Spec.Scope != apiextensionsv1.NamespaceScoped { + t.Fatalf("%s must be namespaced", crd.Name) + } + if len(crd.Spec.Versions) != 1 || !crd.Spec.Versions[0].Served || !crd.Spec.Versions[0].Storage { + t.Fatalf("%s must have one served storage version", crd.Name) + } + version := crd.Spec.Versions[0] + if version.Subresources == nil || version.Subresources.Status == nil { + t.Fatalf("%s lacks the status subresource", crd.Name) + } + if version.Schema == nil || version.Schema.OpenAPIV3Schema == nil { + t.Fatalf("%s lacks an OpenAPI schema", crd.Name) + } + var internal apiextensions.JSONSchemaProps + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(version.Schema.OpenAPIV3Schema, &internal, nil); err != nil { + t.Fatalf("convert %s schema: %v", crd.Name, err) + } + if _, err := structuralschema.NewStructural(&internal); err != nil { + t.Fatalf("%s is not structural: %v", crd.Name, err) + } + root := version.Schema.OpenAPIV3Schema + if root.XPreserveUnknownFields != nil && *root.XPreserveUnknownFields { + t.Fatalf("%s preserves unknown fields", crd.Name) + } + status := root.Properties["status"] + conditions := status.Properties["conditions"] + if conditions.MaxItems == nil || *conditions.MaxItems > 8 { + t.Fatalf("%s status conditions are not bounded", crd.Name) + } + if _, ok := status.Properties["observedGeneration"]; !ok { + t.Fatalf("%s status lacks observedGeneration", crd.Name) + } + assertBoundedSchema(t, crd.Name+".spec", root.Properties["spec"]) + if crd.Name != "t4clusterhosts.cluster.t4.dev" { + hostRef := root.Properties["spec"].Properties["hostRef"] + immutable := false + for _, validation := range hostRef.XValidations { + if validation.Rule == "self == oldSelf" { + immutable = true + } + } + if !immutable { + t.Fatalf("%s hostRef is mutable", crd.Name) + } + } + } +} + +func TestCRDsHaveCrossFieldCELAndForbidClientRuntimeAuthority(t *testing.T) { + for _, name := range []string{"t4clusterhosts.cluster.t4.dev.yaml", "t4workspaces.cluster.t4.dev.yaml", "t4sessions.cluster.t4.dev.yaml"} { + raw, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "deploy", "charts", "t4-cluster", "crds", name)) + if err != nil { + t.Fatal(err) + } + text := string(raw) + if !strings.Contains(text, "x-kubernetes-validations:") { + t.Fatalf("%s has no CEL validation", name) + } + for _, forbidden := range []string{"image:", "prompt:", "shell:", "token:", "ompSession", "agentId", "transcript"} { + if strings.Contains(text, forbidden) { + t.Fatalf("%s exposes forbidden authority field %q", name, forbidden) + } + } + } +} + +func assertBoundedSchema(t *testing.T, path string, schema apiextensionsv1.JSONSchemaProps) { + t.Helper() + if schema.Type == "string" && schema.MaxLength == nil && schema.Enum == nil { + t.Fatalf("%s is an unbounded string", path) + } + if schema.Type == "array" { + if schema.MaxItems == nil { + t.Fatalf("%s is an unbounded array", path) + } + if schema.Items != nil && schema.Items.Schema != nil { + assertBoundedSchema(t, path+"[]", *schema.Items.Schema) + } + } + for key, child := range schema.Properties { + assertBoundedSchema(t, path+"."+key, child) + } +} diff --git a/packages/cluster-operator/api/v1alpha1/zz_generated.deepcopy.go b/packages/cluster-operator/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000..6befe822 --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,186 @@ +// Code generated-style deep-copy implementations kept in source so the API package +// remains buildable without a generation step. DO NOT EDIT mechanically. +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func (in *T4ClusterHost) DeepCopyInto(out *T4ClusterHost) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +func (in *T4ClusterHost) DeepCopy() *T4ClusterHost { + if in == nil { + return nil + } + out := new(T4ClusterHost) + in.DeepCopyInto(out) + return out +} + +func (in *T4ClusterHost) DeepCopyObject() runtime.Object { return in.DeepCopy() } + +func (in *T4ClusterHostList) DeepCopyInto(out *T4ClusterHostList) { + *out = *in + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + out.Items = make([]T4ClusterHost, len(in.Items)) + for i := range in.Items { + in.Items[i].DeepCopyInto(&out.Items[i]) + } + } +} + +func (in *T4ClusterHostList) DeepCopy() *T4ClusterHostList { + if in == nil { + return nil + } + out := new(T4ClusterHostList) + in.DeepCopyInto(out) + return out +} + +func (in *T4ClusterHostList) DeepCopyObject() runtime.Object { return in.DeepCopy() } + +func (in *T4ClusterHostSpec) DeepCopyInto(out *T4ClusterHostSpec) { + *out = *in + if in.RuntimeProfiles != nil { + out.RuntimeProfiles = append([]string(nil), in.RuntimeProfiles...) + } + if in.AllowedOrigins != nil { + out.AllowedOrigins = append([]string(nil), in.AllowedOrigins...) + } + if in.CIProvider != nil { + out.CIProvider = &CIProviderReferences{ConfigMapRef: in.CIProvider.ConfigMapRef, ServiceAccountAudience: in.CIProvider.ServiceAccountAudience} + if in.CIProvider.SecretRef != nil { + secret := *in.CIProvider.SecretRef + out.CIProvider.SecretRef = &secret + } + } +} + +func (in *T4ClusterHostStatus) DeepCopyInto(out *T4ClusterHostStatus) { + *out = *in + if in.Conditions != nil { + out.Conditions = append([]metav1.Condition(nil), in.Conditions...) + } +} + +func (in *T4Workspace) DeepCopyInto(out *T4Workspace) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +func (in *T4Workspace) DeepCopy() *T4Workspace { + if in == nil { + return nil + } + out := new(T4Workspace) + in.DeepCopyInto(out) + return out +} + +func (in *T4Workspace) DeepCopyObject() runtime.Object { return in.DeepCopy() } + +func (in *T4WorkspaceList) DeepCopyInto(out *T4WorkspaceList) { + *out = *in + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + out.Items = make([]T4Workspace, len(in.Items)) + for i := range in.Items { + in.Items[i].DeepCopyInto(&out.Items[i]) + } + } +} + +func (in *T4WorkspaceList) DeepCopy() *T4WorkspaceList { + if in == nil { + return nil + } + out := new(T4WorkspaceList) + in.DeepCopyInto(out) + return out +} + +func (in *T4WorkspaceList) DeepCopyObject() runtime.Object { return in.DeepCopy() } + +func (in *T4WorkspaceSpec) DeepCopyInto(out *T4WorkspaceSpec) { + *out = *in + if in.Repository != nil { + out.Repository = &RepositoryMetadata{RepositoryID: in.Repository.RepositoryID, Ref: in.Repository.Ref, Commit: in.Repository.Commit} + } + out.Size = in.Size.DeepCopy() +} + +func (in *T4WorkspaceStatus) DeepCopyInto(out *T4WorkspaceStatus) { + *out = *in + out.Capacity = in.Capacity.DeepCopy() + if in.Conditions != nil { + out.Conditions = append([]metav1.Condition(nil), in.Conditions...) + } +} + +func (in *T4Session) DeepCopyInto(out *T4Session) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +func (in *T4Session) DeepCopy() *T4Session { + if in == nil { + return nil + } + out := new(T4Session) + in.DeepCopyInto(out) + return out +} + +func (in *T4Session) DeepCopyObject() runtime.Object { return in.DeepCopy() } + +func (in *T4SessionList) DeepCopyInto(out *T4SessionList) { + *out = *in + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + out.Items = make([]T4Session, len(in.Items)) + for i := range in.Items { + in.Items[i].DeepCopyInto(&out.Items[i]) + } + } +} + +func (in *T4SessionList) DeepCopy() *T4SessionList { + if in == nil { + return nil + } + out := new(T4SessionList) + in.DeepCopyInto(out) + return out +} + +func (in *T4SessionList) DeepCopyObject() runtime.Object { return in.DeepCopy() } + +func (in *T4SessionSpec) DeepCopyInto(out *T4SessionSpec) { + *out = *in + if in.InitialPromptSecretRef != nil { + copy := *in.InitialPromptSecretRef + out.InitialPromptSecretRef = © + } + if in.CI != nil { + out.CI = &SessionCIMetadata{RepositoryID: in.CI.RepositoryID, Ref: in.CI.Ref, Commit: in.CI.Commit} + } +} + +func (in *T4SessionStatus) DeepCopyInto(out *T4SessionStatus) { + *out = *in + if in.Conditions != nil { + out.Conditions = append([]metav1.Condition(nil), in.Conditions...) + } +} diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go new file mode 100644 index 00000000..9a9aaf7a --- /dev/null +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -0,0 +1,803 @@ +package charttests + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/yaml" +) + +const fakeDigest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func TestChartIsDefaultOff(t *testing.T) { + output := helmTemplate(t) + if strings.TrimSpace(output) != "" { + t.Fatalf("default values rendered workloads/resources:\n%s", output) + } +} + +func TestEnabledChartRendersHARestrictedWorkloads(t *testing.T) { + output := helmTemplate(t, enabledValues()...) + assertCount(t, output, "kind: Deployment", 2) + assertContains(t, output, + "replicas: 2", + "replicas: 3", + "maxUnavailable: 0", + "kind: PodDisruptionBudget", + "minAvailable: 2", + "kubernetes.io/hostname", + "k3s-worker-02", + "topologySpreadConstraints:", + "podAntiAffinity:", + "readOnlyRootFilesystem: true", + "runAsNonRoot: true", + "allowPrivilegeEscalation: false", + "type: RuntimeDefault", + "drop:", + "- ALL", + "automountServiceAccountToken: false", + "startupProbe:", + "readinessProbe:", + "livenessProbe:", + "preStop:", + "path: /drainz", + "kind: NetworkPolicy", + "policyTypes:", + "kind: Role", + "kind: ClusterRole", + "coordination.k8s.io", + "resources:", + ) + server := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-server\"") + assertContains(t, server, + "automountServiceAccountToken: false", + "name: T4_CLUSTER_TRUSTED_PROXY_CIDRS", + "value: \"192.0.2.0/24\"", + "name: kubernetes-api-access", + "audience: \"https://kubernetes.default.svc\"", + "expirationSeconds: 3600", + ) + if strings.Contains(output, "privileged: true") || strings.Contains(output, "hostNetwork: true") || strings.Contains(output, "hostPID: true") { + t.Fatal("enabled chart contains a privileged shortcut") + } + if strings.Contains(output, "kind: PersistentVolumeClaim") || strings.Contains(output, "nfs:") || strings.Contains(output, "hostPath:") { + t.Fatal("portable chart rendered storage backend or workload PVC") + } +} + +func TestLongReleaseNamesRenderDNSLabelResourceNames(t *testing.T) { + releaseName := strings.Repeat("r", 53) + output := helmTemplateRelease(t, releaseName, append(enabledValues(), + "--set", "ingress.enabled=true", + "--set-string", "ingress.className=tailscale", + "--set-string", "ingress.host=operator.example.ts.net", + "--set", "observability.serviceMonitor.enabled=true", + "--set", "observability.prometheusRule.enabled=true", + )...) + dnsLabel := regexp.MustCompile(`^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$`) + requiredSuffixes := []string{ + "controller", + "server", + "metrics", + "controller-metrics", + "session", + "session-token-reviewer", + "storage-reader", + } + foundSuffixes := make(map[string]bool, len(requiredSuffixes)) + for _, document := range strings.Split(output, "\n---") { + var object struct { + Kind string `json:"kind"` + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + } + if err := yaml.Unmarshal([]byte(document), &object); err != nil { + t.Fatalf("decode rendered object: %v\n%s", err, document) + } + if object.Kind == "" { + continue + } + if len(object.Metadata.Name) > 63 || !dnsLabel.MatchString(object.Metadata.Name) { + t.Fatalf("rendered %s metadata.name %q is not a DNS label of at most 63 characters", object.Kind, object.Metadata.Name) + } + for _, suffix := range requiredSuffixes { + if strings.HasSuffix(object.Metadata.Name, "-"+suffix) { + foundSuffixes[suffix] = true + } + } + } + for _, suffix := range requiredSuffixes { + if !foundSuffixes[suffix] { + t.Fatalf("long release render lacks a metadata.name preserving suffix %q", suffix) + } + } +} + +func TestEachDeploymentUsesZeroUnavailableAndConfiguredAPIAudience(t *testing.T) { + output := helmTemplate(t, append(enabledValues(), "--set-string", "kubernetes.apiAudience=kubernetes.custom.example")...) + controller := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-controller\"") + server := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-server\"") + for name, deployment := range map[string]string{"controller": controller, "server": server} { + assertCount(t, deployment, "maxUnavailable: 0", 1) + assertContains(t, deployment, + "automountServiceAccountToken: false", + "name: T4_KUBERNETES_API_AUDIENCE", + "value: \"kubernetes.custom.example\"", + "audience: \"kubernetes.custom.example\"", + ) + if strings.Contains(deployment, "maxUnavailable: 1") { + t.Fatalf("%s Deployment permits an unavailable replica", name) + } + } + assertContains(t, server, "audience: \"t4-cluster-internal\"") +} + +func TestValuesSchemaRejectsUnsafeNamesProfilesCIDRsAndHalfSelectors(t *testing.T) { + for name, values := range map[string][]string{ + "cluster host name": {"--set-string", "clusterHost.name=Bad_Name"}, + "storage class name": {"--set-string", "storage.adminRWXStorageClass=Bad_Name"}, + "runtime profile": {"--set-string", "clusterHost.runtimeProfiles[0]=-bad"}, + "Woodpecker Secret name": {"--set-string", "woodpecker.existingSecret=Bad_Name", "--set-string", "woodpecker.configMap=woodpecker-config"}, + "Woodpecker ConfigMap name": {"--set-string", "woodpecker.existingSecret=woodpecker-token", "--set-string", "woodpecker.configMap=Bad_Name"}, + "Woodpecker key": {"--set-string", "woodpecker.existingSecret=woodpecker-token", "--set-string", "woodpecker.configMap=woodpecker-config", "--set-string", "woodpecker.tokenKey=bad/key"}, + "Woodpecker audience": {"--set-string", "woodpecker.serviceAccountAudience=/bad", "--set-string", "woodpecker.configMap=woodpecker-config"}, + "IPv4 default route": {"--set-string", "server.trustedProxyCIDRs[0]=0.0.0.0/0"}, + "IPv6 default route": {"--set-string", "server.trustedProxyCIDRs[0]=::/0"}, + "gateway half selector": {"--set-string", "networkPolicy.gatewayIngress.namespaceSelector.matchLabels.scope=gateway"}, + "observability half selector": {"--set-string", "networkPolicy.observability.podSelector.matchLabels.scope=metrics"}, + "OMP ConfigMap name": {"--set-string", "session.omp.configMap=Bad_Name"}, + "OMP models key": {"--set-string", "session.omp.modelsKey=bad/key"}, + "OMP credential Secret name": {"--set-string", "session.omp.credentialSecret=Bad_Name"}, + "OMP credential environment name": {"--set-string", "session.omp.credentialKey=bad-key"}, + "OMP runtime-owned credential environment": {"--set-string", "session.omp.credentialKey=OMP_PROFILE"}, + "OMP PI runtime-owned credential environment": {"--set-string", "session.omp.credentialKey=PI_PROFILE"}, + "identical OMP projection keys": {"--set-string", "session.omp.settingsKey=provider-models"}, + "model route port zero": {"--set", "networkPolicy.modelRoutePorts[0]=0"}, + "model route port above TCP range": {"--set", "networkPolicy.modelRoutePorts[0]=65536"}, + "duplicate model route port": {"--set", "networkPolicy.modelRoutePorts[0]=19481", "--set", "networkPolicy.modelRoutePorts[1]=19481"}, + "noninteger model route port": {"--set-string", "networkPolicy.modelRoutePorts[0]=https"}, + "model route half selector": {"--set-string", "networkPolicy.modelRoute.namespaceSelector.matchLabels.scope=linkedin-bot"}, + "CI provider port zero": {"--set", "networkPolicy.ciProviderPorts[0]=0"}, + "CI provider port above TCP range": {"--set", "networkPolicy.ciProviderPorts[0]=65536"}, + "duplicate CI provider port": {"--set", "networkPolicy.ciProviderPorts[0]=8080", "--set", "networkPolicy.ciProviderPorts[1]=8080"}, + "noninteger CI provider port": {"--set-string", "networkPolicy.ciProviderPorts[0]=http"}, + "CI provider half selector": {"--set-string", "networkPolicy.ciProvider.namespaceSelector.matchLabels.scope=linkedin-bot"}, + "unauthenticated mode with credential references": {"--set", "session.omp.allowUnauthenticated=true"}, + "credential mode with only Secret": {"--set-string", "session.omp.credentialKey="}, + "credential mode with only key": {"--set-string", "session.omp.credentialSecret="}, + } { + t.Run(name, func(t *testing.T) { + helmTemplateMustFail(t, append(enabledValues(), values...)...) + }) + } +} + +func TestValuesSchemaBoundsRoutePortLists(t *testing.T) { + for _, field := range []string{"modelRoutePorts", "ciProviderPorts"} { + t.Run(field, func(t *testing.T) { + values := enabledValues() + for index := 0; index < 17; index++ { + values = append(values, "--set", "networkPolicy."+field+"["+strconv.Itoa(index)+"]="+strconv.Itoa(20000+index)) + } + helmTemplateMustFail(t, values...) + }) + } +} + +func TestEnabledChartRequiresCommonOMPReferencesAndStrictCredentials(t *testing.T) { + for _, key := range []string{"configMap", "modelsKey", "settingsKey", "credentialSecret", "credentialKey"} { + t.Run(key, func(t *testing.T) { + helmTemplateMustFail(t, append(enabledValues(), "--set-string", "session.omp."+key+"=")...) + }) + } +} + +func TestEnabledChartSupportsExplicitUnauthenticatedOMPMode(t *testing.T) { + output := helmTemplate(t, append(enabledValues(), + "--set", "session.omp.allowUnauthenticated=true", + "--set-string", "session.omp.credentialSecret=", + "--set-string", "session.omp.credentialKey=", + )...) + controller := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-controller\"") + assertContains(t, controller, + "name: T4_SESSION_OMP_ALLOW_UNAUTHENTICATED\n value: \"true\"", + "name: T4_SESSION_OMP_CREDENTIAL_SECRET\n value: \"\"", + "name: T4_SESSION_OMP_CREDENTIAL_KEY\n value: \"\"", + ) + assertCount(t, output, "kind: Secret", 0) +} + +func TestSessionOMPReferencesArePassedWithoutCreatingConfigurationObjects(t *testing.T) { + output := helmTemplate(t, enabledValues()...) + controller := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-controller\"") + assertContains(t, controller, + "name: T4_SESSION_OMP_CONFIG_MAP\n value: \"omp-runtime-config\"", + "name: T4_SESSION_OMP_MODELS_KEY\n value: \"provider-models\"", + "name: T4_SESSION_OMP_SETTINGS_KEY\n value: \"agent-settings\"", + "name: T4_SESSION_OMP_CREDENTIAL_SECRET\n value: \"omp-runtime-credential\"", + "name: T4_SESSION_OMP_CREDENTIAL_KEY\n value: \"MODEL_API_KEY\"", + "name: T4_SESSION_OMP_ALLOW_UNAUTHENTICATED\n value: \"false\"", + ) + assertCount(t, output, "kind: ConfigMap", 0) + assertCount(t, output, "kind: Secret", 0) +} + +func TestNumericDNSReferencesStayQuoted(t *testing.T) { + output := helmTemplate(t, append(enabledValues(), + "--set-string", "clusterHost.name=123", + "--set-string", "storage.adminRWXStorageClass=456", + )...) + host := documentContainingKind(t, output, "T4ClusterHost", "name: \"123\"") + assertContains(t, host, "storageClassName: \"456\"") +} + +func TestClusterHostDoesNotAdvertiseIgnoredProjectionConfiguration(t *testing.T) { + root := repoRoot(t) + output := helmTemplate(t, enabledValues()...) + host := documentContainingKind(t, output, "T4ClusterHost", "name: \"t4-cluster\"") + for _, ignored := range []string{"projection:", "maxWorkspaces", "resyncSeconds"} { + if strings.Contains(host, ignored) { + t.Fatalf("rendered T4ClusterHost advertises ignored configuration %q", ignored) + } + } + + crdRaw := mustRead(t, filepath.Join(root, "deploy", "charts", "t4-cluster", "crds", "t4clusterhosts.cluster.t4.dev.yaml")) + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal([]byte(crdRaw), &crd); err != nil { + t.Fatalf("decode cluster host CRD: %v", err) + } + spec := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + if _, ok := spec.Properties["projection"]; ok { + t.Fatal("T4ClusterHost CRD exposes ignored spec.projection") + } + + for _, path := range []string{ + filepath.Join(root, "deploy", "charts", "t4-cluster", "values.yaml"), + filepath.Join(root, "deploy", "charts", "t4-cluster", "values.schema.json"), + filepath.Join(root, "deploy", "charts", "t4-cluster", "templates", "clusterhost.yaml"), + filepath.Join(root, "packages", "cluster-operator", "api", "v1alpha1", "types.go"), + } { + content := strings.ToLower(mustRead(t, path)) + for _, name := range []string{"projection", "maxworkspaces", "resyncseconds"} { + if strings.Contains(content, name) { + t.Fatalf("%s advertises ignored cluster projection configuration %q", path, name) + } + } + } +} + +func TestDNSAndSourceSelectorsAreConfigurableAndReleaseScoped(t *testing.T) { + defaults := helmTemplate(t, enabledValues()...) + defaultDNS := documentContainingKind(t, defaults, "NetworkPolicy", "name: \"release-name-t4-cluster-dns\"") + assertContains(t, defaultDNS, "kubernetes.io/metadata.name: kube-system", "k8s-app: kube-dns") + output := helmTemplate(t, append(enabledValues(), + "--set-string", "networkPolicy.dns.namespaceSelector.matchLabels.scope=custom-dns-namespace", + "--set-string", "networkPolicy.dns.podSelector.matchLabels.scope=custom-dns-pod", + "--set-string", "networkPolicy.gatewayIngress.namespaceSelector.matchLabels.scope=gateway-namespace", + "--set-string", "networkPolicy.gatewayIngress.podSelector.matchLabels.scope=gateway-pod", + "--set-string", "networkPolicy.observability.namespaceSelector.matchLabels.scope=metrics-namespace", + "--set-string", "networkPolicy.observability.podSelector.matchLabels.scope=metrics-pod", + )...) + dns := documentContainingKind(t, output, "NetworkPolicy", "name: \"release-name-t4-cluster-dns\"") + assertContains(t, dns, "scope: custom-dns-namespace", "scope: custom-dns-pod") + gateway := documentContainingKind(t, output, "NetworkPolicy", "name: \"release-name-t4-cluster-gateway-ingress\"") + assertContains(t, gateway, "scope: gateway-namespace", "scope: gateway-pod") + metrics := documentContainingKind(t, output, "NetworkPolicy", "name: \"release-name-t4-cluster-observability\"") + assertContains(t, metrics, + "app.kubernetes.io/instance: \"release-name\"", + "app.kubernetes.io/part-of: \"t4-cluster\"", + "scope: metrics-namespace", + "scope: metrics-pod", + ) +} + +func TestIngressRequiresTLSAndSupportsTailscaleManagedCertificates(t *testing.T) { + output := helmTemplate(t, append(enabledValues(), + "--set", "ingress.enabled=true", + "--set-string", "ingress.className=tailscale", + "--set-string", "ingress.host=operator.example.ts.net", + )...) + ingress := documentContainingKind(t, output, "Ingress", "name: \"release-name-t4-cluster\"") + assertContains(t, ingress, + "ingressClassName: \"tailscale\"", + "tls:", + "hosts: [\"operator.example.ts.net\"]", + ) + if strings.Contains(ingress, "secretName:") { + t.Fatal("Tailscale-managed ingress invented a TLS Secret reference") + } + helmTemplateMustFail(t, append(enabledValues(), + "--set", "ingress.enabled=true", + "--set-string", "ingress.className=nginx", + "--set-string", "ingress.host=operator.example.test", + )...) + helmTemplateMustFail(t, append(enabledValues(), + "--set", "ingress.enabled=true", + "--set-string", "ingress.className=tailscale", + "--set-string", "ingress.host=operator.example.ts.net", + "--set", "ingress.tls.enabled=false", + )...) +} + +func TestRBACSeparatesControllerMutationFromServerProjection(t *testing.T) { + output := helmTemplate(t, enabledValues()...) + controllerRole := documentContaining(t, output, "name: \"release-name-t4-cluster-controller\"") + serverRole := documentContaining(t, output, "name: \"release-name-t4-cluster-server\"") + assertContains(t, controllerRole, "persistentvolumeclaims", "pods", "services", "t4sessions/status", "leases") + assertContains(t, controllerRole, + "resources: [configmaps]", + "resourceNames: [\"omp-runtime-config\"]", + "resources: [secrets]", + "resourceNames: [\"omp-runtime-credential\"]", + "verbs: [get]", + ) + assertContains(t, serverRole, "t4clusterhosts", "t4workspaces", "t4sessions", "create", "list", "watch") + if strings.Contains(serverRole, "secrets") || strings.Contains(serverRole, "persistentvolumeclaims") || strings.Contains(serverRole, "t4sessions/status") { + t.Fatal("server role can read secrets or mutate controller-owned infrastructure/status") + } +} +func TestUnauthenticatedOMPControllerCannotReadSecrets(t *testing.T) { + output := helmTemplate(t, append(enabledValues(), + "--set", "session.omp.allowUnauthenticated=true", + "--set-string", "session.omp.credentialSecret=", + "--set-string", "session.omp.credentialKey=", + )...) + controllerRole := documentContaining(t, output, "name: \"release-name-t4-cluster-controller\"") + if strings.Contains(controllerRole, "resources: [secrets]") { + t.Fatal("unauthenticated OMP controller can read Secrets") + } +} + +func TestChartUsesOnlyProjectedServiceAccountIdentityForInternalPeers(t *testing.T) { + output := helmTemplate(t, enabledValues()...) + assertKindCount(t, output, "ServiceAccount", 3) + assertCount(t, output, "kind: Secret", 0) + server := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-server\"") + assertContains(t, server, + "serviceAccountName: \"release-name-t4-cluster-server\"", + "name: T4_CLUSTER_IDENTITY_TOKEN_FILE", + "/var/run/secrets/t4-cluster-identity/token", + "serviceAccountToken:", + "audience: \"t4-cluster-internal\"", + "expirationSeconds: 600", + ) + controller := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-controller\"") + assertContains(t, controller, + "name: T4_SESSION_SERVICE_ACCOUNT", + "value: \"release-name-t4-cluster-session\"", + "name: T4_CLUSTER_SERVER_SERVICE_ACCOUNT", + "value: \"release-name-t4-cluster-server\"", + ) + sessionRole := documentContainingKind(t, output, "ClusterRole", "name: \"release-name-t4-cluster-session-token-reviewer\"") + assertContains(t, sessionRole, + "apiGroups: [authentication.k8s.io]", + "resources: [tokenreviews]", + "verbs: [create]", + ) + if strings.Count(sessionRole, "- apiGroups:") != 1 || strings.Contains(sessionRole, "get") || strings.Contains(sessionRole, "list") || strings.Contains(sessionRole, "watch") { + t.Fatalf("session ServiceAccount received permissions beyond TokenReview create:\n%s", sessionRole) + } +} + +func TestNetworkPoliciesDefaultDenyAndAllowOnlyDeclaredFlows(t *testing.T) { + output := helmTemplate(t, append(enabledValues(), + "--set", "networkPolicy.kubernetesApiCIDRs[0]=192.0.2.10/32", + "--set", "networkPolicy.modelRouteCIDRs[0]=198.51.100.4/32", + "--set", "networkPolicy.modelRoutePorts[0]=19481", + "--set", "networkPolicy.modelRoutePorts[1]=8443", + "--set", "networkPolicy.ciProviderCIDRs[0]=203.0.113.8/32", + "--set-string", "networkPolicy.modelRoute.namespaceSelector.matchLabels.kubernetes\\.io/metadata\\.name=linkedin-bot", + "--set-string", "networkPolicy.modelRoute.podSelector.matchLabels.app=codex-swap-proxy-fast", + )...) + assertContains(t, output, + "name: \"release-name-t4-cluster-default-deny\"", + "192.0.2.10/32", + "198.51.100.4/32", + "203.0.113.8/32", + "port: 53", + "port: 8787", + ) + sessionPolicy := documentContainingKind(t, output, "NetworkPolicy", "name: \"release-name-t4-cluster-session-host\"") + assertContains(t, sessionPolicy, + "192.0.2.10/32", "198.51.100.4/32", + "kubernetes.io/metadata.name: linkedin-bot", "app: codex-swap-proxy-fast", + "port: 443", "port: 6443", "port: 19481", "port: 8443", + ) + if strings.Count(sessionPolicy, "198.51.100.4/32") != 1 { + t.Fatalf("model CIDR must render once with only its configured TCP ports:\n%s", sessionPolicy) + } + assertCount(t, sessionPolicy, "port: 19481", 2) + assertCount(t, sessionPolicy, "port: 8443", 2) + if strings.Contains(output, "0.0.0.0/0") { + t.Fatal("network policy contains broad Internet egress") + } + + modelOnly := helmTemplate(t, append(enabledValues(), + "--set", "networkPolicy.modelRouteCIDRs[0]=198.51.100.4/32", + "--set", "networkPolicy.modelRoutePorts[0]=19481", + )...) + modelOnlyPolicy := documentContainingKind(t, modelOnly, "NetworkPolicy", "name: \"release-name-t4-cluster-session-host\"") + assertContains(t, modelOnlyPolicy, "198.51.100.4/32", "port: 19481") + if strings.Contains(modelOnlyPolicy, "port: 443") { + t.Fatalf("model route retained a fixed HTTPS port:\n%s", modelOnlyPolicy) + } + + withoutPorts := helmTemplate(t, append(enabledValues(), + "--set", "networkPolicy.modelRouteCIDRs[0]=198.51.100.4/32", + "--set-string", "networkPolicy.modelRoute.namespaceSelector.matchLabels.scope=linkedin-bot", + "--set-string", "networkPolicy.modelRoute.podSelector.matchLabels.scope=codex-swap-proxy-fast", + )...) + withoutPortsPolicy := documentContainingKind(t, withoutPorts, "NetworkPolicy", "name: \"release-name-t4-cluster-session-host\"") + if strings.Contains(withoutPortsPolicy, "198.51.100.4/32") || strings.Contains(withoutPortsPolicy, "linkedin-bot") || strings.Contains(withoutPortsPolicy, "codex-swap-proxy-fast") { + t.Fatalf("model destination without an explicit route port broadened egress:\n%s", withoutPortsPolicy) + } +} + +func TestCIProviderRoutesUseOnlyConfiguredDestinationsAndPorts(t *testing.T) { + defaults := helmTemplate(t, enabledValues()...) + defaultServerPolicy := documentContainingKind(t, defaults, "NetworkPolicy", "name: \"release-name-t4-cluster-server-egress\"") + if strings.Contains(defaultServerPolicy, "port: 443") { + t.Fatalf("default CI port rendered without a configured destination:\n%s", defaultServerPolicy) + } + + output := helmTemplate(t, append(enabledValues(), + "--set", "networkPolicy.ciProviderCIDRs[0]=203.0.113.8/32", + "--set", "networkPolicy.ciProviderPorts[0]=8080", + "--set-string", "networkPolicy.ciProvider.namespaceSelector.matchLabels.kubernetes\\.io/metadata\\.name=linkedin-bot", + "--set-string", "networkPolicy.ciProvider.podSelector.matchLabels.app=woodpecker-server", + )...) + serverPolicy := documentContainingKind(t, output, "NetworkPolicy", "name: \"release-name-t4-cluster-server-egress\"") + assertContains(t, serverPolicy, + "203.0.113.8/32", + "kubernetes.io/metadata.name: linkedin-bot", + "app: woodpecker-server", + "port: 8080", + ) + assertCount(t, serverPolicy, "port: 8080", 2) + if strings.Contains(serverPolicy, "port: 443") { + t.Fatalf("CI route retained a fixed HTTPS port:\n%s", serverPolicy) + } +} + +func TestWoodpeckerCanUseRotatingProjectedServiceAccountIdentity(t *testing.T) { + values := append(enabledValues(), + "--set", "woodpecker.configMap=woodpecker-config", + "--set", "woodpecker.serviceAccountAudience=woodpecker-ci-trigger", + ) + output := helmTemplate(t, values...) + server := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-server\"") + assertContains(t, server, + "name: T4_WOODPECKER_TOKEN_FILE", + "/var/run/secrets/t4-ci/token", + "audience: \"woodpecker-ci-trigger\"", + "expirationSeconds: 600", + ) + host := documentContainingKind(t, output, "T4ClusterHost", "name: \"t4-cluster\"") + assertContains(t, host, "serviceAccountAudience: \"woodpecker-ci-trigger\"", "name: \"woodpecker-config\"") +} + +func TestCRDsRemainExplicitAcrossUpgradeAndUninstall(t *testing.T) { + withoutCRDs := helmTemplate(t, enabledValues()...) + if strings.Contains(withoutCRDs, "kind: CustomResourceDefinition") { + t.Fatal("CRDs must live in Helm crds/, not upgrade-rendered templates") + } + withCRDs := helmTemplate(t, append([]string{"--include-crds"}, enabledValues()...)...) + assertCount(t, withCRDs, "kind: CustomResourceDefinition", 3) + assertContains(t, withCRDs, "t4clusterhosts.cluster.t4.dev", "t4workspaces.cluster.t4.dev", "t4sessions.cluster.t4.dev") + + docs, err := os.ReadFile(filepath.Join(repoRoot(t), "docs", "CLUSTER_OPERATOR.md")) + if err != nil { + t.Fatal(err) + } + for _, required := range []string{"helm upgrade", "helm rollback", "helm uninstall", "kubectl apply --server-side -f deploy/charts/t4-cluster/crds/", "condition=Established", "Do not rely on `helm upgrade` to change CRDs", "Retain", "Delete", "CRDs are not removed"} { + if !strings.Contains(string(docs), required) { + t.Fatalf("operator guide lacks upgrade/uninstall contract %q", required) + } + } +} + +func TestWorkspaceRetentionPolicyIsImmutable(t *testing.T) { + root := repoRoot(t) + raw := mustRead(t, filepath.Join(root, "deploy", "charts", "t4-cluster", "crds", "t4workspaces.cluster.t4.dev.yaml")) + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal([]byte(raw), &crd); err != nil { + t.Fatalf("decode workspace CRD: %v", err) + } + if len(crd.Spec.Versions) != 1 || crd.Spec.Versions[0].Schema == nil || crd.Spec.Versions[0].Schema.OpenAPIV3Schema == nil { + t.Fatal("workspace CRD lacks its single versioned OpenAPI schema") + } + retentionPolicy, ok := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"].Properties["retentionPolicy"] + if !ok { + t.Fatal("workspace CRD lacks spec.retentionPolicy") + } + if len(retentionPolicy.Enum) != 2 { + t.Fatalf("retention policy enum = %v, want exactly Retain and Delete", retentionPolicy.Enum) + } + allowed := map[string]bool{`"Retain"`: false, `"Delete"`: false} + for _, value := range retentionPolicy.Enum { + if _, expected := allowed[string(value.Raw)]; !expected { + t.Fatalf("retention policy permits unexpected initial value %s", value.Raw) + } + allowed[string(value.Raw)] = true + } + for value, found := range allowed { + if !found { + t.Fatalf("retention policy rejects initial value %s", value) + } + } + + immutable := false + for _, validation := range retentionPolicy.XValidations { + if validation.Rule == "self == oldSelf" && validation.Message == "retentionPolicy is immutable" { + immutable = true + } + } + if !immutable { + t.Fatal("spec.retentionPolicy lacks the immutable CEL transition rule and clear message") + } + + api := mustRead(t, filepath.Join(root, "packages", "cluster-operator", "api", "v1alpha1", "types.go")) + assertContains(t, api, + "// +kubebuilder:validation:Enum=Retain;Delete\ntype RetentionPolicy string", + "// +kubebuilder:validation:XValidation:rule=\"self == oldSelf\",message=\"retentionPolicy is immutable\"\n\tRetentionPolicy RetentionPolicy", + ) +} + +func TestImageContractsArePinnedAndAuthorityCompatible(t *testing.T) { + root := repoRoot(t) + controller := mustRead(t, filepath.Join(root, "cluster", "images", "controller", "Dockerfile")) + server := mustRead(t, filepath.Join(root, "cluster", "images", "cluster-server", "Dockerfile")) + session := mustRead(t, filepath.Join(root, "cluster", "images", "session-runtime", "Dockerfile")) + entrypoint := mustRead(t, filepath.Join(root, "cluster", "images", "session-runtime", "session-entrypoint.sh")) + for name, content := range map[string]string{"controller": controller, "server": server, "session": session} { + if !strings.Contains(content, "@sha256:") { + t.Fatalf("%s image uses an unpinned base", name) + } + } + assertContains(t, session, + "8476f4451ed95c5d5401785d279a93d3c659fac4", + "t4code-17.0.5-appserver-10", + "t4-omp-authority/1", + "session-entrypoint.sh", + "chromium", + "Xvfb", + ) + assertContains(t, entrypoint, "packages/cluster-server/src/session-host-main.ts") + for name, content := range map[string]string{"server": server, "session": session} { + assertContains(t, content, "pnpm install --frozen-lockfile") + if strings.Contains(content, "bun install --ignore-scripts --lockfile-only") { + t.Fatalf("%s image synthesizes an uncommitted dependency lock", name) + } + } + if strings.Contains(session, "ARG BUN_IMAGE") || strings.Contains(session, "ARG OMP_TAG") || strings.Contains(session, "ARG OMP_COMMIT") { + t.Fatal("session runtime permits overriding a labeled runtime pin") + } + assertContains(t, session, + "refs/tags/t4code-17.0.5-appserver-10", + "git checkout --detach \"8476f4451ed95c5d5401785d279a93d3c659fac4\"", + "snapshot.debian.org/archive/debian/20250721T000000Z", + ) + assertContains(t, server, "snapshot.debian.org/archive/debian/20250721T000000Z") + assertContains(t, controller, "ARG TARGETOS\n", "ARG TARGETARCH\n") + if strings.Contains(controller, "TARGETARCH=amd64") || strings.Contains(controller, "org.opencontainers.image.architecture") { + t.Fatal("controller image hardcodes or claims a single/unbuilt architecture") + } + assertContains(t, server, "packages/cluster-server/src/main.ts") + assertContains(t, entrypoint, + "T4_CLUSTER_SERVER_SERVICE_ACCOUNT", + "/var/run/secrets/kubernetes.io/serviceaccount/token", + "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + "/var/run/secrets/kubernetes.io/serviceaccount/namespace", + "T4_OMP_CONFIG_SOURCE_DIR", + "T4_OMP_ALLOW_UNAUTHENTICATED", + "T4_OMP_CREDENTIAL_KEY", + `if [[ "${T4_OMP_ALLOW_UNAUTHENTICATED}" == "false" ]]`, + `export HOME="${T4_SESSION_STATE_ROOT}/home"`, + `export PI_CODING_AGENT_DIR="${HOME}/.omp/profiles/${T4_SESSION_NAME}/agent"`, + `T4_*|OMP_*|PI_*|XDG_*|LD_*`, + `install -m 0600 "${models_source}"`, + `install -m 0600 "${settings_source}"`, + `"${PI_CODING_AGENT_DIR}/models.yml"`, + `"${PI_CODING_AGENT_DIR}/config.yml"`, + ) +} + +func TestSessionEntrypointFailsClosedBeforeGUIWithoutPrivateOMPInputs(t *testing.T) { + entrypoint := filepath.Join(repoRoot(t), "cluster", "images", "session-runtime", "session-entrypoint.sh") + const secretSentinel = "must-not-appear-in-logs" + for _, test := range []struct { + name string + writeModels bool + models string + writeSettings bool + settings string + credential string + condition string + }{ + {name: "missing models file", writeSettings: true, settings: "settings", credential: secretSentinel, condition: "omp_models"}, + {name: "empty settings file", writeModels: true, models: "models", writeSettings: true, credential: secretSentinel, condition: "omp_settings"}, + {name: "empty credential", writeModels: true, models: "models", writeSettings: true, settings: "settings", condition: "omp_credential"}, + } { + t.Run(test.name, func(t *testing.T) { + root := t.TempDir() + source := filepath.Join(root, "omp-source") + projection := filepath.Join(root, "kubernetes") + bin := filepath.Join(root, "bin") + for _, directory := range []string{source, projection, bin} { + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + } + if test.writeModels { + if err := os.WriteFile(filepath.Join(source, "models.yml"), []byte(test.models), 0o600); err != nil { + t.Fatal(err) + } + } + if test.writeSettings { + if err := os.WriteFile(filepath.Join(source, "config.yml"), []byte(test.settings), 0o600); err != nil { + t.Fatal(err) + } + } + for _, name := range []string{"token", "ca.crt", "namespace"} { + if err := os.WriteFile(filepath.Join(projection, name), []byte("projected"), 0o600); err != nil { + t.Fatal(err) + } + } + marker := filepath.Join(root, "xvfb-started") + fakeXvfb := "#!/usr/bin/env bash\nprintf started > \"${T4_TEST_XVFB_MARKER}\"\n" + if err := os.WriteFile(filepath.Join(bin, "Xvfb"), []byte(fakeXvfb), 0o700); err != nil { + t.Fatal(err) + } + command := exec.Command("bash", entrypoint, "MODEL_API_KEY") + command.Env = append(os.Environ(), + "PATH="+bin+":"+os.Getenv("PATH"), + "T4_SESSION_STATE_ROOT=/workspace/.t4/sessions/session-a", + "T4_SESSION_NAME=session-a", + "T4_AUTHORITY_STATE_DIR=/workspace/.t4/sessions/session-a/authority", + "T4_BROWSER_STATE_DIR=/workspace/.t4/sessions/session-a/browser", + "T4_CLUSTER_SERVER_SERVICE_ACCOUNT=t4-cluster-server", + "T4_KUBERNETES_TOKEN_PATH="+filepath.Join(projection, "token"), + "T4_KUBERNETES_CA_PATH="+filepath.Join(projection, "ca.crt"), + "T4_KUBERNETES_NAMESPACE_PATH="+filepath.Join(projection, "namespace"), + "T4_OMP_CONFIG_SOURCE_DIR="+source, + "T4_OMP_CREDENTIAL_KEY=MODEL_API_KEY", + "T4_TEST_XVFB_MARKER="+marker, + "MODEL_API_KEY="+test.credential, + ) + output, err := command.CombinedOutput() + exitError, ok := err.(*exec.ExitError) + if !ok || exitError.ExitCode() != 64 { + t.Fatalf("entrypoint exit = %v, want code 64; output=%s", err, output) + } + if !strings.Contains(string(output), `"condition":"`+test.condition+`"`) { + t.Fatalf("entrypoint output lacks bounded failure condition %q: %s", test.condition, output) + } + if strings.Contains(string(output), secretSentinel) { + t.Fatalf("entrypoint logged credential value: %s", output) + } + if _, err := os.Stat(marker); !os.IsNotExist(err) { + t.Fatalf("Xvfb started before OMP configuration passed validation: %v", err) + } + }) + } +} + +func helmTemplate(t *testing.T, extra ...string) string { + t.Helper() + return helmTemplateRelease(t, "release-name", extra...) +} + +func helmTemplateRelease(t *testing.T, releaseName string, extra ...string) string { + t.Helper() + args := []string{"template", releaseName, filepath.Join(repoRoot(t), "deploy", "charts", "t4-cluster"), "--namespace", "t4-system"} + args = append(args, extra...) + command := exec.Command("helm", args...) + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("helm %s: %v\n%s", strings.Join(args, " "), err, output) + } + return string(output) +} +func helmTemplateMustFail(t *testing.T, extra ...string) { + t.Helper() + args := []string{"template", "release-name", filepath.Join(repoRoot(t), "deploy", "charts", "t4-cluster"), "--namespace", "t4-system"} + args = append(args, extra...) + command := exec.Command("helm", args...) + if output, err := command.CombinedOutput(); err == nil { + t.Fatalf("helm unexpectedly accepted invalid values: %s", output) + } +} + +func enabledValues() []string { + return []string{ + "--set", "enabled=true", + "--set", "storage.adminRWXStorageClass=portable-rwx", + "--set", "images.controller.digest=" + fakeDigest, + "--set", "images.server.digest=" + fakeDigest, + "--set", "images.sessionRuntime.digest=" + fakeDigest, + "--set", "server.trustedProxyCIDRs[0]=192.0.2.0/24", + "--set", "session.omp.configMap=omp-runtime-config", + "--set", "session.omp.modelsKey=provider-models", + "--set", "session.omp.settingsKey=agent-settings", + "--set", "session.omp.credentialSecret=omp-runtime-credential", + "--set", "session.omp.credentialKey=MODEL_API_KEY", + "--set", "session.omp.allowUnauthenticated=false", + } +} + +func repoRoot(t *testing.T) string { + t.Helper() + root, err := filepath.Abs(filepath.Join("..", "..", "..")) + if err != nil { + t.Fatal(err) + } + return root +} + +func mustRead(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func assertContains(t *testing.T, value string, required ...string) { + t.Helper() + for _, item := range required { + if !strings.Contains(value, item) { + t.Fatalf("output lacks %q", item) + } + } +} + +func assertCount(t *testing.T, value, needle string, want int) { + t.Helper() + if got := strings.Count(value, needle); got != want { + t.Fatalf("count(%q) = %d, want %d", needle, got, want) + } +} + +func assertKindCount(t *testing.T, rendered, kind string, want int) { + t.Helper() + needle := "kind: " + kind + got := 0 + for _, line := range strings.Split(rendered, "\n") { + if line == needle { + got++ + } + } + if got != want { + t.Fatalf("kind %q count = %d, want %d", kind, got, want) + } +} + +func documentContaining(t *testing.T, rendered, needle string) string { + t.Helper() + for _, document := range strings.Split(rendered, "\n---") { + if strings.Contains(document, "kind: Role\n") && strings.Contains(document, needle) { + return document + } + } + t.Fatalf("no rendered document contains %q", needle) + return "" +} + +func documentContainingKind(t *testing.T, rendered, kind, needle string) string { + t.Helper() + for _, document := range strings.Split(rendered, "\n---") { + if strings.Contains(document, "kind: "+kind+"\n") && strings.Contains(document, needle) { + return document + } + } + t.Fatalf("no rendered %s contains %q", kind, needle) + return "" +} diff --git a/packages/cluster-operator/cmd/manager/main.go b/packages/cluster-operator/cmd/manager/main.go new file mode 100644 index 00000000..d216db5a --- /dev/null +++ b/packages/cluster-operator/cmd/manager/main.go @@ -0,0 +1,148 @@ +package main + +import ( + "flag" + "os" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apiresource "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" + "github.com/LycaonLLC/t4-code/packages/cluster-operator/controllers" +) + +var scheme = runtime.NewScheme() + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(storagev1.AddToScheme(scheme)) + utilruntime.Must(clusterv1alpha1.AddToScheme(scheme)) +} + +func managerOptions() ctrl.Options { + leaseDuration := 30 * time.Second + renewDeadline := 20 * time.Second + retryPeriod := 5 * time.Second + options := ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: ":8080"}, + HealthProbeBindAddress: ":8081", + LeaderElection: true, + LeaderElectionResourceLock: "leases", + LeaderElectionID: "t4-cluster-operator.cluster.t4.dev", + LeaderElectionReleaseOnCancel: true, + LeaseDuration: &leaseDuration, + RenewDeadline: &renewDeadline, + RetryPeriod: &retryPeriod, + } + if namespace := strings.TrimSpace(os.Getenv("POD_NAMESPACE")); namespace != "" { + options.Cache = cache.Options{DefaultNamespaces: map[string]cache.Config{namespace: {}}} + } + return options +} + +func main() { + var development bool + flag.BoolVar(&development, "development", false, "enable development logging") + zapOptions := zap.Options{Development: development} + zapOptions.BindFlags(flag.CommandLine) + flag.Parse() + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) + + manager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), managerOptions()) + if err != nil { + ctrl.Log.Error(err, "unable to create controller manager") + os.Exit(1) + } + if err := (&controllers.ClusterHostReconciler{Client: manager.GetClient(), Scheme: manager.GetScheme()}).SetupWithManager(manager); err != nil { + ctrl.Log.Error(err, "unable to register T4ClusterHost controller") + os.Exit(1) + } + if err := (&controllers.WorkspaceReconciler{Client: manager.GetClient(), Scheme: manager.GetScheme()}).SetupWithManager(manager); err != nil { + ctrl.Log.Error(err, "unable to register T4Workspace controller") + os.Exit(1) + } + excludedNodes := splitNonempty(os.Getenv("T4_SESSION_EXCLUDED_NODES")) + sessionServiceAccount, serverServiceAccount := sessionServiceAccountNames() + if err := (&controllers.SessionReconciler{ + Client: manager.GetClient(), APIReader: manager.GetAPIReader(), Scheme: manager.GetScheme(), + RuntimeImage: os.Getenv("T4_SESSION_RUNTIME_IMAGE"), + SessionServiceAccountName: sessionServiceAccount, + ServerServiceAccountName: serverServiceAccount, + KubernetesAPIAudience: envOr("T4_KUBERNETES_API_AUDIENCE", controllers.DefaultKubernetesAPIAudience), + OMPConfig: sessionOMPConfigFromEnv(), + ExcludedNodeNames: excludedNodes, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: apiresource.MustParse(envOr("T4_SESSION_REQUEST_CPU", "500m")), + corev1.ResourceMemory: apiresource.MustParse(envOr("T4_SESSION_REQUEST_MEMORY", "1Gi")), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: apiresource.MustParse(envOr("T4_SESSION_LIMIT_CPU", "4")), + corev1.ResourceMemory: apiresource.MustParse(envOr("T4_SESSION_LIMIT_MEMORY", "8Gi")), + }, + }, + SharedMemorySize: apiresource.MustParse(envOr("T4_SESSION_SHM_SIZE", "1Gi")), + TemporarySize: apiresource.MustParse(envOr("T4_SESSION_TEMPORARY_SIZE", "2Gi")), + }).SetupWithManager(manager); err != nil { + ctrl.Log.Error(err, "unable to register T4Session controller") + os.Exit(1) + } + if err := manager.AddHealthzCheck("healthz", healthz.Ping); err != nil { + ctrl.Log.Error(err, "unable to install health check") + os.Exit(1) + } + if err := manager.AddReadyzCheck("readyz", healthz.Ping); err != nil { + ctrl.Log.Error(err, "unable to install readiness check") + os.Exit(1) + } + if err := manager.Start(ctrl.SetupSignalHandler()); err != nil { + ctrl.Log.Error(err, "controller manager stopped") + os.Exit(1) + } +} + +func splitNonempty(value string) []string { + var result []string + for _, item := range strings.Split(value, ",") { + if item = strings.TrimSpace(item); item != "" { + result = append(result, item) + } + } + return result +} + +func sessionOMPConfigFromEnv() controllers.SessionOMPConfig { + return controllers.SessionOMPConfig{ + ConfigMapName: os.Getenv("T4_SESSION_OMP_CONFIG_MAP"), + ModelsKey: os.Getenv("T4_SESSION_OMP_MODELS_KEY"), + SettingsKey: os.Getenv("T4_SESSION_OMP_SETTINGS_KEY"), + CredentialSecretName: os.Getenv("T4_SESSION_OMP_CREDENTIAL_SECRET"), + CredentialKey: os.Getenv("T4_SESSION_OMP_CREDENTIAL_KEY"), + AllowUnauthenticated: strings.EqualFold(strings.TrimSpace(os.Getenv("T4_SESSION_OMP_ALLOW_UNAUTHENTICATED")), "true"), + } +} + +func sessionServiceAccountNames() (string, string) { + return envOr("T4_SESSION_SERVICE_ACCOUNT", controllers.DefaultSessionServiceAccount), + envOr("T4_CLUSTER_SERVER_SERVICE_ACCOUNT", controllers.DefaultServerServiceAccount) +} + +func envOr(name, fallback string) string { + if value := strings.TrimSpace(os.Getenv(name)); value != "" { + return value + } + return fallback +} diff --git a/packages/cluster-operator/cmd/manager/main_test.go b/packages/cluster-operator/cmd/manager/main_test.go new file mode 100644 index 00000000..585c8a08 --- /dev/null +++ b/packages/cluster-operator/cmd/manager/main_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "testing" + "time" + + "github.com/LycaonLLC/t4-code/packages/cluster-operator/controllers" +) + +func TestManagerUsesLeaseLeaderElection(t *testing.T) { + options := managerOptions() + if !options.LeaderElection { + t.Fatal("controller manager must enable leader election") + } + if options.LeaderElectionResourceLock != "leases" { + t.Fatalf("leader-election resource lock = %q, want leases", options.LeaderElectionResourceLock) + } + if options.LeaderElectionID != "t4-cluster-operator.cluster.t4.dev" { + t.Fatalf("leader-election ID = %q", options.LeaderElectionID) + } + if options.LeaseDuration == nil || options.RenewDeadline == nil || options.RetryPeriod == nil { + t.Fatal("leader election timing must be explicit") + } + if *options.LeaseDuration < 15*time.Second || *options.RenewDeadline >= *options.LeaseDuration || *options.RetryPeriod >= *options.RenewDeadline { + t.Fatalf("unsafe leader-election timing: lease=%v renew=%v retry=%v", *options.LeaseDuration, *options.RenewDeadline, *options.RetryPeriod) + } +} + +func TestManagerConfiguresDedicatedSessionAndServerServiceAccounts(t *testing.T) { + t.Setenv("T4_SESSION_SERVICE_ACCOUNT", "release-session") + t.Setenv("T4_CLUSTER_SERVER_SERVICE_ACCOUNT", "release-server") + session, server := sessionServiceAccountNames() + if session != "release-session" || server != "release-server" { + t.Fatalf("ServiceAccounts = %q/%q", session, server) + } + t.Setenv("T4_SESSION_SERVICE_ACCOUNT", "") + t.Setenv("T4_CLUSTER_SERVER_SERVICE_ACCOUNT", "") + session, server = sessionServiceAccountNames() + if session != controllers.DefaultSessionServiceAccount || server != controllers.DefaultServerServiceAccount { + t.Fatalf("default ServiceAccounts = %q/%q", session, server) + } +} + +func TestManagerReadsSessionOMPReferencesWithoutSecretValues(t *testing.T) { + t.Setenv("T4_SESSION_OMP_CONFIG_MAP", "omp-runtime-config") + t.Setenv("T4_SESSION_OMP_MODELS_KEY", "provider-models") + t.Setenv("T4_SESSION_OMP_SETTINGS_KEY", "agent-settings") + t.Setenv("T4_SESSION_OMP_CREDENTIAL_SECRET", "omp-runtime-credential") + t.Setenv("T4_SESSION_OMP_CREDENTIAL_KEY", "MODEL_API_KEY") + t.Setenv("T4_SESSION_OMP_ALLOW_UNAUTHENTICATED", "false") + + got := sessionOMPConfigFromEnv() + want := controllers.SessionOMPConfig{ + ConfigMapName: "omp-runtime-config", + ModelsKey: "provider-models", + SettingsKey: "agent-settings", + CredentialSecretName: "omp-runtime-credential", + CredentialKey: "MODEL_API_KEY", + } + if got != want { + t.Fatalf("session OMP references = %#v, want %#v", got, want) + } + + t.Setenv("T4_SESSION_OMP_CREDENTIAL_SECRET", "") + t.Setenv("T4_SESSION_OMP_CREDENTIAL_KEY", "") + t.Setenv("T4_SESSION_OMP_ALLOW_UNAUTHENTICATED", "true") + got = sessionOMPConfigFromEnv() + want.CredentialSecretName = "" + want.CredentialKey = "" + want.AllowUnauthenticated = true + if got != want { + t.Fatalf("unauthenticated session OMP references = %#v, want %#v", got, want) + } +} diff --git a/packages/cluster-operator/controllers/clusterhost_controller.go b/packages/cluster-operator/controllers/clusterhost_controller.go new file mode 100644 index 00000000..fbd078e1 --- /dev/null +++ b/packages/cluster-operator/controllers/clusterhost_controller.go @@ -0,0 +1,91 @@ +package controllers + +import ( + "context" + "reflect" + "time" + + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" +) + +type ClusterHostReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +func (r *ClusterHostReconciler) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) { + var host clusterv1alpha1.T4ClusterHost + found := false + defer func() { + observeReconcile(metricKindClusterHost, request.NamespacedName, host.Status.Conditions, conditionObjectPresent(&host, found, err), err) + }() + if err := r.Get(ctx, request.NamespacedName, &host); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + found = true + original := host.Status + if host.Status.Conditions != nil { + original.Conditions = append([]metav1.Condition(nil), host.Status.Conditions...) + } + host.Status.ObservedGeneration = host.Generation + + var storageClass storagev1.StorageClass + storageReady := true + if err := r.Get(ctx, types.NamespacedName{Name: host.Spec.StorageClassName}, &storageClass); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + storageReady = false + meta.SetStatusCondition(&host.Status.Conditions, condition("StorageReady", metav1.ConditionFalse, ReasonStorageClassNotFound, "selected StorageClass does not exist", host.Generation)) + } else if !storageClassAllowsRWX(storageClass.Annotations) { + storageReady = false + meta.SetStatusCondition(&host.Status.Conditions, condition("StorageReady", metav1.ConditionFalse, ReasonStorageClassNotRWX, "selected StorageClass is not administrator-declared ReadWriteMany", host.Generation)) + } else { + meta.SetStatusCondition(&host.Status.Conditions, condition("StorageReady", metav1.ConditionTrue, ReasonStorageReady, "selected StorageClass supports ReadWriteMany", host.Generation)) + } + + ciReady := true + ciReason := "NotConfigured" + ciMessage := "CI provider is optional and not configured" + if host.Spec.CIProvider != nil { + hasSecret := host.Spec.CIProvider.SecretRef != nil && host.Spec.CIProvider.SecretRef.Name != "" + hasProjectedIdentity := host.Spec.CIProvider.ServiceAccountAudience != "" + ciReady = host.Spec.CIProvider.ConfigMapRef.Name != "" && hasSecret != hasProjectedIdentity + if ciReady { + ciReason, ciMessage = "ReferencesConfigured", "CI provider configuration and one server-side credential source are configured" + } else { + ciReason, ciMessage = "ReferencesIncomplete", "CI provider requires a ConfigMap and exactly one Secret or projected ServiceAccount identity" + } + } + status := metav1.ConditionFalse + if ciReady { + status = metav1.ConditionTrue + } + meta.SetStatusCondition(&host.Status.Conditions, condition("CIReady", status, ciReason, ciMessage, host.Generation)) + + available := storageReady && len(host.Spec.RuntimeProfiles) > 0 + if available { + meta.SetStatusCondition(&host.Status.Conditions, condition("Available", metav1.ConditionTrue, "ConfigurationAccepted", "cluster host infrastructure configuration is available", host.Generation)) + } else { + meta.SetStatusCondition(&host.Status.Conditions, condition("Available", metav1.ConditionFalse, "ConfigurationNotReady", "cluster host infrastructure configuration is not ready", host.Generation)) + } + if !reflect.DeepEqual(original, host.Status) { + if err := r.Status().Update(ctx, &host); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil +} + +func (r *ClusterHostReconciler) SetupWithManager(manager ctrl.Manager) error { + return ctrl.NewControllerManagedBy(manager).For(&clusterv1alpha1.T4ClusterHost{}).Complete(r) +} diff --git a/packages/cluster-operator/controllers/helpers.go b/packages/cluster-operator/controllers/helpers.go new file mode 100644 index 00000000..ec5681af --- /dev/null +++ b/packages/cluster-operator/controllers/helpers.go @@ -0,0 +1,113 @@ +package controllers + +import ( + "crypto/sha256" + "encoding/hex" + "reflect" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" +) + +const ( + ReasonStorageClassNotFound = "StorageClassNotFound" + ReasonStorageClassNotRWX = "StorageClassNotRWX" + ReasonStorageReady = "StorageClassSupportsRWX" +) + +func WorkspacePVCName(workspace *clusterv1alpha1.T4Workspace) string { + return stableName("t4-ws-", workspace.Name, workspace.UID) +} + +func SessionPodName(session *clusterv1alpha1.T4Session) string { + return stableName("t4-session-", session.Name, session.UID) +} + +func SessionServiceName(session *clusterv1alpha1.T4Session) string { + return stableName("t4-session-", session.Name, session.UID) +} + +func stableName(prefix, name string, uid types.UID) string { + identity := name + ":" + string(uid) + sum := sha256.Sum256([]byte(identity)) + suffix := hex.EncodeToString(sum[:6]) + var body strings.Builder + body.Grow(len(name)) + lastWasSeparator := false + for i := range len(name) { + character := name[i] + if character >= 'A' && character <= 'Z' { + character += 'a' - 'A' + } + if character >= 'a' && character <= 'z' || character >= '0' && character <= '9' { + body.WriteByte(character) + lastWasSeparator = false + } else if body.Len() > 0 && !lastWasSeparator { + body.WriteByte('-') + lastWasSeparator = true + } + } + name = strings.Trim(body.String(), "-") + maxName := 63 - len(prefix) - 1 - len(suffix) + if len(name) > maxName { + name = strings.TrimRight(name[:maxName], "-") + } + if name == "" { + name = "resource" + } + return prefix + name + "-" + suffix +} + +func storageClassAllowsRWX(annotations map[string]string) bool { + for _, mode := range strings.Split(annotations[clusterv1alpha1.RWXStorageClassAnnotation], ",") { + if strings.TrimSpace(mode) == string(corev1.ReadWriteMany) { + return true + } + } + return false +} + +func pvcHasRWX(pvc *corev1.PersistentVolumeClaim) bool { + for _, mode := range pvc.Spec.AccessModes { + if mode == corev1.ReadWriteMany { + return true + } + } + return false +} + +func hasString(values []string, wanted string) bool { + for _, value := range values { + if value == wanted { + return true + } + } + return false +} + +func removeString(values []string, unwanted string) []string { + result := values[:0] + for _, value := range values { + if value != unwanted { + result = append(result, value) + } + } + return result +} + +func condition(conditionType string, status metav1.ConditionStatus, reason, message string, generation int64) metav1.Condition { + return metav1.Condition{ + Type: conditionType, Status: status, Reason: reason, Message: message, + ObservedGeneration: generation, LastTransitionTime: metav1.NewTime(time.Now().UTC()), + } +} + +func objectChanged(before, after client.Object) bool { + return !reflect.DeepEqual(before, after) +} diff --git a/packages/cluster-operator/controllers/metrics.go b/packages/cluster-operator/controllers/metrics.go new file mode 100644 index 00000000..f37a798a --- /dev/null +++ b/packages/cluster-operator/controllers/metrics.go @@ -0,0 +1,136 @@ +package controllers + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + controllermetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const ( + metricKindClusterHost = "clusterhost" + metricKindWorkspace = "workspace" + metricKindSession = "session" +) + +var ( + reconcileTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "t4_cluster_reconcile_total", + Help: "Total completed T4 cluster reconciliations by resource kind and result.", + }, + []string{"kind", "result"}, + ) + conditionGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "t4_cluster_condition", + Help: "Number of currently observed T4 cluster resources by kind, condition, and boolean status.", + }, + []string{"kind", "condition", "status"}, + ) + registerControllerMetricsOnce sync.Once + conditionStore = aggregateConditionStore{ + resources: make(map[string]map[types.NamespacedName]map[string]metav1.ConditionStatus), + } +) + +var knownConditions = map[string][]string{ + metricKindClusterHost: {"Available", "CIReady", "StorageReady"}, + metricKindWorkspace: {"HostReady", "Ready", "StorageReady"}, + metricKindSession: {"Available", "HostReady", "RuntimeConfigured", "WorkspaceReady"}, +} + +type aggregateConditionStore struct { + mu sync.Mutex + resources map[string]map[types.NamespacedName]map[string]metav1.ConditionStatus +} + +func init() { + registerControllerMetrics() +} + +func registerControllerMetrics() { + registerControllerMetricsOnce.Do(func() { + controllermetrics.Registry.MustRegister(reconcileTotal, conditionGauge) + for kind, conditions := range knownConditions { + reconcileTotal.WithLabelValues(kind, "success") + reconcileTotal.WithLabelValues(kind, "error") + for _, conditionType := range conditions { + conditionGauge.WithLabelValues(kind, conditionType, "true").Set(0) + conditionGauge.WithLabelValues(kind, conditionType, "false").Set(0) + } + } + }) +} + +func observeReconcile(kind string, key types.NamespacedName, conditions []metav1.Condition, objectPresent bool, reconcileErr error) { + result := "success" + if reconcileErr != nil { + result = "error" + } + reconcileTotal.WithLabelValues(kind, result).Inc() + + if !objectPresent && reconcileErr != nil { + return + } + conditionStore.project(kind, key, conditions, objectPresent) +} + +func conditionObjectPresent(object metav1.Object, fetched bool, reconcileErr error) bool { + if !fetched { + return false + } + return reconcileErr != nil || object.GetDeletionTimestamp().IsZero() || len(object.GetFinalizers()) > 0 +} + +func (s *aggregateConditionStore) project(kind string, key types.NamespacedName, conditions []metav1.Condition, objectPresent bool) { + s.mu.Lock() + defer s.mu.Unlock() + + resources := s.resources[kind] + if resources == nil { + resources = make(map[types.NamespacedName]map[string]metav1.ConditionStatus) + s.resources[kind] = resources + } + if objectPresent { + current := resources[key] + if current == nil { + current = make(map[string]metav1.ConditionStatus, len(knownConditions[kind])) + } else { + clear(current) + } + for i := range conditions { + if containsCondition(knownConditions[kind], conditions[i].Type) && (conditions[i].Status == metav1.ConditionTrue || conditions[i].Status == metav1.ConditionFalse) { + current[conditions[i].Type] = conditions[i].Status + } + } + resources[key] = current + } else { + delete(resources, key) + } + + for _, conditionType := range knownConditions[kind] { + var trueCount, falseCount int + for _, current := range resources { + switch current[conditionType] { + case metav1.ConditionTrue: + trueCount++ + case metav1.ConditionFalse: + falseCount++ + } + } + conditionGauge.WithLabelValues(kind, conditionType, "true").Set(float64(trueCount)) + conditionGauge.WithLabelValues(kind, conditionType, "false").Set(float64(falseCount)) + } +} + +func containsCondition(conditions []string, wanted string) bool { + for _, condition := range conditions { + if condition == wanted { + return true + } + } + return false +} diff --git a/packages/cluster-operator/controllers/metrics_test.go b/packages/cluster-operator/controllers/metrics_test.go new file mode 100644 index 00000000..f0078ec7 --- /dev/null +++ b/packages/cluster-operator/controllers/metrics_test.go @@ -0,0 +1,350 @@ +package controllers + +import ( + "context" + "errors" + "reflect" + "sort" + "sync" + "testing" + + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + controllermetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" +) + +func TestReconcilersEmitBoundedCompletionMetrics(t *testing.T) { + scheme := metricsTestScheme(t) + missingKey := client.ObjectKey{Namespace: "team", Name: "missing"} + newClient := func() client.Client { + return fake.NewClientBuilder().WithScheme(scheme).Build() + } + forcedError := errors.New("forced get failure") + + tests := []struct { + kind string + reconcile func(client.Client) error + }{ + { + kind: metricKindClusterHost, + reconcile: func(c client.Client) error { + _, err := (&ClusterHostReconciler{Client: c, Scheme: scheme}).Reconcile(t.Context(), requestFor(missingKey)) + return err + }, + }, + { + kind: metricKindWorkspace, + reconcile: func(c client.Client) error { + _, err := (&WorkspaceReconciler{Client: c, Scheme: scheme}).Reconcile(t.Context(), requestFor(missingKey)) + return err + }, + }, + { + kind: metricKindSession, + reconcile: func(c client.Client) error { + _, err := (&SessionReconciler{Client: c, Scheme: scheme}).Reconcile(t.Context(), requestFor(missingKey)) + return err + }, + }, + } + + for _, test := range tests { + t.Run(test.kind, func(t *testing.T) { + successLabels := map[string]string{"kind": test.kind, "result": "success"} + errorLabels := map[string]string{"kind": test.kind, "result": "error"} + beforeSuccess, _ := gatheredMetricValue(t, "t4_cluster_reconcile_total", successLabels) + beforeError, _ := gatheredMetricValue(t, "t4_cluster_reconcile_total", errorLabels) + + if err := test.reconcile(newClient()); err != nil { + t.Fatalf("not-found reconcile returned an error: %v", err) + } + if err := test.reconcile(&getErrorClient{Client: newClient(), err: forcedError}); !errors.Is(err, forcedError) { + t.Fatalf("error reconcile = %v, want %v", err, forcedError) + } + + afterSuccess, ok := gatheredMetricValue(t, "t4_cluster_reconcile_total", successLabels) + if !ok || afterSuccess-beforeSuccess != 1 { + t.Fatalf("success counter delta = %v, want 1", afterSuccess-beforeSuccess) + } + afterError, ok := gatheredMetricValue(t, "t4_cluster_reconcile_total", errorLabels) + if !ok || afterError-beforeError != 1 { + t.Fatalf("error counter delta = %v, want 1", afterError-beforeError) + } + }) + } +} + +func TestConditionMetricProjectsKnownTransitionsWithBoundedLabels(t *testing.T) { + scheme := metricsTestScheme(t) + host := &clusterv1alpha1.T4ClusterHost{ + ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team", Generation: 1}, + Spec: clusterv1alpha1.T4ClusterHostSpec{ + StorageClassName: "portable-rwx", + RuntimeProfiles: []string{"default"}, + }, + Status: clusterv1alpha1.T4ClusterHostStatus{Conditions: []metav1.Condition{{ + Type: "owner-team-a-secret-token", Status: metav1.ConditionTrue, Reason: "Untrusted", Message: "must not become a metric label", + }}}, + } + hostB := host.DeepCopy() + hostB.Name = "host-b" + hostB.Status.Conditions = nil + storageClass := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "portable-rwx", + Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}, + }, + Provisioner: "example.invalid/csi", + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4ClusterHost{}).WithObjects(host, hostB, storageClass).Build() + r := &ClusterHostReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(t.Context(), requestFor(client.ObjectKeyFromObject(host))); err != nil { + t.Fatal(err) + } + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindClusterHost, "condition": "StorageReady", "status": "true"}, 1) + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindClusterHost, "condition": "StorageReady", "status": "false"}, 0) + + var updatedStorageClass storagev1.StorageClass + if err := c.Get(t.Context(), client.ObjectKeyFromObject(storageClass), &updatedStorageClass); err != nil { + t.Fatal(err) + } + updatedStorageClass.Annotations = nil + if err := c.Update(t.Context(), &updatedStorageClass); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(t.Context(), requestFor(client.ObjectKeyFromObject(host))); err != nil { + t.Fatal(err) + } + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindClusterHost, "condition": "StorageReady", "status": "true"}, 0) + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindClusterHost, "condition": "StorageReady", "status": "false"}, 1) + updatedStorageClass.Annotations = map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)} + if err := c.Update(t.Context(), &updatedStorageClass); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(t.Context(), requestFor(client.ObjectKeyFromObject(hostB))); err != nil { + t.Fatal(err) + } + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindClusterHost, "condition": "StorageReady", "status": "true"}, 1) + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindClusterHost, "condition": "StorageReady", "status": "false"}, 1) + + var deletingHost clusterv1alpha1.T4ClusterHost + if err := c.Get(t.Context(), client.ObjectKeyFromObject(host), &deletingHost); err != nil { + t.Fatal(err) + } + if err := c.Delete(t.Context(), &deletingHost); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(t.Context(), requestFor(client.ObjectKeyFromObject(host))); err != nil { + t.Fatal(err) + } + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindClusterHost, "condition": "StorageReady", "status": "true"}, 1) + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindClusterHost, "condition": "StorageReady", "status": "false"}, 0) + + if err := c.Get(t.Context(), client.ObjectKeyFromObject(hostB), &deletingHost); err != nil { + t.Fatal(err) + } + if err := c.Delete(t.Context(), &deletingHost); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(t.Context(), requestFor(client.ObjectKeyFromObject(hostB))); err != nil { + t.Fatal(err) + } + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindClusterHost, "condition": "StorageReady", "status": "true"}, 0) + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindClusterHost, "condition": "StorageReady", "status": "false"}, 0) + + assertMetricLabelsAreBounded(t) +} + +func TestDeletionReconcilesProjectTerminatingConditions(t *testing.T) { + scheme := metricsTestScheme(t) + deletionTime := metav1.Now() + + workspace := &clusterv1alpha1.T4Workspace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "workspace-a", Namespace: "team", UID: "workspace-uid", + DeletionTimestamp: &deletionTime, Finalizers: []string{clusterv1alpha1.WorkspaceFinalizer}, + }, + Spec: clusterv1alpha1.T4WorkspaceSpec{RetentionPolicy: clusterv1alpha1.RetentionPolicyDelete}, + } + blockingSession := &clusterv1alpha1.T4Session{ + ObjectMeta: metav1.ObjectMeta{Name: "blocking-session", Namespace: workspace.Namespace}, + Spec: clusterv1alpha1.T4SessionSpec{WorkspaceRef: workspace.Name}, + } + workspaceClient := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, blockingSession).Build() + workspaceReconciler := &WorkspaceReconciler{Client: workspaceClient, Scheme: scheme} + if _, err := workspaceReconciler.Reconcile(t.Context(), requestFor(client.ObjectKeyFromObject(workspace))); err != nil { + t.Fatal(err) + } + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindWorkspace, "condition": "Ready", "status": "true"}, 0) + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindWorkspace, "condition": "Ready", "status": "false"}, 1) + if err := workspaceClient.Delete(t.Context(), blockingSession); err != nil { + t.Fatal(err) + } + if _, err := workspaceReconciler.Reconcile(t.Context(), requestFor(client.ObjectKeyFromObject(workspace))); err != nil { + t.Fatal(err) + } + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindWorkspace, "condition": "Ready", "status": "true"}, 0) + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindWorkspace, "condition": "Ready", "status": "false"}, 0) + + session := &clusterv1alpha1.T4Session{ + ObjectMeta: metav1.ObjectMeta{ + Name: "session-a", Namespace: "team", UID: "session-uid", + DeletionTimestamp: &deletionTime, Finalizers: []string{clusterv1alpha1.SessionFinalizer}, + }, + } + controller := true + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: SessionPodName(session), Namespace: session.Namespace, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Session", Name: session.Name, UID: session.UID, Controller: &controller, + }}, + }} + sessionClient := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod).Build() + sessionReconciler := &SessionReconciler{Client: sessionClient, Scheme: scheme} + if _, err := sessionReconciler.Reconcile(t.Context(), requestFor(client.ObjectKeyFromObject(session))); err != nil { + t.Fatal(err) + } + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindSession, "condition": "Available", "status": "true"}, 0) + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindSession, "condition": "Available", "status": "false"}, 1) + if _, err := sessionReconciler.Reconcile(t.Context(), requestFor(client.ObjectKeyFromObject(session))); err != nil { + t.Fatal(err) + } + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindSession, "condition": "Available", "status": "true"}, 0) + assertMetricValue(t, "t4_cluster_condition", map[string]string{"kind": metricKindSession, "condition": "Available", "status": "false"}, 0) +} + +func TestControllerMetricRegistrationIsConcurrentAndIdempotent(t *testing.T) { + var callers sync.WaitGroup + for range 32 { + callers.Add(1) + go func() { + defer callers.Done() + registerControllerMetrics() + }() + } + callers.Wait() + + families, err := controllermetrics.Registry.Gather() + if err != nil { + t.Fatal(err) + } + counts := map[string]int{} + for _, family := range families { + counts[family.GetName()]++ + } + for _, name := range []string{"t4_cluster_reconcile_total", "t4_cluster_condition"} { + if counts[name] != 1 { + t.Fatalf("registered metric family %q count = %d, want 1", name, counts[name]) + } + } +} + +type getErrorClient struct { + client.Client + err error +} + +func (c *getErrorClient) Get(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error { + return c.err +} + +func metricsTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, storagev1.AddToScheme, clusterv1alpha1.AddToScheme} { + if err := add(scheme); err != nil { + t.Fatal(err) + } + } + return scheme +} + +func requestFor(key client.ObjectKey) ctrl.Request { + return ctrl.Request{NamespacedName: key} +} + +func assertMetricValue(t *testing.T, name string, labels map[string]string, want float64) { + t.Helper() + got, ok := gatheredMetricValue(t, name, labels) + if !ok || got != want { + t.Fatalf("%s%v = %v (present %t), want %v", name, labels, got, ok, want) + } +} + +func gatheredMetricValue(t *testing.T, name string, labels map[string]string) (float64, bool) { + t.Helper() + families, err := controllermetrics.Registry.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + if family.GetName() != name { + continue + } + for _, metric := range family.GetMetric() { + metricLabels := make(map[string]string, len(metric.GetLabel())) + for _, pair := range metric.GetLabel() { + metricLabels[pair.GetName()] = pair.GetValue() + } + if !reflect.DeepEqual(metricLabels, labels) { + continue + } + if metric.Counter != nil { + return metric.GetCounter().GetValue(), true + } + if metric.Gauge != nil { + return metric.GetGauge().GetValue(), true + } + } + } + return 0, false +} + +func assertMetricLabelsAreBounded(t *testing.T) { + t.Helper() + families, err := controllermetrics.Registry.Gather() + if err != nil { + t.Fatal(err) + } + allowedConditions := map[string]map[string]bool{ + metricKindClusterHost: {"Available": true, "CIReady": true, "StorageReady": true}, + metricKindWorkspace: {"HostReady": true, "Ready": true, "StorageReady": true}, + metricKindSession: {"Available": true, "HostReady": true, "RuntimeConfigured": true, "WorkspaceReady": true}, + } + for _, family := range families { + if family.GetName() != "t4_cluster_reconcile_total" && family.GetName() != "t4_cluster_condition" { + continue + } + for _, metric := range family.GetMetric() { + labels := make(map[string]string, len(metric.GetLabel())) + keys := make([]string, 0, len(metric.GetLabel())) + for _, pair := range metric.GetLabel() { + labels[pair.GetName()] = pair.GetValue() + keys = append(keys, pair.GetName()) + } + sort.Strings(keys) + if family.GetName() == "t4_cluster_reconcile_total" { + if !reflect.DeepEqual(keys, []string{"kind", "result"}) { + t.Fatalf("reconcile labels = %v, want only kind/result", keys) + } + if allowedConditions[labels["kind"]] == nil || labels["result"] != "success" && labels["result"] != "error" { + t.Fatalf("unbounded reconcile label values: %v", labels) + } + continue + } + if !reflect.DeepEqual(keys, []string{"condition", "kind", "status"}) { + t.Fatalf("condition labels = %v, want only kind/condition/status", keys) + } + if !allowedConditions[labels["kind"]][labels["condition"]] || labels["status"] != "true" && labels["status"] != "false" { + t.Fatalf("unbounded condition label values: %v", labels) + } + } + } +} diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go new file mode 100644 index 00000000..1f0da613 --- /dev/null +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -0,0 +1,1271 @@ +package controllers_test + +import ( + "context" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apiresource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilvalidation "k8s.io/apimachinery/pkg/util/validation" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" + "github.com/LycaonLLC/t4-code/packages/cluster-operator/controllers" +) + +const ( + testRuntimeImage = "registry.example/session@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + otherTestRuntimeImage = "registry.example/session@sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210" +) + +func TestWorkspaceReconcileIsIdempotentAcrossDuplicateEvents(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4ClusterHost{}, &clusterv1alpha1.T4Workspace{}, &clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), rwxStorageClass(), workspace).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + reconcileMany(t, 4, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) + return err + }) + + var pvcs corev1.PersistentVolumeClaimList + if err := c.List(context.Background(), &pvcs, client.InNamespace("team")); err != nil { + t.Fatal(err) + } + if len(pvcs.Items) != 1 { + t.Fatalf("duplicate events created %d PVCs, want 1", len(pvcs.Items)) + } + pvc := pvcs.Items[0] + if len(pvc.Spec.AccessModes) != 1 || pvc.Spec.AccessModes[0] != corev1.ReadWriteMany { + t.Fatalf("PVC access modes = %v, want only ReadWriteMany", pvc.Spec.AccessModes) + } + if pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName != "portable-rwx" { + t.Fatalf("PVC storage class = %v", pvc.Spec.StorageClassName) + } + + var got clusterv1alpha1.T4Workspace + if err := c.Get(context.Background(), client.ObjectKeyFromObject(workspace), &got); err != nil { + t.Fatal(err) + } + if got.Status.ObservedGeneration != got.Generation || got.Status.PVCName != pvc.Name { + t.Fatalf("workspace status not converged: %#v", got.Status) + } + if !contains(got.Finalizers, clusterv1alpha1.WorkspaceFinalizer) { + t.Fatal("workspace protection finalizer missing") + } +} + +func TestRetainWorkspaceCreatesPVCWithoutGarbageCollectableOwner(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) + workspace.UID = "workspace-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), rwxStorageClass(), workspace).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) + return err + }) + var pvc corev1.PersistentVolumeClaim + if err := c.Get(context.Background(), types.NamespacedName{Namespace: workspace.Namespace, Name: controllers.WorkspacePVCName(workspace)}, &pvc); err != nil { + t.Fatal(err) + } + if len(pvc.OwnerReferences) != 0 || pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] != string(workspace.UID) { + t.Fatalf("retained PVC is exposed to owner garbage collection: %#v", pvc.ObjectMeta) + } +} + +func TestWorkspaceStorageFailsClosedWhenClassMissingOrNotRWX(t *testing.T) { + for _, test := range []struct { + name string + class *storagev1.StorageClass + reason string + }{ + {name: "missing", reason: controllers.ReasonStorageClassNotFound}, + {name: "not-rwx", class: &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "portable-rwx"}, Provisioner: "example.invalid/csi"}, reason: controllers.ReasonStorageClassNotRWX}, + } { + t.Run(test.name, func(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + workspace.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue, Reason: "PVCBound", ObservedGeneration: workspace.Generation}} + objects := []client.Object{testHost(), workspace} + if test.class != nil { + objects = append(objects, test.class) + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(objects...).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "team", Name: "workspace-a"}}) + return err + }) + var pvcs corev1.PersistentVolumeClaimList + if err := c.List(context.Background(), &pvcs, client.InNamespace("team")); err != nil { + t.Fatal(err) + } + if len(pvcs.Items) != 0 { + t.Fatalf("fail-closed path created %d PVCs", len(pvcs.Items)) + } + var got clusterv1alpha1.T4Workspace + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: "workspace-a"}, &got); err != nil { + t.Fatal(err) + } + condition := findCondition(got.Status.Conditions, "StorageReady") + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != test.reason { + t.Fatalf("StorageReady = %#v, want False/%s", condition, test.reason) + } + ready := findCondition(got.Status.Conditions, "Ready") + if got.Status.Phase != clusterv1alpha1.InfrastructureFailed || ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != test.reason { + t.Fatalf("revoked workspace status = %#v, Ready = %#v, want Failed and False/%s", got.Status, ready, test.reason) + } + }) + } +} + +func TestRetainDeletionOrphansPVCBeforeRemovingFinalizer(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) + workspace.UID = "workspace-uid" + workspace.Finalizers = []string{clusterv1alpha1.WorkspaceFinalizer} + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllers.WorkspacePVCName(workspace), + Namespace: workspace.Namespace, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true), + }}, + }, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(testHost(), rwxStorageClass(), workspace, pvc).Build() + if err := c.Delete(context.Background(), workspace); err != nil { + t.Fatal(err) + } + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) + return err + }) + var retained corev1.PersistentVolumeClaim + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pvc), &retained); err != nil { + t.Fatalf("retained PVC was deleted: %v", err) + } + if len(retained.OwnerReferences) != 0 || retained.Annotations[clusterv1alpha1.RetainedPVCAnnotation] != "true" { + t.Fatalf("retained PVC was not orphaned safely: %#v", retained.ObjectMeta) + } + var gone clusterv1alpha1.T4Workspace + if err := c.Get(context.Background(), client.ObjectKeyFromObject(workspace), &gone); !apierrors.IsNotFound(err) { + t.Fatalf("workspace should be deleted after retention, got %v", err) + } +} + +func TestWorkspaceDeletionWaitsForSessionResources(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) + workspace.UID = "workspace-uid" + workspace.Finalizers = []string{clusterv1alpha1.WorkspaceFinalizer} + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllers.WorkspacePVCName(workspace), Namespace: workspace.Namespace, + OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, + }, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + } + session := testSession() + session.Spec.WorkspaceRef = workspace.Name + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc, session).Build() + if err := c.Delete(context.Background(), workspace); err != nil { + t.Fatal(err) + } + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var waiting clusterv1alpha1.T4Workspace + if err := c.Get(context.Background(), client.ObjectKeyFromObject(workspace), &waiting); err != nil { + t.Fatalf("workspace deletion did not wait: %v", err) + } + condition := findCondition(waiting.Status.Conditions, "Ready") + if condition == nil || condition.Reason != "SessionsRemain" { + t.Fatalf("Ready = %#v, want SessionsRemain", condition) + } + var retained corev1.PersistentVolumeClaim + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pvc), &retained); err != nil { + t.Fatalf("workspace PVC changed during wait: %v", err) + } + if len(retained.OwnerReferences) != 1 { + t.Fatalf("workspace PVC was orphaned before sessions exited: %#v", retained.OwnerReferences) + } +} + +func TestSessionFailsClosedWhenAnyOMPReferenceIsMissing(t *testing.T) { + for _, test := range []struct { + name string + remove func(*controllers.SessionOMPConfig) + reason string + }{ + {name: "ConfigMap", remove: func(config *controllers.SessionOMPConfig) { config.ConfigMapName = "" }, reason: "OMPReferencesMissing"}, + {name: "models key", remove: func(config *controllers.SessionOMPConfig) { config.ModelsKey = "" }, reason: "OMPReferencesMissing"}, + {name: "settings key", remove: func(config *controllers.SessionOMPConfig) { config.SettingsKey = "" }, reason: "OMPReferencesMissing"}, + {name: "strict credentials", remove: func(config *controllers.SessionOMPConfig) { config.CredentialSecretName, config.CredentialKey = "", "" }, reason: "OMPReferencesMissing"}, + } { + t.Run(test.name, func(t *testing.T) { + scheme := testScheme(t) + session := testSession() + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session).Build() + r := configuredSessionReconciler(c, scheme) + test.remove(&r.OMPConfig) + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, c, 0, 0) + var got clusterv1alpha1.T4Session + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + condition := findCondition(got.Status.Conditions, "RuntimeConfigured") + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != test.reason { + t.Fatalf("RuntimeConfigured = %#v, want False/%s", condition, test.reason) + } + }) + } +} + +func TestSessionRejectsInvalidOMPAuthenticationAndProjectionReferences(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*controllers.SessionOMPConfig) + }{ + {name: "credential Secret without key", mutate: func(config *controllers.SessionOMPConfig) { config.CredentialKey = "" }}, + {name: "credential key without Secret", mutate: func(config *controllers.SessionOMPConfig) { config.CredentialSecretName = "" }}, + {name: "unauthenticated plus credentials", mutate: func(config *controllers.SessionOMPConfig) { config.AllowUnauthenticated = true }}, + {name: "invalid credential environment name", mutate: func(config *controllers.SessionOMPConfig) { config.CredentialKey = "bad-key" }}, + {name: "runtime-owned credential environment", mutate: func(config *controllers.SessionOMPConfig) { config.CredentialKey = "OMP_PROFILE" }}, + {name: "PI runtime-owned credential environment", mutate: func(config *controllers.SessionOMPConfig) { config.CredentialKey = "PI_PROFILE" }}, + {name: "identical projected keys", mutate: func(config *controllers.SessionOMPConfig) { config.SettingsKey = config.ModelsKey }}, + } { + t.Run(test.name, func(t *testing.T) { + scheme := testScheme(t) + session := testSession() + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session).Build() + r := configuredSessionReconciler(c, scheme) + test.mutate(&r.OMPConfig) + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, c, 0, 0) + var got clusterv1alpha1.T4Session + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + condition := findCondition(got.Status.Conditions, "RuntimeConfigured") + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != "OMPReferencesInvalid" { + t.Fatalf("RuntimeConfigured = %#v, want False/OMPReferencesInvalid", condition) + } + }) + } +} + +func TestSessionRuntimeImageMustBeImmutableDigest(t *testing.T) { + for _, test := range []struct { + name string + image string + wantReason string + }{ + {name: "tag only", image: "registry.example/session:latest", wantReason: "RuntimeImageInvalid"}, + {name: "malformed digest", image: "registry.example/session@sha256:deadbeef", wantReason: "RuntimeImageInvalid"}, + {name: "uppercase algorithm", image: "registry.example/session@SHA256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", wantReason: "RuntimeImageInvalid"}, + {name: "uppercase digest", image: "registry.example/session@sha256:ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789", wantReason: "RuntimeImageInvalid"}, + {name: "registry port and path", image: "registry.example:5443/team/session-runtime@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, + } { + t.Run(test.name, func(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + r.RuntimeImage = test.image + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if test.wantReason == "" { + assertObjectCounts(t, c, 1, 1) + var pod corev1.Pod + if err := c.Get(context.Background(), types.NamespacedName{Namespace: session.Namespace, Name: controllers.SessionPodName(session)}, &pod); err != nil { + t.Fatal(err) + } + if pod.Spec.Containers[0].Image != test.image { + t.Fatalf("runtime image = %q, want %q", pod.Spec.Containers[0].Image, test.image) + } + return + } + assertObjectCounts(t, c, 0, 0) + var got clusterv1alpha1.T4Session + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + condition := findCondition(got.Status.Conditions, "RuntimeConfigured") + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != test.wantReason { + t.Fatalf("RuntimeConfigured = %#v, want False/%s", condition, test.wantReason) + } + }) + } +} + +func TestSessionAuthorityRevocationDeletesOwnedPodAndService(t *testing.T) { + for _, test := range []struct { + name string + conditionType string + wantReason string + revoke func(context.Context, client.Client, *controllers.SessionReconciler) error + }{ + {name: "runtime image", conditionType: "RuntimeConfigured", wantReason: "RuntimeImageInvalid", revoke: func(_ context.Context, _ client.Client, r *controllers.SessionReconciler) error { + r.RuntimeImage = "registry.example/session:latest" + return nil + }}, + {name: "runtime profile", conditionType: "RuntimeConfigured", wantReason: "RuntimeProfileNotAllowed", revoke: func(ctx context.Context, c client.Client, _ *controllers.SessionReconciler) error { + var host clusterv1alpha1.T4ClusterHost + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "host-a"}, &host); err != nil { + return err + } + host.Spec.RuntimeProfiles = nil + return c.Update(ctx, &host) + }}, + {name: "storage declaration", conditionType: "WorkspaceReady", wantReason: controllers.ReasonStorageClassNotRWX, revoke: func(ctx context.Context, c client.Client, _ *controllers.SessionReconciler) error { + var storageClass storagev1.StorageClass + if err := c.Get(ctx, types.NamespacedName{Name: "portable-rwx"}, &storageClass); err != nil { + return err + } + storageClass.Annotations = nil + return c.Update(ctx, &storageClass) + }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + assertObjectCounts(t, c, 1, 1) + if err := test.revoke(ctx, c, r); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, c, 0, 0) + var got clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + condition := findCondition(got.Status.Conditions, test.conditionType) + available := findCondition(got.Status.Conditions, "Available") + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != test.wantReason || + available == nil || available.Status != metav1.ConditionFalse || available.Reason != test.wantReason || + got.Status.PodName != "" || got.Status.ServiceName != "" { + t.Fatalf("revoked session status = %#v, condition = %#v, available = %#v", got.Status, condition, available) + } + }) + } +} + +func TestSessionNamesProduceSafeRuntimeIdentities(t *testing.T) { + for _, sessionName := range []string{ + "release.2026.07.21", + "session-with-a-very-long-name-that-exceeds-sixty-three-characters-and-remains-valid", + } { + t.Run(sessionName, func(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.Name = sessionName + session.UID = types.UID("uid-" + sessionName) + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var pod corev1.Pod + if err := c.Get(context.Background(), types.NamespacedName{Namespace: session.Namespace, Name: controllers.SessionPodName(session)}, &pod); err != nil { + t.Fatal(err) + } + var service corev1.Service + if err := c.Get(context.Background(), types.NamespacedName{Namespace: session.Namespace, Name: controllers.SessionServiceName(session)}, &service); err != nil { + t.Fatal(err) + } + for kind, name := range map[string]string{"Pod": pod.Name, "Service": service.Name} { + if len(name) > 63 || len(utilvalidation.IsDNS1123Label(name)) != 0 { + t.Fatalf("%s name %q is not a DNS label", kind, name) + } + } + values := map[string]string{} + for _, env := range pod.Spec.Containers[0].Env { + values[env.Name] = env.Value + } + stateID := strings.TrimPrefix(pod.Name, "t4-session-") + if len(stateID) > 63 || len(utilvalidation.IsDNS1123Label(stateID)) != 0 || values["T4_SESSION_NAME"] != stateID || values["T4_SESSION_STATE_ID"] != stateID { + t.Fatalf("runtime identity = name %q state %q, want safe state ID %q", values["T4_SESSION_NAME"], values["T4_SESSION_STATE_ID"], stateID) + } + }) + } +} + +func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + workspace.Status.Phase = clusterv1alpha1.InfrastructurePending + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending}, + } + session := testSession() + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + r.RuntimeImage = testRuntimeImage + r.KubernetesAPIAudience = "kubernetes.custom.example" + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + assertObjectCounts(t, c, 0, 0) + + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pvc), pvc); err != nil { + t.Fatal(err) + } + pvc.Status.Phase = corev1.ClaimBound + pvc.Status.Capacity = corev1.ResourceList{corev1.ResourceStorage: apiresource.MustParse("10Gi")} + if err := c.Status().Update(context.Background(), pvc); err != nil { + t.Fatal(err) + } + reconcileMany(t, 4, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + assertObjectCounts(t, c, 1, 1) + + var pods corev1.PodList + if err := c.List(context.Background(), &pods, client.InNamespace("team")); err != nil { + t.Fatal(err) + } + pod := pods.Items[0] + if pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { + t.Fatal("session pod must disable automatic ServiceAccount token mounting") + } + if pod.Spec.Containers[0].Image != r.RuntimeImage { + t.Fatalf("controller did not use administrator-owned runtime image: %q", pod.Spec.Containers[0].Image) + } + if pod.Spec.Containers[0].SecurityContext == nil || pod.Spec.Containers[0].SecurityContext.Privileged != nil && *pod.Spec.Containers[0].SecurityContext.Privileged { + t.Fatal("session runtime is not restricted") + } + if !hasMount(pod.Spec.Containers[0].VolumeMounts, "workspace", "/workspace") || + !hasMount(pod.Spec.Containers[0].VolumeMounts, "shared-memory", "/dev/shm") || + !hasReadOnlyMount(pod.Spec.Containers[0].VolumeMounts, "omp-config-source", "/run/t4-omp-config-source") { + t.Fatalf("session mounts = %#v", pod.Spec.Containers[0].VolumeMounts) + } + if pod.Spec.ServiceAccountName != controllers.DefaultSessionServiceAccount { + t.Fatalf("session ServiceAccount = %q", pod.Spec.ServiceAccountName) + } + serverIdentity := "" + var credential *corev1.EnvVar + configSource := "" + allowUnauthenticated := "" + for i := range pod.Spec.Containers[0].Env { + env := &pod.Spec.Containers[0].Env[i] + if env.Name == "T4_CLUSTER_SERVER_SERVICE_ACCOUNT" { + serverIdentity = env.Value + } + if env.Name == "T4_OMP_CONFIG_SOURCE_DIR" { + configSource = env.Value + } + if env.Name == "T4_OMP_ALLOW_UNAUTHENTICATED" { + allowUnauthenticated = env.Value + } + if env.Name == r.OMPConfig.CredentialKey { + credential = env + } + } + if serverIdentity != controllers.DefaultServerServiceAccount { + t.Fatalf("expected server ServiceAccount = %q", serverIdentity) + } + if configSource != "/run/t4-omp-config-source" || allowUnauthenticated != "false" { + t.Fatalf("OMP preflight environment = source %q allowUnauthenticated %q", configSource, allowUnauthenticated) + } + if len(pod.Spec.Containers[0].Args) != 1 || pod.Spec.Containers[0].Args[0] != r.OMPConfig.CredentialKey { + t.Fatalf("credential key argument = %#v, want %q", pod.Spec.Containers[0].Args, r.OMPConfig.CredentialKey) + } + if credential == nil || credential.Value != "" || credential.ValueFrom == nil || credential.ValueFrom.SecretKeyRef == nil { + t.Fatalf("credential environment reference = %#v", credential) + } + secretRef := credential.ValueFrom.SecretKeyRef + if credential.Name != r.OMPConfig.CredentialKey || secretRef.Name != r.OMPConfig.CredentialSecretName || secretRef.Key != r.OMPConfig.CredentialKey || secretRef.Optional == nil || *secretRef.Optional { + t.Fatalf("credential SecretKeyRef is not exact and non-optional: %#v", credential) + } + apiAudience := "" + for _, env := range pod.Spec.Containers[0].Env { + if env.Name == "T4_KUBERNETES_API_AUDIENCE" { + apiAudience = env.Value + } + } + if apiAudience != r.KubernetesAPIAudience { + t.Fatalf("reviewer API audience environment = %q", apiAudience) + } + if !hasMount(pod.Spec.Containers[0].VolumeMounts, "kubernetes-api-access", "/var/run/secrets/kubernetes.io/serviceaccount") { + t.Fatal("explicit Kubernetes reviewer projection is not mounted") + } + var projection *corev1.ProjectedVolumeSource + for _, volume := range pod.Spec.Volumes { + if volume.Name == "kubernetes-api-access" { + projection = volume.Projected + } + } + if projection == nil || len(projection.Sources) != 3 { + t.Fatalf("Kubernetes reviewer projection = %#v", projection) + } + serviceToken := projection.Sources[0].ServiceAccountToken + if serviceToken == nil || serviceToken.Audience != r.KubernetesAPIAudience || serviceToken.ExpirationSeconds == nil || *serviceToken.ExpirationSeconds != controllers.SessionReviewerTokenExpirationSeconds || serviceToken.Path != "token" { + t.Fatalf("reviewer token projection = %#v", serviceToken) + } + clusterCA := projection.Sources[1].ConfigMap + if clusterCA == nil || clusterCA.Name != "kube-root-ca.crt" || len(clusterCA.Items) != 1 || clusterCA.Items[0].Key != "ca.crt" || clusterCA.Items[0].Path != "ca.crt" { + t.Fatalf("cluster CA projection = %#v", clusterCA) + } + namespace := projection.Sources[2].DownwardAPI + if namespace == nil || len(namespace.Items) != 1 || namespace.Items[0].Path != "namespace" || namespace.Items[0].FieldRef == nil || namespace.Items[0].FieldRef.FieldPath != "metadata.namespace" { + t.Fatalf("namespace projection = %#v", namespace) + } + var ompConfig *corev1.ConfigMapVolumeSource + for i := range pod.Spec.Volumes { + if pod.Spec.Volumes[i].Name == "omp-config-source" { + ompConfig = pod.Spec.Volumes[i].ConfigMap + } + } + if ompConfig == nil || ompConfig.Name != r.OMPConfig.ConfigMapName || ompConfig.Optional == nil || *ompConfig.Optional || ompConfig.DefaultMode == nil || *ompConfig.DefaultMode != 0440 || len(ompConfig.Items) != 2 { + t.Fatalf("OMP ConfigMap projection = %#v", ompConfig) + } + if got := ompConfig.Items[0]; got.Key != r.OMPConfig.ModelsKey || got.Path != "models.yml" || got.Mode == nil || *got.Mode != 0440 { + t.Fatalf("OMP models projection = %#v", got) + } + if got := ompConfig.Items[1]; got.Key != r.OMPConfig.SettingsKey || got.Path != "config.yml" || got.Mode == nil || *got.Mode != 0440 { + t.Fatalf("OMP settings projection = %#v", got) + } +} + +func TestSessionUnauthenticatedOMPModeOmitsCredentialSecretReference(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + r.OMPConfig.AllowUnauthenticated = true + r.OMPConfig.CredentialSecretName = "" + r.OMPConfig.CredentialKey = "" + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + var pod corev1.Pod + if err := c.Get(context.Background(), types.NamespacedName{Namespace: session.Namespace, Name: controllers.SessionPodName(session)}, &pod); err != nil { + t.Fatal(err) + } + allowUnauthenticated := "" + for _, env := range pod.Spec.Containers[0].Env { + if env.Name == "T4_OMP_ALLOW_UNAUTHENTICATED" { + allowUnauthenticated = env.Value + } + if len(pod.Spec.Containers[0].Args) != 0 || env.ValueFrom != nil && env.ValueFrom.SecretKeyRef != nil { + t.Fatalf("unauthenticated OMP mode retained a credential reference: %#v", env) + } + } + if allowUnauthenticated != "true" { + t.Fatalf("T4_OMP_ALLOW_UNAUTHENTICATED = %q, want true", allowUnauthenticated) + } +} + +func TestSessionRejectsUnownedDeterministicResources(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: "team"}} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), workspace, pvc, session, service).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var got clusterv1alpha1.T4Session + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + condition := findCondition(got.Status.Conditions, "Available") + if condition == nil || condition.Reason != "ServiceOwnershipConflict" || got.Status.Phase != clusterv1alpha1.InfrastructureFailed { + t.Fatalf("collision status = %#v/%q", condition, got.Status.Phase) + } + var pods corev1.PodList + if err := c.List(context.Background(), &pods, client.InNamespace("team")); err != nil { + t.Fatal(err) + } + if len(pods.Items) != 0 { + t.Fatalf("collision created %d pods", len(pods.Items)) + } +} + +func TestSessionRecreatesPodWhenImmutableDesiredStateChanges(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + r.RuntimeImage = testRuntimeImage + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + var original corev1.Pod + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)}, &original); err != nil { + t.Fatal(err) + } + originalHash := original.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] + r.RuntimeImage = otherTestRuntimeImage + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var deleted corev1.Pod + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)}, &deleted); !apierrors.IsNotFound(err) { + t.Fatalf("outdated pod remains after immutable desired state changed: %v", err) + } + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + var replacement corev1.Pod + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)}, &replacement); err != nil { + t.Fatal(err) + } + if replacement.Spec.Containers[0].Image != r.RuntimeImage || replacement.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] == originalHash { + t.Fatalf("replacement pod did not converge: image=%q annotations=%#v", replacement.Spec.Containers[0].Image, replacement.Annotations) + } +} + +func TestSessionPodHashIncludesEveryOMPReference(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*controllers.SessionOMPConfig) + }{ + {name: "ConfigMap", mutate: func(config *controllers.SessionOMPConfig) { config.ConfigMapName = "other-omp-config" }}, + {name: "models key", mutate: func(config *controllers.SessionOMPConfig) { config.ModelsKey = "other-models" }}, + {name: "settings key", mutate: func(config *controllers.SessionOMPConfig) { config.SettingsKey = "other-settings" }}, + {name: "credential Secret", mutate: func(config *controllers.SessionOMPConfig) { config.CredentialSecretName = "other-credential" }}, + {name: "credential key", mutate: func(config *controllers.SessionOMPConfig) { config.CredentialKey = "OTHER_MODEL_API_KEY" }}, + {name: "authentication mode", mutate: func(config *controllers.SessionOMPConfig) { + config.AllowUnauthenticated = true + config.CredentialSecretName = "" + config.CredentialKey = "" + }}, + } { + t.Run(test.name, func(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + test.mutate(&r.OMPConfig) + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var pod corev1.Pod + if err := c.Get(context.Background(), types.NamespacedName{Namespace: session.Namespace, Name: controllers.SessionPodName(session)}, &pod); !apierrors.IsNotFound(err) { + t.Fatalf("pod hash ignored changed %s reference: %v", test.name, err) + } + }) + } +} +func TestSessionRecreatesPodWhenOMPResourceVersionChanges(t *testing.T) { + for _, test := range []struct { + name string + mutate func(context.Context, client.Client) error + }{ + {name: "ConfigMap", mutate: func(ctx context.Context, c client.Client) error { + var configMap corev1.ConfigMap + key := types.NamespacedName{Namespace: "team", Name: "omp-runtime-config"} + if err := c.Get(ctx, key, &configMap); err != nil { + return err + } + configMap.Data["provider-models"] = "changed models" + return c.Update(ctx, &configMap) + }}, + {name: "Secret", mutate: func(ctx context.Context, c client.Client) error { + var secret corev1.Secret + key := types.NamespacedName{Namespace: "team", Name: "omp-runtime-credential"} + if err := c.Get(ctx, key, &secret); err != nil { + return err + } + secret.Data["MODEL_API_KEY"] = []byte("rotated credential") + return c.Update(ctx, &secret) + }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + if err := test.mutate(ctx, c); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var pod corev1.Pod + if err := c.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: controllers.SessionPodName(session)}, &pod); !apierrors.IsNotFound(err) { + t.Fatalf("pod retained stale %s resourceVersion: %v", test.name, err) + } + }) + } +} +func TestSessionRuntimeReferenceRevocationStopsAuthority(t *testing.T) { + for _, test := range []struct { + name string + revoke func(context.Context, client.Client) error + wantReason string + }{ + {name: "ConfigMap deletion", wantReason: "OMPConfigMapNotFound", revoke: func(ctx context.Context, c client.Client) error { + return c.Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}}) + }}, + {name: "credential removal", wantReason: "OMPCredentialSecretInvalid", revoke: func(ctx context.Context, c client.Client) error { + var secret corev1.Secret + key := types.NamespacedName{Namespace: "team", Name: "omp-runtime-credential"} + if err := c.Get(ctx, key, &secret); err != nil { + return err + } + delete(secret.Data, "MODEL_API_KEY") + return c.Update(ctx, &secret) + }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + var running clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &running); err != nil { + t.Fatal(err) + } + if running.Status.PodName == "" || running.Status.ServiceName == "" { + t.Fatalf("running route was not published: %#v", running.Status) + } + if err := test.revoke(ctx, c); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var pod corev1.Pod + if err := c.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: controllers.SessionPodName(session)}, &pod); !apierrors.IsNotFound(err) { + t.Fatalf("revoked runtime retained authority pod: %v", err) + } + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + condition := findCondition(failed.Status.Conditions, "RuntimeConfigured") + if failed.Status.PodName != "" || failed.Status.ServiceName != "" || failed.Status.Phase != clusterv1alpha1.InfrastructureFailed || condition == nil || condition.Reason != test.wantReason { + t.Fatalf("revoked runtime remained routable: status=%#v condition=%#v", failed.Status, condition) + } + }) + } +} + +func TestSessionFailsClosedWhenOMPObjectsAreMissing(t *testing.T) { + for _, test := range []struct { + name string + configMap *corev1.ConfigMap + wantReason string + }{ + {name: "ConfigMap", wantReason: "OMPConfigMapNotFound"}, + {name: "Secret", configMap: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}, Data: map[string]string{"provider-models": "models", "agent-settings": "settings"}}, wantReason: "OMPCredentialSecretNotFound"}, + } { + t.Run(test.name, func(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}} + session := testSession() + objects := []client.Object{testHost(), rwxStorageClass(), workspace, pvc, session} + if test.configMap != nil { + objects = append(objects, test.configMap) + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}).WithObjects(objects...).Build() + r := &controllers.SessionReconciler{Client: c, APIReader: c, Scheme: scheme, RuntimeImage: testRuntimeImage, OMPConfig: controllers.SessionOMPConfig{ConfigMapName: "omp-runtime-config", ModelsKey: "provider-models", SettingsKey: "agent-settings", CredentialSecretName: "omp-runtime-credential", CredentialKey: "MODEL_API_KEY"}} + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, c, 0, 0) + var got clusterv1alpha1.T4Session + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + condition := findCondition(got.Status.Conditions, "RuntimeConfigured") + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != test.wantReason { + t.Fatalf("RuntimeConfigured = %#v, want False/%s", condition, test.wantReason) + } + }) + } +} + +func TestSessionRecreatesExternallyExposedOwnedService(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + serviceKey := types.NamespacedName{Namespace: "team", Name: controllers.SessionServiceName(session)} + var service corev1.Service + if err := c.Get(context.Background(), serviceKey, &service); err != nil { + t.Fatal(err) + } + service.Spec.Type = corev1.ServiceTypeNodePort + service.Spec.ExternalIPs = []string{"192.0.2.8"} + service.Spec.Ports[0].NodePort = 32080 + if err := c.Update(context.Background(), &service); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if err := c.Get(context.Background(), serviceKey, &service); !apierrors.IsNotFound(err) { + t.Fatalf("externally exposed Service was not deleted for safe recreation: %v", err) + } + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + if err := c.Get(context.Background(), serviceKey, &service); err != nil { + t.Fatal(err) + } + if service.Spec.Type != corev1.ServiceTypeClusterIP || len(service.Spec.ExternalIPs) != 0 || service.Spec.Ports[0].NodePort != 0 { + t.Fatalf("recreated Service retains external exposure: %#v", service.Spec) + } +} + +func TestSessionRestoresRequiredPodSelectorLabelsBeforeAvailability(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + podKey := types.NamespacedName{Namespace: "team", Name: controllers.SessionPodName(session)} + var pod corev1.Pod + if err := c.Get(context.Background(), podKey, &pod); err != nil { + t.Fatal(err) + } + delete(pod.Labels, "cluster.t4.dev/session") + if err := c.Update(context.Background(), &pod); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if err := c.Get(context.Background(), podKey, &pod); err != nil { + t.Fatal(err) + } + if pod.Labels["cluster.t4.dev/session"] != controllers.SessionPodName(session) { + t.Fatalf("required selector labels were not restored: %#v", pod.Labels) + } + var got clusterv1alpha1.T4Session + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + if got.Status.Phase == clusterv1alpha1.InfrastructureRunning { + t.Fatal("session became available in the same reconcile that repaired endpoint labels") + } +} + +func TestWorkspaceDeletionRefusesForeignDeterministicPVC(t *testing.T) { + for _, policy := range []clusterv1alpha1.RetentionPolicy{clusterv1alpha1.RetentionPolicyRetain, clusterv1alpha1.RetentionPolicyDelete} { + for _, mismatch := range []string{"uid-annotation", "controller-owner"} { + t.Run(string(policy)+"/"+mismatch, func(t *testing.T) { + scheme := testScheme(t) + workspace := testWorkspace(policy) + workspace.UID = "workspace-uid" + workspace.Finalizers = []string{clusterv1alpha1.WorkspaceFinalizer} + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: controllers.WorkspacePVCName(workspace), Namespace: workspace.Namespace, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, + }} + if mismatch == "uid-annotation" { + pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] = "foreign-workspace-uid" + } else { + pvc.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: "foreign", UID: "foreign-workspace-uid", Controller: ptr(true), + }} + } + expectedOwnerCount := len(pvc.OwnerReferences) + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc).Build() + if err := c.Delete(context.Background(), workspace); err != nil { + t.Fatal(err) + } + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var untouched corev1.PersistentVolumeClaim + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pvc), &untouched); err != nil { + t.Fatalf("foreign deterministic PVC was deleted: %v", err) + } + if untouched.Annotations[clusterv1alpha1.RetainedPVCAnnotation] != "" || len(untouched.OwnerReferences) != expectedOwnerCount { + t.Fatalf("foreign deterministic PVC was mutated: %#v", untouched.ObjectMeta) + } + var waiting clusterv1alpha1.T4Workspace + if err := c.Get(context.Background(), client.ObjectKeyFromObject(workspace), &waiting); err != nil { + t.Fatalf("workspace finalizer was removed on conflict: %v", err) + } + condition := findCondition(waiting.Status.Conditions, "Ready") + if !contains(waiting.Finalizers, clusterv1alpha1.WorkspaceFinalizer) || condition == nil || condition.Reason != "CleanupOwnershipConflict" { + t.Fatalf("workspace cleanup conflict not retained: %#v", waiting) + } + }) + } + } +} + +func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { + for _, foreignKind := range []string{"Pod", "Service"} { + t.Run(foreignKind, func(t *testing.T) { + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + session.Finalizers = []string{clusterv1alpha1.SessionFinalizer} + controller := true + ownerReferences := []metav1.OwnerReference{{ + APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Session", Name: session.Name, UID: session.UID, Controller: &controller, + }} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace, OwnerReferences: ownerReferences}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace, OwnerReferences: ownerReferences}} + if foreignKind == "Pod" { + pod.OwnerReferences = nil + } else { + service.OwnerReferences = nil + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() + if err := c.Delete(context.Background(), session); err != nil { + t.Fatal(err) + } + r := &controllers.SessionReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, c, 1, 1) + var waiting clusterv1alpha1.T4Session + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &waiting); err != nil { + t.Fatalf("session finalizer was removed on cleanup conflict: %v", err) + } + condition := findCondition(waiting.Status.Conditions, "Available") + if !contains(waiting.Finalizers, clusterv1alpha1.SessionFinalizer) || condition == nil || condition.Reason != "CleanupOwnershipConflict" { + t.Fatalf("session cleanup conflict not retained: %#v", waiting) + } + }) + } +} + +func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + session.Finalizers = []string{clusterv1alpha1.SessionFinalizer} + controller := true + ownerReferences := []metav1.OwnerReference{{ + APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Session", Name: session.Name, UID: session.UID, Controller: &controller, + }} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: "team", OwnerReferences: ownerReferences}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: "team", OwnerReferences: ownerReferences}} + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() + if err := c.Delete(context.Background(), session); err != nil { + t.Fatal(err) + } + r := configuredSessionReconciler(c, scheme) + reconcileMany(t, 3, func() error { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + assertObjectCounts(t, c, 0, 0) + var gone clusterv1alpha1.T4Session + if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &gone); !apierrors.IsNotFound(err) { + t.Fatalf("session finalizer removed before cleanup completed: %v", err) + } +} + +func configuredSessionReconciler(c client.Client, scheme *runtime.Scheme) *controllers.SessionReconciler { + for _, object := range []client.Object{ + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}, Data: map[string]string{ + "provider-models": "models", "agent-settings": "settings", "other-models": "other models", "other-settings": "other settings", + }}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "other-omp-config", Namespace: "team"}, Data: map[string]string{ + "provider-models": "models", "agent-settings": "settings", "other-models": "other models", "other-settings": "other settings", + }}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-credential", Namespace: "team"}, Data: map[string][]byte{ + "MODEL_API_KEY": []byte("credential"), "OTHER_MODEL_API_KEY": []byte("other credential"), + }}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "other-credential", Namespace: "team"}, Data: map[string][]byte{ + "MODEL_API_KEY": []byte("credential"), "OTHER_MODEL_API_KEY": []byte("other credential"), + }}, + rwxStorageClass(), + } { + if err := c.Create(context.Background(), object); err != nil && !apierrors.IsAlreadyExists(err) { + panic(err) + } + } + return &controllers.SessionReconciler{ + Client: c, + APIReader: c, + Scheme: scheme, + RuntimeImage: testRuntimeImage, + OMPConfig: controllers.SessionOMPConfig{ + ConfigMapName: "omp-runtime-config", + ModelsKey: "provider-models", + SettingsKey: "agent-settings", + CredentialSecretName: "omp-runtime-credential", + CredentialKey: "MODEL_API_KEY", + }, + } +} + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, storagev1.AddToScheme, clusterv1alpha1.AddToScheme} { + if err := add(scheme); err != nil { + t.Fatal(err) + } + } + return scheme +} + +func testHost() *clusterv1alpha1.T4ClusterHost { + return &clusterv1alpha1.T4ClusterHost{ + ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team", UID: "host-uid"}, + Spec: clusterv1alpha1.T4ClusterHostSpec{StorageClassName: "portable-rwx", RuntimeProfiles: []string{"default"}}, + } +} + +func rwxStorageClass() *storagev1.StorageClass { + return &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "portable-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, + Provisioner: "example.invalid/csi", + } +} + +func testWorkspace(policy clusterv1alpha1.RetentionPolicy) *clusterv1alpha1.T4Workspace { + return &clusterv1alpha1.T4Workspace{ + ObjectMeta: metav1.ObjectMeta{Name: "workspace-a", Namespace: "team", Generation: 3}, + Spec: clusterv1alpha1.T4WorkspaceSpec{ + HostRef: "host-a", DisplayName: "Workspace A", Owner: "team-a", Size: apiresource.MustParse("10Gi"), RetentionPolicy: policy, + }, + } +} + +func testSession() *clusterv1alpha1.T4Session { + return &clusterv1alpha1.T4Session{ + ObjectMeta: metav1.ObjectMeta{Name: "session-a", Namespace: "team", Generation: 2}, + Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a", Title: "Session A", RuntimeProfile: "default", GUIEnabled: true}, + } +} + +func reconcileMany(t *testing.T, count int, reconcile func() error) { + t.Helper() + for i := 0; i < count; i++ { + if err := reconcile(); err != nil { + t.Fatalf("reconcile %d: %v", i+1, err) + } + } +} + +func assertObjectCounts(t *testing.T, c client.Client, wantPods, wantServices int) { + t.Helper() + var pods corev1.PodList + var services corev1.ServiceList + if err := c.List(context.Background(), &pods, client.InNamespace("team")); err != nil { + t.Fatal(err) + } + if err := c.List(context.Background(), &services, client.InNamespace("team")); err != nil { + t.Fatal(err) + } + if len(pods.Items) != wantPods || len(services.Items) != wantServices { + t.Fatalf("pods/services = %d/%d, want %d/%d", len(pods.Items), len(services.Items), wantPods, wantServices) + } +} + +func findCondition(conditions []metav1.Condition, conditionType string) *metav1.Condition { + for i := range conditions { + if conditions[i].Type == conditionType { + return &conditions[i] + } + } + return nil +} + +func contains(values []string, wanted string) bool { + for _, value := range values { + if value == wanted { + return true + } + } + return false +} + +func hasMount(mounts []corev1.VolumeMount, name, path string) bool { + for _, mount := range mounts { + if mount.Name == name && mount.MountPath == path { + return true + } + } + return false +} + +func hasReadOnlyMount(mounts []corev1.VolumeMount, name, path string) bool { + for _, mount := range mounts { + if mount.Name == name && mount.MountPath == path && mount.ReadOnly { + return true + } + } + return false +} + +func ptr[T any](value T) *T { return &value } diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go new file mode 100644 index 00000000..c2353a12 --- /dev/null +++ b/packages/cluster-operator/controllers/session_controller.go @@ -0,0 +1,655 @@ +package controllers + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "reflect" + "regexp" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/api/meta" + apiresource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + intstr "k8s.io/apimachinery/pkg/util/intstr" + utilvalidation "k8s.io/apimachinery/pkg/util/validation" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" +) + +const ( + DefaultSessionServiceAccount = "t4-cluster-session" + DefaultServerServiceAccount = "t4-cluster-server" + DefaultKubernetesAPIAudience = "https://kubernetes.default.svc" + SessionReviewerTokenExpirationSeconds int64 = 3600 +) + +var ( + configMapKeyPattern = regexp.MustCompile(`^[-._A-Za-z0-9]+$`) + envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + runtimeImagePattern = regexp.MustCompile(`^(?:(?:[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?|\[[A-Fa-f0-9:]+\])(?::[0-9]+)?/)?[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)*@sha256:[a-f0-9]{64}$`) +) + +func reservedCredentialEnvironment(name string) bool { + if strings.HasPrefix(name, "T4_") || strings.HasPrefix(name, "OMP_") || strings.HasPrefix(name, "PI_") || strings.HasPrefix(name, "XDG_") || strings.HasPrefix(name, "LD_") { + return true + } + switch name { + case "HOME", "DISPLAY", "PATH", "BASH_ENV", "ENV", "SHELLOPTS", "NODE_OPTIONS", "BUN_OPTIONS": + return true + default: + return false + } +} + +type SessionOMPConfig struct { + ConfigMapName string + ModelsKey string + SettingsKey string + CredentialSecretName string + CredentialKey string + AllowUnauthenticated bool +} +type ompResourceVersions struct { + ConfigMap string `json:"configMap"` + CredentialSecret string `json:"credentialSecret,omitempty"` +} + +func (r *SessionReconciler) loadOMPResourceVersions(ctx context.Context, namespace string) (ompResourceVersions, string, string, error) { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + var configMap corev1.ConfigMap + if err := reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: r.OMPConfig.ConfigMapName}, &configMap); err != nil { + if apierrors.IsNotFound(err) { + return ompResourceVersions{}, "OMPConfigMapNotFound", "administrator-owned OMP ConfigMap does not exist", nil + } + return ompResourceVersions{}, "", "", err + } + if configMap.Data[r.OMPConfig.ModelsKey] == "" || configMap.Data[r.OMPConfig.SettingsKey] == "" { + return ompResourceVersions{}, "OMPConfigMapInvalid", "administrator-owned OMP ConfigMap must contain nonempty models and settings keys", nil + } + versions := ompResourceVersions{ConfigMap: configMap.ResourceVersion} + if r.OMPConfig.AllowUnauthenticated { + return versions, "", "", nil + } + var secret corev1.Secret + if err := reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: r.OMPConfig.CredentialSecretName}, &secret); err != nil { + if apierrors.IsNotFound(err) { + return ompResourceVersions{}, "OMPCredentialSecretNotFound", "administrator-owned OMP credential Secret does not exist", nil + } + return ompResourceVersions{}, "", "", err + } + if len(secret.Data[r.OMPConfig.CredentialKey]) == 0 { + return ompResourceVersions{}, "OMPCredentialSecretInvalid", "administrator-owned OMP credential Secret must contain the configured nonempty key", nil + } + versions.CredentialSecret = secret.ResourceVersion + return versions, "", "", nil +} + +func (config SessionOMPConfig) validationFailure() (string, string) { + if config.ConfigMapName == "" || config.ModelsKey == "" || config.SettingsKey == "" { + return "OMPReferencesMissing", "administrator-owned OMP ConfigMap and configuration keys are not configured" + } + if config.AllowUnauthenticated { + if config.CredentialSecretName != "" || config.CredentialKey != "" { + return "OMPReferencesInvalid", "unauthenticated OMP mode cannot include credential Secret references" + } + } else { + if config.CredentialSecretName == "" && config.CredentialKey == "" { + return "OMPReferencesMissing", "administrator-owned OMP credential Secret and key are not configured" + } + if config.CredentialSecretName == "" || config.CredentialKey == "" { + return "OMPReferencesInvalid", "OMP credential Secret and key must be configured together" + } + } + if len(utilvalidation.IsDNS1123Subdomain(config.ConfigMapName)) != 0 || + len(config.ModelsKey) > 253 || !configMapKeyPattern.MatchString(config.ModelsKey) || + len(config.SettingsKey) > 253 || !configMapKeyPattern.MatchString(config.SettingsKey) || + config.ModelsKey == config.SettingsKey || + (config.CredentialSecretName != "" && len(utilvalidation.IsDNS1123Subdomain(config.CredentialSecretName)) != 0) || + (config.CredentialKey != "" && (len(config.CredentialKey) > 253 || !envVarNamePattern.MatchString(config.CredentialKey) || reservedCredentialEnvironment(config.CredentialKey))) { + return "OMPReferencesInvalid", "administrator-owned OMP configuration references are invalid" + } + return "", "" +} + +func runtimeImageValidationFailure(image string) (string, string) { + if image == "" { + return "RuntimeImageMissing", "administrator-owned session runtime image is not configured" + } + digestSeparator := strings.Index(image, "@sha256:") + if digestSeparator <= 0 || digestSeparator > 255 || !runtimeImagePattern.MatchString(image) { + return "RuntimeImageInvalid", "administrator-owned session runtime image must be an exact repository@sha256 digest with 64 lowercase hexadecimal characters" + } + return "", "" +} + +type SessionReconciler struct { + client.Client + Scheme *runtime.Scheme + APIReader client.Reader + RuntimeImage string + SessionServiceAccountName string + ServerServiceAccountName string + KubernetesAPIAudience string + OMPConfig SessionOMPConfig + ExcludedNodeNames []string + Resources corev1.ResourceRequirements + SharedMemorySize apiresource.Quantity + TemporarySize apiresource.Quantity +} + +func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) { + var session clusterv1alpha1.T4Session + found := false + defer func() { + observeReconcile(metricKindSession, request.NamespacedName, session.Status.Conditions, conditionObjectPresent(&session, found, err), err) + }() + if err := r.Get(ctx, request.NamespacedName, &session); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + found = true + if !session.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, &session) + } + if controllerutil.AddFinalizer(&session, clusterv1alpha1.SessionFinalizer) { + if err := r.Update(ctx, &session); err != nil { + return ctrl.Result{}, err + } + } + if reason, message := runtimeImageValidationFailure(r.RuntimeImage); reason != "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", reason, message) + } + if reason, message := r.OMPConfig.validationFailure(); reason != "" { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", reason, message) + } + + var host clusterv1alpha1.T4ClusterHost + if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.HostRef}, &host); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") + } + return ctrl.Result{}, err + } + if !hasString(host.Spec.RuntimeProfiles, session.Spec.RuntimeProfile) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", "RuntimeProfileNotAllowed", "runtime profile is not allowed by the referenced T4ClusterHost") + } + var storageClass storagev1.StorageClass + if err := r.Get(ctx, types.NamespacedName{Name: host.Spec.StorageClassName}, &storageClass); err != nil { + if apierrors.IsNotFound(err) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", ReasonStorageClassNotFound, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost does not exist", host.Spec.StorageClassName)) + } + return ctrl.Result{}, err + } + if !storageClassAllowsRWX(storageClass.Annotations) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", ReasonStorageClassNotRWX, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost is not administrator-declared ReadWriteMany", host.Spec.StorageClassName)) + } + var workspace clusterv1alpha1.T4Workspace + if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.WorkspaceRef}, &workspace); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") + } + return ctrl.Result{}, err + } + if workspace.Spec.HostRef != session.Spec.HostRef { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") + } + if workspace.Status.PVCName == "" { + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") + } + var pvc corev1.PersistentVolumeClaim + if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: workspace.Status.PVCName}, &pvc); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") + } + return ctrl.Result{}, err + } + if pvc.Status.Phase != corev1.ClaimBound || !pvcHasRWX(&pvc) { + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") + } + runtimeVersions, reason, message, err := r.loadOMPResourceVersions(ctx, session.Namespace) + if err != nil { + return ctrl.Result{}, err + } + if reason != "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", reason, message) + } + + serviceName := SessionServiceName(&session) + podName := SessionPodName(&session) + labels := map[string]string{ + "app.kubernetes.io/name": "t4-session-runtime", + "app.kubernetes.io/part-of": "t4-cluster", + "cluster.t4.dev/session": podName, + } + desiredService := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: session.Namespace, Labels: labels}, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: labels, + Ports: []corev1.ServicePort{{Name: "host", Port: 8787, TargetPort: intstr.FromString("host"), Protocol: corev1.ProtocolTCP}}, + }, + } + if err := controllerutil.SetControllerReference(&session, &desiredService, r.Scheme); err != nil { + return ctrl.Result{}, err + } + var service corev1.Service + if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: serviceName}, &service); apierrors.IsNotFound(err) { + service = desiredService + if err := r.Create(ctx, &service); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + } else if err != nil { + return ctrl.Result{}, err + } else if !metav1.IsControlledBy(&service, &session) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "Available", "ServiceOwnershipConflict", "deterministic session Service is not controlled by this session") + } else if !serviceExposureIsInternal(&service) { + if err := r.Delete(ctx, &service); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err := r.updateSessionPending(ctx, &session, podName, serviceName, "ServiceExposureChanged", "session Service is being recreated with ClusterIP-only exposure"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Second}, nil + } else if !reflect.DeepEqual(service.Spec.Selector, desiredService.Spec.Selector) || !reflect.DeepEqual(service.Spec.Ports, desiredService.Spec.Ports) || !reflect.DeepEqual(service.Labels, desiredService.Labels) { + service.Spec.Selector = desiredService.Spec.Selector + service.Spec.Ports = desiredService.Spec.Ports + service.Labels = desiredService.Labels + if err := r.Update(ctx, &service); err != nil { + return ctrl.Result{}, err + } + } + + desiredPod, err := r.desiredPod(&session, workspace.Status.PVCName, podName, labels, runtimeVersions) + if err != nil { + return ctrl.Result{}, err + } + if err := controllerutil.SetControllerReference(&session, &desiredPod, r.Scheme); err != nil { + return ctrl.Result{}, err + } + var pod corev1.Pod + if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: podName}, &pod); apierrors.IsNotFound(err) { + pod = desiredPod + if err := r.Create(ctx, &pod); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + } else if err != nil { + return ctrl.Result{}, err + } else if !metav1.IsControlledBy(&pod, &session) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "Available", "PodOwnershipConflict", "deterministic session Pod is not controlled by this session") + } else if !labelsContain(pod.Labels, desiredPod.Labels) { + if pod.Labels == nil { + pod.Labels = map[string]string{} + } + for key, value := range desiredPod.Labels { + pod.Labels[key] = value + } + if err := r.Update(ctx, &pod); err != nil { + return ctrl.Result{}, err + } + if err := r.updateSessionPending(ctx, &session, podName, serviceName, "PodLabelsChanged", "session Pod selector labels are being restored"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Second}, nil + } else if pod.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] != desiredPod.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] { + if err := r.Delete(ctx, &pod); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err := r.updateSessionPending(ctx, &session, podName, serviceName, "PodSpecChanged", "session Pod is being recreated to apply immutable desired state"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Second}, nil + } + + original := session.Status + if session.Status.Conditions != nil { + original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } + session.Status.ObservedGeneration = session.Generation + session.Status.PodName = podName + session.Status.ServiceName = serviceName + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionTrue, "PVCBoundRWX", "workspace PVC is Bound and ReadWriteMany", session.Generation)) + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionTrue, "OMPReferencesReady", "administrator-owned OMP runtime references are configured", session.Generation)) + if podReady(&pod) { + session.Status.Phase = clusterv1alpha1.InfrastructureRunning + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionTrue, "PodReady", "session infrastructure pod is ready", session.Generation)) + } else if pod.Status.Phase == corev1.PodFailed { + session.Status.Phase = clusterv1alpha1.InfrastructureFailed + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "PodFailed", "session infrastructure pod failed", session.Generation)) + } else { + session.Status.Phase = clusterv1alpha1.InfrastructurePending + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "PodStarting", "session infrastructure pod is starting", session.Generation)) + } + if !reflect.DeepEqual(original, session.Status) { + if err := r.Status().Update(ctx, &session); err != nil { + return ctrl.Result{}, err + } + } + if !podReady(&pod) { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil +} + +func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcName, podName string, labels map[string]string, runtimeVersions ompResourceVersions) (corev1.Pod, error) { + falseValue := false + trueValue := true + runAsUser := int64(10001) + fsGroup := int64(10001) + grace := int64(45) + shmSize := r.SharedMemorySize.DeepCopy() + if shmSize.IsZero() { + shmSize = apiresource.MustParse("1Gi") + } + temporarySize := r.TemporarySize.DeepCopy() + if temporarySize.IsZero() { + temporarySize = apiresource.MustParse("2Gi") + } + resources := r.Resources.DeepCopy() + if resources.Requests == nil { + resources.Requests = corev1.ResourceList{corev1.ResourceCPU: apiresource.MustParse("500m"), corev1.ResourceMemory: apiresource.MustParse("1Gi")} + } + if resources.Limits == nil { + resources.Limits = corev1.ResourceList{corev1.ResourceCPU: apiresource.MustParse("4"), corev1.ResourceMemory: apiresource.MustParse("8Gi")} + } + sessionServiceAccount := r.SessionServiceAccountName + if sessionServiceAccount == "" { + sessionServiceAccount = DefaultSessionServiceAccount + } + serverServiceAccount := r.ServerServiceAccountName + if serverServiceAccount == "" { + serverServiceAccount = DefaultServerServiceAccount + } + kubernetesAPIAudience := r.KubernetesAPIAudience + if kubernetesAPIAudience == "" { + kubernetesAPIAudience = DefaultKubernetesAPIAudience + } + excluded := r.ExcludedNodeNames + if len(excluded) == 0 { + excluded = []string{"k3s-worker-02"} + } + stateID := strings.TrimPrefix(podName, "t4-session-") + container := corev1.Container{ + Name: "session-runtime", Image: r.RuntimeImage, ImagePullPolicy: corev1.PullIfNotPresent, + Ports: []corev1.ContainerPort{{Name: "host", ContainerPort: 8787, Protocol: corev1.ProtocolTCP}}, + Env: []corev1.EnvVar{ + {Name: "T4_CLUSTER_SERVER_SERVICE_ACCOUNT", Value: serverServiceAccount}, + {Name: "T4_KUBERNETES_TOKEN_PATH", Value: "/var/run/secrets/kubernetes.io/serviceaccount/token"}, + {Name: "T4_KUBERNETES_CA_PATH", Value: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"}, + {Name: "T4_KUBERNETES_NAMESPACE_PATH", Value: "/var/run/secrets/kubernetes.io/serviceaccount/namespace"}, + {Name: "T4_KUBERNETES_API_AUDIENCE", Value: kubernetesAPIAudience}, + {Name: "T4_SESSION_NAME", Value: stateID}, + {Name: "T4_SESSION_STATE_ID", Value: stateID}, + {Name: "T4_WORKSPACE_ROOT", Value: "/workspace"}, + {Name: "T4_SESSION_STATE_ROOT", Value: "/workspace/.t4/sessions/" + stateID}, + {Name: "T4_AUTHORITY_STATE_DIR", Value: "/workspace/.t4/sessions/" + stateID + "/authority"}, + {Name: "T4_BROWSER_STATE_DIR", Value: "/workspace/.t4/sessions/" + stateID + "/browser"}, + {Name: "T4_GUI_ENABLED", Value: fmt.Sprintf("%t", session.Spec.GUIEnabled)}, + {Name: "DISPLAY", Value: ":99"}, + {Name: "T4_OMP_CONFIG_SOURCE_DIR", Value: "/run/t4-omp-config-source"}, + {Name: "T4_OMP_ALLOW_UNAUTHENTICATED", Value: fmt.Sprintf("%t", r.OMPConfig.AllowUnauthenticated)}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "workspace", MountPath: "/workspace"}, + {Name: "runtime", MountPath: "/run"}, + {Name: "temporary", MountPath: "/tmp"}, + {Name: "shared-memory", MountPath: "/dev/shm"}, + {Name: "kubernetes-api-access", MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", ReadOnly: true}, + {Name: "omp-config-source", MountPath: "/run/t4-omp-config-source", ReadOnly: true}, + }, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &falseValue, ReadOnlyRootFilesystem: &trueValue, RunAsNonRoot: &trueValue, RunAsUser: &runAsUser, + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + }, + Resources: *resources, + StartupProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString("host")}}, FailureThreshold: 30, PeriodSeconds: 2, TimeoutSeconds: 1}, + ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString("host")}}, FailureThreshold: 3, PeriodSeconds: 5, TimeoutSeconds: 2}, + LivenessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString("host")}}, FailureThreshold: 3, PeriodSeconds: 10, TimeoutSeconds: 2}, + } + if !r.OMPConfig.AllowUnauthenticated { + container.Args = []string{r.OMPConfig.CredentialKey} + container.Env = append(container.Env, corev1.EnvVar{ + Name: r.OMPConfig.CredentialKey, + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: r.OMPConfig.CredentialSecretName}, + Key: r.OMPConfig.CredentialKey, + Optional: &falseValue, + }}, + }) + } + volumes := []corev1.Volume{ + {Name: "workspace", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcName}}}, + {Name: "runtime", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory, SizeLimit: ptrQuantity(apiresource.MustParse("128Mi"))}}}, + {Name: "temporary", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &temporarySize}}}, + {Name: "shared-memory", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory, SizeLimit: &shmSize}}}, + {Name: "kubernetes-api-access", VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: ptr(int32(0440)), + Sources: []corev1.VolumeProjection{ + {ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Audience: kubernetesAPIAudience, ExpirationSeconds: ptr(SessionReviewerTokenExpirationSeconds), Path: "token"}}, + {ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "kube-root-ca.crt"}, Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}}}, + {DownwardAPI: &corev1.DownwardAPIProjection{Items: []corev1.DownwardAPIVolumeFile{{Path: "namespace", FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}}, + }, + }}}, + {Name: "omp-config-source", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: r.OMPConfig.ConfigMapName}, + DefaultMode: ptr(int32(0440)), + Optional: &falseValue, + Items: []corev1.KeyToPath{ + {Key: r.OMPConfig.ModelsKey, Path: "models.yml", Mode: ptr(int32(0440))}, + {Key: r.OMPConfig.SettingsKey, Path: "config.yml", Mode: ptr(int32(0440))}, + }, + }}}, + } + if session.Spec.InitialPromptSecretRef != nil { + volumes = append(volumes, corev1.Volume{Name: "initial-prompt", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: session.Spec.InitialPromptSecretRef.Name, Items: []corev1.KeyToPath{{Key: "prompt", Path: "prompt", Mode: ptr(int32(0440))}}}}}) + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: "initial-prompt", MountPath: "/run/t4-initial-prompt", ReadOnly: true}) + container.Env = append(container.Env, corev1.EnvVar{Name: "T4_INITIAL_PROMPT_FILE", Value: "/run/t4-initial-prompt/prompt"}) + } + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: session.Namespace, Labels: labels}, + Spec: corev1.PodSpec{ + AutomountServiceAccountToken: &falseValue, + ServiceAccountName: sessionServiceAccount, + EnableServiceLinks: &falseValue, + TerminationGracePeriodSeconds: &grace, + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &trueValue, RunAsUser: &runAsUser, FSGroup: &fsGroup, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, + Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "kubernetes.io/hostname", Operator: corev1.NodeSelectorOpNotIn, Values: excluded}}}}}}}, + Containers: []corev1.Container{container}, Volumes: volumes, + }, + } + hashInput, err := json.Marshal(struct { + PodSpec corev1.PodSpec `json:"podSpec"` + OMPResourceVersions ompResourceVersions `json:"ompResourceVersions"` + }{PodSpec: pod.Spec, OMPResourceVersions: runtimeVersions}) + if err != nil { + return corev1.Pod{}, fmt.Errorf("serialize desired session Pod: %w", err) + } + hash := sha256.Sum256(hashInput) + pod.Annotations = map[string]string{clusterv1alpha1.SessionPodSpecHashAnnotation: fmt.Sprintf("%x", hash)} + return pod, nil +} + +func (r *SessionReconciler) updateSessionPending(ctx context.Context, session *clusterv1alpha1.T4Session, podName, serviceName, reason, message string) error { + original := session.Status + if session.Status.Conditions != nil { + original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } + session.Status.ObservedGeneration = session.Generation + session.Status.PodName = podName + session.Status.ServiceName = serviceName + session.Status.Phase = clusterv1alpha1.InfrastructurePending + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, reason, message, session.Generation)) + if reflect.DeepEqual(original, session.Status) { + return nil + } + return r.Status().Update(ctx, session) +} + +func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *clusterv1alpha1.T4Session) (ctrl.Result, error) { + if !controllerutil.ContainsFinalizer(session, clusterv1alpha1.SessionFinalizer) { + return ctrl.Result{}, nil + } + originalStatus := session.Status + if session.Status.Conditions != nil { + originalStatus.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } + session.Status.ObservedGeneration = session.Generation + session.Status.Phase = clusterv1alpha1.InfrastructureTerminating + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "Terminating", "session infrastructure is terminating", session.Generation)) + if !reflect.DeepEqual(originalStatus, session.Status) { + if err := r.Status().Update(ctx, session); err != nil { + return ctrl.Result{}, err + } + } + objects := []client.Object{ + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: SessionPodName(session), Namespace: session.Namespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, + } + existing := make([]client.Object, 0, len(objects)) + for _, object := range objects { + err := r.Get(ctx, client.ObjectKeyFromObject(object), object) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return ctrl.Result{}, err + } + if !metav1.IsControlledBy(object, session) { + before := session.Status + if session.Status.Conditions != nil { + before.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "CleanupOwnershipConflict", fmt.Sprintf("deterministic %T is not controlled by this session", object), session.Generation)) + if !reflect.DeepEqual(before, session.Status) { + if err := r.Status().Update(ctx, session); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + existing = append(existing, object) + } + for _, object := range existing { + if object.GetDeletionTimestamp().IsZero() { + if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + } + if len(existing) > 0 { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + controllerutil.RemoveFinalizer(session, clusterv1alpha1.SessionFinalizer) + return ctrl.Result{}, r.Update(ctx, session) +} + +func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, session *clusterv1alpha1.T4Session) error { + objects := []client.Object{ + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: SessionPodName(session), Namespace: session.Namespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, + } + for _, object := range objects { + if err := r.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { + if err := client.IgnoreNotFound(err); err != nil { + return err + } + continue + } + if !metav1.IsControlledBy(object, session) { + continue + } + if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return nil +} + +func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, conditionType, reason, message string) error { + original := session.Status + if session.Status.Conditions != nil { + original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } + session.Status.ObservedGeneration = session.Generation + session.Status.PodName = "" + session.Status.ServiceName = "" + session.Status.Phase = clusterv1alpha1.InfrastructureFailed + meta.SetStatusCondition(&session.Status.Conditions, condition(conditionType, metav1.ConditionFalse, reason, message, session.Generation)) + if conditionType != "Available" { + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, reason, message, session.Generation)) + } + if reflect.DeepEqual(original, session.Status) { + return nil + } + return r.Status().Update(ctx, session) +} + +func serviceExposureIsInternal(service *corev1.Service) bool { + if service.Spec.Type != corev1.ServiceTypeClusterIP || service.Spec.ClusterIP == corev1.ClusterIPNone || service.Spec.ExternalName != "" || + len(service.Spec.ExternalIPs) != 0 || service.Spec.LoadBalancerIP != "" || len(service.Spec.LoadBalancerSourceRanges) != 0 || + service.Spec.LoadBalancerClass != nil || service.Spec.HealthCheckNodePort != 0 || service.Spec.AllocateLoadBalancerNodePorts != nil { + return false + } + for _, port := range service.Spec.Ports { + if port.NodePort != 0 { + return false + } + } + return true +} + +func labelsContain(actual, required map[string]string) bool { + for key, value := range required { + if actual[key] != value { + return false + } + } + return true +} + +func (r *SessionReconciler) SetupWithManager(manager ctrl.Manager) error { + return ctrl.NewControllerManagedBy(manager). + For(&clusterv1alpha1.T4Session{}). + Owns(&corev1.Pod{}). + Owns(&corev1.Service{}). + Complete(r) +} + +func podReady(pod *corev1.Pod) bool { + for _, item := range pod.Status.Conditions { + if item.Type == corev1.PodReady { + return item.Status == corev1.ConditionTrue + } + } + return false +} + +func ptrQuantity(value apiresource.Quantity) *apiresource.Quantity { return &value } +func ptr[T any](value T) *T { return &value } diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go new file mode 100644 index 00000000..fbc60f45 --- /dev/null +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -0,0 +1,352 @@ +package controllers + +import ( + "context" + "fmt" + "reflect" + "time" + + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" +) + +type WorkspaceReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +const ( + workspacePVCPartOfLabel = "app.kubernetes.io/part-of" + workspacePVCPartOfValue = "t4-cluster" + workspacePVCWorkspaceLabel = "cluster.t4.dev/workspace" + hostStorageClassIndexField = "t4.workspace.host.storageClassName" + workspaceHostRefIndexField = "t4.workspace.spec.hostRef" +) + +func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) { + var workspace clusterv1alpha1.T4Workspace + found := false + defer func() { + observeReconcile(metricKindWorkspace, request.NamespacedName, workspace.Status.Conditions, conditionObjectPresent(&workspace, found, err), err) + }() + if err := r.Get(ctx, request.NamespacedName, &workspace); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + found = true + if !workspace.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, &workspace) + } + if controllerutil.AddFinalizer(&workspace, clusterv1alpha1.WorkspaceFinalizer) { + if err := r.Update(ctx, &workspace); err != nil { + return ctrl.Result{}, err + } + } + + var host clusterv1alpha1.T4ClusterHost + if err := r.Get(ctx, types.NamespacedName{Namespace: workspace.Namespace, Name: workspace.Spec.HostRef}, &host); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") + } + return ctrl.Result{}, err + } + storageClassName := host.Spec.StorageClassName + var storageClass storagev1.StorageClass + if err := r.Get(ctx, types.NamespacedName{Name: storageClassName}, &storageClass); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassNotFound, fmt.Sprintf("StorageClass %q does not exist", storageClassName)) + } + return ctrl.Result{}, err + } + if !storageClassAllowsRWX(storageClass.Annotations) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassNotRWX, fmt.Sprintf("StorageClass %q is not administrator-declared ReadWriteMany", storageClassName)) + } + + pvcName := WorkspacePVCName(&workspace) + var pvc corev1.PersistentVolumeClaim + err = r.Get(ctx, types.NamespacedName{Namespace: workspace.Namespace, Name: pvcName}, &pvc) + if apierrors.IsNotFound(err) { + volumeMode := corev1.PersistentVolumeFilesystem + pvc = corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, Namespace: workspace.Namespace, + Labels: map[string]string{ + workspacePVCPartOfLabel: workspacePVCPartOfValue, + workspacePVCWorkspaceLabel: workspace.Name, + }, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + StorageClassName: &storageClassName, + VolumeMode: &volumeMode, + Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: workspace.Spec.Size.DeepCopy()}}, + }, + } + if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyDelete { + if err := controllerutil.SetControllerReference(&workspace, &pvc, r.Scheme); err != nil { + return ctrl.Result{}, err + } + } + if err := r.Create(ctx, &pvc); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + } else if err != nil { + return ctrl.Result{}, err + } else if !workspaceOwnsPVC(&workspace, &pvc) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "deterministic workspace PVC does not belong to this workspace") + } else if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyRetain && metav1.IsControlledBy(&pvc, &workspace) { + before := pvc.DeepCopy() + pvc.OwnerReferences = removeWorkspaceOwnerReference(pvc.OwnerReferences, workspace.UID) + if !reflect.DeepEqual(before.OwnerReferences, pvc.OwnerReferences) { + if err := r.Update(ctx, &pvc); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + } + + original := workspace.Status + original.Capacity = workspace.Status.Capacity.DeepCopy() + if workspace.Status.Conditions != nil { + original.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) + } + workspace.Status.ObservedGeneration = workspace.Generation + workspace.Status.PVCName = pvcName + workspace.Status.PVCPhase = pvc.Status.Phase + capacity := pvc.Status.Capacity[corev1.ResourceStorage] + workspace.Status.Capacity = capacity.DeepCopy() + meta.SetStatusCondition(&workspace.Status.Conditions, condition("StorageReady", metav1.ConditionTrue, ReasonStorageReady, "RWX StorageClass and workspace PVC are accepted", workspace.Generation)) + switch pvc.Status.Phase { + case corev1.ClaimBound: + if !pvcHasRWX(&pvc) { + workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCNotRWX", "bound workspace PVC does not request ReadWriteMany", workspace.Generation)) + } else { + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionTrue, "PVCBound", "workspace PVC is bound with ReadWriteMany access", workspace.Generation)) + } + case corev1.ClaimLost: + workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCLost", "workspace PVC lost its volume", workspace.Generation)) + default: + workspace.Status.Phase = clusterv1alpha1.InfrastructurePending + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCBinding", "workspace PVC is waiting to bind", workspace.Generation)) + } + if !reflect.DeepEqual(original, workspace.Status) { + if err := r.Status().Update(ctx, &workspace); err != nil { + return ctrl.Result{}, err + } + } + if pvc.Status.Phase != corev1.ClaimBound { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + return ctrl.Result{}, nil +} + +func workspaceOwnsPVC(workspace *clusterv1alpha1.T4Workspace, pvc *corev1.PersistentVolumeClaim) bool { + if pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] != string(workspace.UID) { + return false + } + controller := metav1.GetControllerOf(pvc) + if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyDelete { + return controller != nil && controller.UID == workspace.UID + } + return controller == nil || controller.UID == workspace.UID +} + +func removeWorkspaceOwnerReference(references []metav1.OwnerReference, uid types.UID) []metav1.OwnerReference { + output := references[:0] + for _, reference := range references { + if reference.UID != uid { + output = append(output, reference) + } + } + return output +} + +func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *clusterv1alpha1.T4Workspace) (ctrl.Result, error) { + if !controllerutil.ContainsFinalizer(workspace, clusterv1alpha1.WorkspaceFinalizer) { + return ctrl.Result{}, nil + } + originalStatus := workspace.Status + if workspace.Status.Conditions != nil { + originalStatus.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) + } + workspace.Status.ObservedGeneration = workspace.Generation + workspace.Status.Phase = clusterv1alpha1.InfrastructureTerminating + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "Terminating", "workspace infrastructure is terminating", workspace.Generation)) + if !reflect.DeepEqual(originalStatus, workspace.Status) { + if err := r.Status().Update(ctx, workspace); err != nil { + return ctrl.Result{}, err + } + } + var sessions clusterv1alpha1.T4SessionList + if err := r.List(ctx, &sessions, client.InNamespace(workspace.Namespace)); err != nil { + return ctrl.Result{}, err + } + remainingSessions := 0 + for i := range sessions.Items { + if sessions.Items[i].Spec.WorkspaceRef == workspace.Name { + remainingSessions++ + } + } + if remainingSessions > 0 { + before := workspace.Status + if workspace.Status.Conditions != nil { + before.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) + } + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "SessionsRemain", fmt.Sprintf("workspace deletion is waiting for %d session resources", remainingSessions), workspace.Generation)) + if !reflect.DeepEqual(before, workspace.Status) { + if err := r.Status().Update(ctx, workspace); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + pvcKey := types.NamespacedName{Namespace: workspace.Namespace, Name: WorkspacePVCName(workspace)} + var pvc corev1.PersistentVolumeClaim + err := r.Get(ctx, pvcKey, &pvc) + if err == nil && !workspaceOwnsPVC(workspace, &pvc) { + before := workspace.Status + if workspace.Status.Conditions != nil { + before.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) + } + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "CleanupOwnershipConflict", "deterministic workspace PVC does not belong to this workspace", workspace.Generation)) + if !reflect.DeepEqual(before, workspace.Status) { + if err := r.Status().Update(ctx, workspace); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyRetain { + if err == nil { + before := pvc.DeepCopy() + pvc.OwnerReferences = removeWorkspaceOwnerReference(pvc.OwnerReferences, workspace.UID) + if pvc.Annotations == nil { + pvc.Annotations = map[string]string{} + } + pvc.Annotations[clusterv1alpha1.RetainedPVCAnnotation] = "true" + if !reflect.DeepEqual(before.ObjectMeta, pvc.ObjectMeta) { + if err := r.Update(ctx, &pvc); err != nil { + return ctrl.Result{}, err + } + } + } else if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } else { + if err == nil { + if err := r.Delete(ctx, &pvc); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Second}, nil + } + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + controllerutil.RemoveFinalizer(workspace, clusterv1alpha1.WorkspaceFinalizer) + return ctrl.Result{}, r.Update(ctx, workspace) +} + +func (r *WorkspaceReconciler) updateWorkspaceFailure(ctx context.Context, workspace *clusterv1alpha1.T4Workspace, conditionType, reason, message string) error { + original := workspace.Status + original.Capacity = workspace.Status.Capacity.DeepCopy() + if workspace.Status.Conditions != nil { + original.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) + } + workspace.Status.ObservedGeneration = workspace.Generation + workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed + meta.SetStatusCondition(&workspace.Status.Conditions, condition(conditionType, metav1.ConditionFalse, reason, message, workspace.Generation)) + if conditionType != "Ready" { + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, reason, message, workspace.Generation)) + } + if reflect.DeepEqual(original, workspace.Status) { + return nil + } + return r.Status().Update(ctx, workspace) +} + +func workspaceRequestsForPVC(_ context.Context, object client.Object) []ctrl.Request { + pvc, ok := object.(*corev1.PersistentVolumeClaim) + if !ok || pvc.Namespace == "" || pvc.Labels[workspacePVCPartOfLabel] != workspacePVCPartOfValue { + return nil + } + workspaceName := pvc.Labels[workspacePVCWorkspaceLabel] + workspaceUID := pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] + if workspaceName == "" || workspaceUID == "" { + return nil + } + workspaceIdentity := &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceName, UID: types.UID(workspaceUID)}} + if pvc.Name != WorkspacePVCName(workspaceIdentity) { + return nil + } + return []ctrl.Request{{NamespacedName: types.NamespacedName{Namespace: pvc.Namespace, Name: workspaceName}}} +} + +func indexHostByStorageClass(object client.Object) []string { + host, ok := object.(*clusterv1alpha1.T4ClusterHost) + if !ok || host.Spec.StorageClassName == "" { + return nil + } + return []string{host.Spec.StorageClassName} +} + +func indexWorkspaceByHostRef(object client.Object) []string { + workspace, ok := object.(*clusterv1alpha1.T4Workspace) + if !ok || workspace.Spec.HostRef == "" { + return nil + } + return []string{workspace.Spec.HostRef} +} + +func (r *WorkspaceReconciler) workspaceRequestsForStorageClass(ctx context.Context, object client.Object) []ctrl.Request { + storageClass, ok := object.(*storagev1.StorageClass) + if !ok || storageClass.Name == "" { + return nil + } + var hosts clusterv1alpha1.T4ClusterHostList + if err := r.List(ctx, &hosts, client.MatchingFields{hostStorageClassIndexField: storageClass.Name}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "unable to map StorageClass to cluster hosts", "storageClass", storageClass.Name) + return nil + } + requests := make([]ctrl.Request, 0) + for i := range hosts.Items { + host := &hosts.Items[i] + var workspaces clusterv1alpha1.T4WorkspaceList + if err := r.List(ctx, &workspaces, client.InNamespace(host.Namespace), client.MatchingFields{workspaceHostRefIndexField: host.Name}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "unable to map cluster host to workspaces", "clusterHost", client.ObjectKeyFromObject(host)) + continue + } + for j := range workspaces.Items { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(&workspaces.Items[j])}) + } + } + return requests +} + +func (r *WorkspaceReconciler) SetupWithManager(manager ctrl.Manager) error { + if err := manager.GetFieldIndexer().IndexField(context.Background(), &clusterv1alpha1.T4ClusterHost{}, hostStorageClassIndexField, indexHostByStorageClass); err != nil { + return fmt.Errorf("index T4ClusterHost by StorageClass: %w", err) + } + if err := manager.GetFieldIndexer().IndexField(context.Background(), &clusterv1alpha1.T4Workspace{}, workspaceHostRefIndexField, indexWorkspaceByHostRef); err != nil { + return fmt.Errorf("index T4Workspace by host reference: %w", err) + } + return ctrl.NewControllerManagedBy(manager). + For(&clusterv1alpha1.T4Workspace{}). + Watches(&corev1.PersistentVolumeClaim{}, handler.EnqueueRequestsFromMapFunc(workspaceRequestsForPVC)). + Watches(&storagev1.StorageClass{}, handler.EnqueueRequestsFromMapFunc(r.workspaceRequestsForStorageClass)). + Complete(r) +} diff --git a/packages/cluster-operator/controllers/workspace_controller_test.go b/packages/cluster-operator/controllers/workspace_controller_test.go new file mode 100644 index 00000000..412705d0 --- /dev/null +++ b/packages/cluster-operator/controllers/workspace_controller_test.go @@ -0,0 +1,109 @@ +package controllers + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" +) + +func TestWorkspaceRequestsForPVC(t *testing.T) { + workspace := &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{ + Name: "workspace-a", Namespace: "team", UID: "workspace-uid", + }} + validPVC := func() *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: WorkspacePVCName(workspace), + Namespace: workspace.Namespace, + Labels: map[string]string{ + workspacePVCPartOfLabel: workspacePVCPartOfValue, + workspacePVCWorkspaceLabel: workspace.Name, + }, + Annotations: map[string]string{ + clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID), + }, + }} + } + + tests := []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + wantKey *types.NamespacedName + }{ + { + name: "retained workspace PVC without owner reference", + mutate: func(*corev1.PersistentVolumeClaim) {}, + wantKey: &types.NamespacedName{Namespace: workspace.Namespace, Name: workspace.Name}, + }, + {name: "unrelated PVC", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Labels = nil }}, + {name: "wrong part-of label", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Labels[workspacePVCPartOfLabel] = "other-system" }}, + {name: "missing workspace label", mutate: func(pvc *corev1.PersistentVolumeClaim) { delete(pvc.Labels, workspacePVCWorkspaceLabel) }}, + {name: "missing workspace UID", mutate: func(pvc *corev1.PersistentVolumeClaim) { + delete(pvc.Annotations, clusterv1alpha1.WorkspaceUIDAnnotation) + }}, + {name: "wrong deterministic name", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Name = "unrelated" }}, + {name: "missing namespace", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Namespace = "" }}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pvc := validPVC() + test.mutate(pvc) + requests := workspaceRequestsForPVC(context.Background(), pvc) + if test.wantKey == nil { + if len(requests) != 0 { + t.Fatalf("requests = %#v, want none", requests) + } + return + } + if len(requests) != 1 || requests[0].NamespacedName != *test.wantKey { + t.Fatalf("requests = %#v, want exactly %v", requests, *test.wantKey) + } + }) + } +} + +func TestWorkspaceRequestsForStorageClassOnlyEnqueuesAffectedWorkspaces(t *testing.T) { + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{storagev1.AddToScheme, clusterv1alpha1.AddToScheme} { + if err := add(scheme); err != nil { + t.Fatal(err) + } + } + objects := []client.Object{ + &clusterv1alpha1.T4ClusterHost{ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team"}, Spec: clusterv1alpha1.T4ClusterHostSpec{StorageClassName: "portable-rwx"}}, + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-a", Namespace: "team"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-a"}}, + &clusterv1alpha1.T4ClusterHost{ObjectMeta: metav1.ObjectMeta{Name: "host-b", Namespace: "other"}, Spec: clusterv1alpha1.T4ClusterHostSpec{StorageClassName: "portable-rwx"}}, + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-b", Namespace: "other"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-b"}}, + &clusterv1alpha1.T4ClusterHost{ObjectMeta: metav1.ObjectMeta{Name: "unrelated-host", Namespace: "team"}, Spec: clusterv1alpha1.T4ClusterHostSpec{StorageClassName: "other-class"}}, + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "unrelated-workspace", Namespace: "team"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "unrelated-host"}}, + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithIndex(&clusterv1alpha1.T4ClusterHost{}, hostStorageClassIndexField, indexHostByStorageClass). + WithIndex(&clusterv1alpha1.T4Workspace{}, workspaceHostRefIndexField, indexWorkspaceByHostRef). + WithObjects(objects...).Build() + r := &WorkspaceReconciler{Client: c, Scheme: scheme} + + requests := r.workspaceRequestsForStorageClass(context.Background(), &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "portable-rwx"}}) + got := make(map[types.NamespacedName]int, len(requests)) + for _, request := range requests { + got[request.NamespacedName]++ + } + want := []types.NamespacedName{{Namespace: "team", Name: "workspace-a"}, {Namespace: "other", Name: "workspace-b"}} + if len(got) != len(want) { + t.Fatalf("requests = %#v, want exactly %v", requests, want) + } + for _, key := range want { + if got[key] != 1 { + t.Fatalf("requests = %#v, want %v exactly once", requests, key) + } + } +} diff --git a/packages/cluster-operator/go.mod b/packages/cluster-operator/go.mod new file mode 100644 index 00000000..83e424b3 --- /dev/null +++ b/packages/cluster-operator/go.mod @@ -0,0 +1,65 @@ +module github.com/LycaonLLC/t4-code/packages/cluster-operator + +go 1.24.0 + +require ( + github.com/prometheus/client_golang v1.22.0 + k8s.io/api v0.33.2 + k8s.io/apiextensions-apiserver v0.33.2 + k8s.io/apimachinery v0.33.2 + k8s.io/client-go v0.33.2 + sigs.k8s.io/controller-runtime v0.21.0 + sigs.k8s.io/yaml v1.4.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect +) diff --git a/packages/cluster-operator/go.sum b/packages/cluster-operator/go.sum new file mode 100644 index 00000000..e291ac10 --- /dev/null +++ b/packages/cluster-operator/go.sum @@ -0,0 +1,197 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.33.2 h1:YgwIS5jKfA+BZg//OQhkJNIfie/kmRsO0BmNaVSimvY= +k8s.io/api v0.33.2/go.mod h1:fhrbphQJSM2cXzCWgqU29xLDuks4mu7ti9vveEnpSXs= +k8s.io/apiextensions-apiserver v0.33.2 h1:6gnkIbngnaUflR3XwE1mCefN3YS8yTD631JXQhsU6M8= +k8s.io/apiextensions-apiserver v0.33.2/go.mod h1:IvVanieYsEHJImTKXGP6XCOjTwv2LUMos0YWc9O+QP8= +k8s.io/apimachinery v0.33.2 h1:IHFVhqg59mb8PJWTLi8m1mAoepkUNYmptHsV+Z1m5jY= +k8s.io/apimachinery v0.33.2/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/client-go v0.33.2 h1:z8CIcc0P581x/J1ZYf4CNzRKxRvQAwoAolYPbtQes+E= +k8s.io/client-go v0.33.2/go.mod h1:9mCgT4wROvL948w6f6ArJNb7yQd7QsvqavDeZHvNmHo= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= +sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/packages/cluster-server/package.json b/packages/cluster-server/package.json new file mode 100644 index 00000000..7b7b3a24 --- /dev/null +++ b/packages/cluster-server/package.json @@ -0,0 +1,28 @@ +{ + "name": "@t4-code/cluster-server", + "version": "0.1.30", + "private": true, + "type": "module", + "description": "Stateless Kubernetes-backed omp-app/1 cluster gateway", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { ".": { "types": "./src/index.ts", "import": "./src/index.ts" }, "./*": { "types": "./src/*.ts", "import": "./src/*.ts" } }, + "scripts": { + "build": "tsgo -p tsconfig.json --noEmit", + "check": "tsgo -p tsconfig.json --noEmit", + "test": "bun --bun run test:vp", + "test:vp": "vp test run", + "typecheck": "tsgo -p tsconfig.json --noEmit" + }, + "dependencies": { + "@t4-code/host-service": "workspace:*", + "@t4-code/host-wire": "workspace:*" + }, + "devDependencies": { + "@types/bun": "1.3.5", + "@types/node": "24.12.4", + "typescript": "~6.0.3", + "vite-plus": "0.2.2" + }, + "engines": { "bun": ">=1.3.14" } +} diff --git a/packages/cluster-server/src/ci-projection-runner.ts b/packages/cluster-server/src/ci-projection-runner.ts new file mode 100644 index 00000000..dd0aec3a --- /dev/null +++ b/packages/cluster-server/src/ci-projection-runner.ts @@ -0,0 +1,101 @@ +import type { SessionCiState } from "@t4-code/host-wire"; +import type { ClusterInfrastructureProjection, SessionCiCorrelation } from "./kubernetes-projection.ts"; +import type { CiProvider } from "./woodpecker.ts"; + +export interface CiProjectionRunnerOptions { + readonly projection: ClusterInfrastructureProjection; + readonly provider: CiProvider; + readonly pollMs?: number; + readonly onError?: (error: unknown) => void; +} + +/** Reconstructs bounded CI observations from the configured provider on every replica. */ +export class CiProjectionRunner { + readonly #projection: ClusterInfrastructureProjection; + readonly #provider: CiProvider; + readonly #pollMs: number; + readonly #onError?: (error: unknown) => void; + #timer: ReturnType | undefined; + #inflight: Promise | undefined; + #unsubscribe: (() => void) | undefined; + #correlationFingerprint = ""; + #started = false; + + constructor(options: CiProjectionRunnerOptions) { + this.#projection = options.projection; + this.#provider = options.provider; + this.#pollMs = options.pollMs ?? 5_000; + if (!Number.isSafeInteger(this.#pollMs) || this.#pollMs < 10 || this.#pollMs > 60_000) + throw new Error("CI projection poll interval is invalid"); + this.#onError = options.onError; + } + + start(): void { + if (this.#started) throw new Error("CI projection runner already started"); + this.#started = true; + this.#unsubscribe = this.#projection.subscribeSessions(() => this.#scheduleWhenCorrelationsChange()); + this.#correlationFingerprint = this.#fingerprint(this.#projection.sessionCiCorrelations()); + this.#schedule(0); + } + + async stop(): Promise { + if (!this.#started) return; + this.#started = false; + this.#unsubscribe?.(); + this.#unsubscribe = undefined; + clearTimeout(this.#timer); + this.#timer = undefined; + await this.#inflight; + } + + #fingerprint(correlations: readonly SessionCiCorrelation[]): string { + return JSON.stringify([...correlations].sort((left, right) => left.sessionId.localeCompare(right.sessionId))); + } + + #scheduleWhenCorrelationsChange(): void { + const fingerprint = this.#fingerprint(this.#projection.sessionCiCorrelations()); + if (fingerprint === this.#correlationFingerprint) return; + this.#correlationFingerprint = fingerprint; + this.#schedule(0); + } + + #schedule(delayMs: number): void { + if (!this.#started) return; + clearTimeout(this.#timer); + this.#timer = setTimeout(() => { + this.#timer = undefined; + if (this.#inflight) return; + const operation = this.#refresh(); + this.#inflight = operation; + const finish = (): void => { + if (this.#inflight === operation) this.#inflight = undefined; + this.#schedule(this.#pollMs); + }; + void operation.then(finish, error => { + this.#onError?.(error); + finish(); + }); + }, delayMs); + } + + async #refresh(): Promise { + const correlations = this.#projection.sessionCiCorrelations(); + this.#correlationFingerprint = this.#fingerprint(correlations); + await Promise.all(correlations.map(async correlation => { + let state: SessionCiState; + try { + state = await this.#provider.query(correlation); + this.#projection.setSessionCiState(correlation.sessionId, state); + } catch (error) { + this.#projection.setSessionCiState(correlation.sessionId, { + provider: "woodpecker", + correlation: "unknown", + repositoryId: correlation.repositoryId, + ref: correlation.ref, + commit: correlation.commit, + }); + this.#onError?.(error); + } + })); + } +} diff --git a/packages/cluster-server/src/config.ts b/packages/cluster-server/src/config.ts new file mode 100644 index 00000000..addb6e9e --- /dev/null +++ b/packages/cluster-server/src/config.ts @@ -0,0 +1,200 @@ +import { open, type FileHandle } from "node:fs/promises"; +import { isAbsolute } from "node:path"; +import { isIP } from "node:net"; + +const DNS_LABEL = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; +const DNS_SUBDOMAIN = /^(?:[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?)(?:\.[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?)*$/u; +const AUDIENCE = /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/u; +export interface ClusterServerConfig { + readonly namespace: string; + readonly podName: string; + readonly epoch: string; + readonly hostName: string; + readonly gatewayPort: number; + readonly adminPort: number; + readonly kubernetesBaseUrl: string; + readonly kubernetesTokenPath: string; + readonly kubernetesCaPath: string; + readonly kubernetesApiAudience: string; + readonly identityTokenPath: string; + readonly serverServiceAccountName: string; + readonly trustedProxyAddresses: readonly string[]; + readonly trustedProxyCidrs: readonly string[]; + readonly woodpecker?: { + readonly baseUrl: string; + readonly webBaseUrl?: string; + readonly repositories: Readonly>; + readonly token?: string; + readonly tokenFile?: string; + }; +} +function required(env: Readonly>, name: string): string { + const value = env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} +function dns(value: string, name: string): string { + if (!DNS_LABEL.test(value)) throw new Error(`${name} is invalid`); + return value; +} +function dnsSubdomain(value: string, name: string): string { + if (value.length > 253 || !DNS_SUBDOMAIN.test(value)) throw new Error(`${name} is invalid`); + return value; +} +function audience(value: string, name: string): string { + if (value.length > 253 || !AUDIENCE.test(value)) throw new Error(`${name} is invalid`); + return value; +} +function port(value: string | undefined, fallback: number, name: string): number { + const result = Number(value ?? fallback); + if (!Number.isSafeInteger(result) || result < 1 || result > 65_535) throw new Error(`${name} is invalid`); + return result; +} +function absolutePath(value: string, name: string): string { + if (!isAbsolute(value)) throw new Error(`${name} must be absolute`); + return value; +} +function repositories(value: string): Readonly> { + if (new TextEncoder().encode(value).byteLength > 65_536) throw new Error("T4_WOODPECKER_REPOSITORIES exceeds limit"); + let input: unknown; + try { input = JSON.parse(value); } catch { throw new Error("T4_WOODPECKER_REPOSITORIES is invalid JSON"); } + if (!input || typeof input !== "object" || Array.isArray(input) || Object.keys(input).length > 128) + throw new Error("T4_WOODPECKER_REPOSITORIES is invalid"); + const output: Record = {}; + for (const [id, raw] of Object.entries(input as Record)) { + if (!raw || typeof raw !== "object" || Array.isArray(raw) || Object.keys(raw).length !== 1 || typeof (raw as Record).slug !== "string") + throw new Error("T4_WOODPECKER_REPOSITORIES entry is invalid"); + output[id] = { slug: (raw as { slug: string }).slug }; + } + return Object.freeze(output); +} +function proxyAddresses(value: string | undefined): readonly string[] { + if (!value) return []; + const values = [...new Set(value.split(",").map(item => item.trim()))]; + if (values.length > 64 || values.some(item => !item || isIP(item) === 0)) + throw new Error("T4_CLUSTER_TRUSTED_PROXY_ADDRESSES is invalid"); + return Object.freeze(values); +} +function ipv6Value(value: string): bigint { + const halves = value.split("::"); + if (halves.length > 2) throw new Error("invalid IPv6 address"); + const left = halves[0] ? halves[0].split(":") : []; + const right = halves[1] ? halves[1].split(":") : []; + const words = [...left, ...Array(8 - left.length - right.length).fill("0"), ...right]; + if (words.length !== 8) throw new Error("invalid IPv6 address"); + return words.reduce((result, word) => (result << 16n) | BigInt(Number.parseInt(word, 16)), 0n); +} +function canonicalCidr(value: string): string { + const pieces = value.split("/"); + if (pieces.length !== 2 || !/^(?:0|[1-9][0-9]*)$/u.test(pieces[1]!)) + throw new Error("T4_CLUSTER_TRUSTED_PROXY_CIDRS is invalid"); + const address = pieces[0]!; + const family = isIP(address); + const prefix = Number(pieces[1]); + const width = family === 4 ? 32 : family === 6 ? 128 : 0; + if (prefix < 1 || prefix > width) throw new Error("T4_CLUSTER_TRUSTED_PROXY_CIDRS is invalid"); + const canonicalAddress = family === 4 + ? address.split(".").map(Number).join(".") + : new URL(`http://[${address}]/`).hostname.slice(1, -1); + if (canonicalAddress !== address.toLowerCase()) throw new Error("T4_CLUSTER_TRUSTED_PROXY_CIDRS must use canonical network addresses"); + const numeric = family === 4 + ? address.split(".").map(Number).reduce((result, octet) => (result << 8n) | BigInt(octet), 0n) + : ipv6Value(address); + const hostBits = BigInt(width - prefix); + if (hostBits > 0n && (numeric & ((1n << hostBits) - 1n)) !== 0n) + throw new Error("T4_CLUSTER_TRUSTED_PROXY_CIDRS must use canonical network addresses"); + return `${canonicalAddress}/${prefix}`; +} +function proxyCidrs(value: string | undefined): readonly string[] { + if (!value) return []; + const values = [...new Set(value.split(",").map(item => canonicalCidr(item.trim())))]; + if (values.length > 64) throw new Error("T4_CLUSTER_TRUSTED_PROXY_CIDRS exceeds limit"); + return Object.freeze(values); +} + + +export function clusterServerConfigFromEnv(env: Readonly>): ClusterServerConfig { + const namespace = dns(required(env, "POD_NAMESPACE"), "POD_NAMESPACE"); + const podName = dns(required(env, "POD_NAME"), "POD_NAME"); + const podUid = required(env, "POD_UID"); + if (!/^[A-Za-z0-9-]{8,128}$/u.test(podUid)) throw new Error("POD_UID is invalid"); + const serviceHost = required(env, "KUBERNETES_SERVICE_HOST"); + const servicePort = port(env.KUBERNETES_SERVICE_PORT_HTTPS ?? env.KUBERNETES_SERVICE_PORT, 443, "KUBERNETES_SERVICE_PORT"); + const identityTokenPath = absolutePath(required(env, "T4_CLUSTER_IDENTITY_TOKEN_FILE"), "T4_CLUSTER_IDENTITY_TOKEN_FILE"); + const serverServiceAccountName = dns(required(env, "T4_CLUSTER_SERVER_SERVICE_ACCOUNT"), "T4_CLUSTER_SERVER_SERVICE_ACCOUNT"); + const woodpeckerBaseUrl = env.T4_WOODPECKER_BASE_URL; + const woodpeckerWebBaseUrl = env.T4_WOODPECKER_WEB_BASE_URL; + const woodpeckerRepositories = env.T4_WOODPECKER_REPOSITORIES; + const woodpeckerToken = env.T4_WOODPECKER_TOKEN; + const woodpeckerTokenFile = env.T4_WOODPECKER_TOKEN_FILE; + if (woodpeckerToken && woodpeckerTokenFile) throw new Error("Woodpecker configuration requires exactly one credential source"); + const woodpeckerConfigured = Boolean(woodpeckerBaseUrl || woodpeckerRepositories || woodpeckerToken || woodpeckerTokenFile); + if (woodpeckerConfigured && (!woodpeckerBaseUrl || !woodpeckerRepositories || !(woodpeckerToken || woodpeckerTokenFile))) + throw new Error("Woodpecker configuration must be complete"); + return { + namespace, + podName, + epoch: `replica:${podUid}`, + hostName: dnsSubdomain(required(env, "T4_CLUSTER_HOST_NAME"), "T4_CLUSTER_HOST_NAME"), + gatewayPort: port(env.T4_CLUSTER_SERVER_PORT, 8080, "T4_CLUSTER_SERVER_PORT"), + adminPort: port(env.T4_CLUSTER_ADMIN_PORT, 9090, "T4_CLUSTER_ADMIN_PORT"), + trustedProxyAddresses: proxyAddresses(env.T4_CLUSTER_TRUSTED_PROXY_ADDRESSES), + trustedProxyCidrs: proxyCidrs(env.T4_CLUSTER_TRUSTED_PROXY_CIDRS), + kubernetesBaseUrl: `https://${serviceHost}:${servicePort}`, + kubernetesTokenPath: absolutePath(env.T4_KUBERNETES_TOKEN_PATH ?? "/var/run/secrets/kubernetes.io/serviceaccount/token", "T4_KUBERNETES_TOKEN_PATH"), + kubernetesCaPath: absolutePath(env.T4_KUBERNETES_CA_PATH ?? "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", "T4_KUBERNETES_CA_PATH"), + kubernetesApiAudience: audience(env.T4_KUBERNETES_API_AUDIENCE ?? "https://kubernetes.default.svc", "T4_KUBERNETES_API_AUDIENCE"), + identityTokenPath, + serverServiceAccountName, + ...(woodpeckerConfigured ? { + woodpecker: { + baseUrl: woodpeckerBaseUrl!, + ...(woodpeckerWebBaseUrl ? { webBaseUrl: woodpeckerWebBaseUrl } : {}), + repositories: repositories(woodpeckerRepositories!), + ...(woodpeckerToken ? { token: woodpeckerToken } : { tokenFile: absolutePath(woodpeckerTokenFile!, "T4_WOODPECKER_TOKEN_FILE") }), + }, + } : {}), + }; +} +export async function readBoundedRegularFile(path: string, maximumBytes: number, description: string): Promise { + const invalid = `${description} file is invalid`; + let file: FileHandle | undefined; + try { + file = await open(path, "r"); + const metadata = await file.stat(); + if (!metadata.isFile() || metadata.size < 1 || metadata.size > maximumBytes) throw new Error(invalid); + const buffer = Buffer.allocUnsafe(metadata.size + 1); + let bytesRead = 0; + while (bytesRead < buffer.byteLength) { + const result = await file.read(buffer, bytesRead, buffer.byteLength - bytesRead, bytesRead); + if (result.bytesRead === 0) break; + bytesRead += result.bytesRead; + } + if (bytesRead < 1 || bytesRead > metadata.size || bytesRead > maximumBytes) throw new Error(invalid); + try { + return new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, bytesRead)); + } catch { + throw new Error(invalid); + } + } catch (error) { + if (error instanceof Error && error.message === invalid) throw error; + throw new Error(invalid); + } finally { + await file?.close().catch(() => undefined); + } +} +export async function readClusterIdentityToken(path: string): Promise { + const token = (await readBoundedRegularFile(path, 16_384, "cluster identity token")).trim(); + if (new TextEncoder().encode(token).byteLength < 32 || /\s/u.test(token)) throw new Error("cluster identity token file is invalid"); + return token; +} +export async function readKubernetesToken(path: string): Promise { + const token = (await readBoundedRegularFile(path, 16_384, "Kubernetes token")).trim(); + if (!token || /\s/u.test(token)) throw new Error("Kubernetes token file is invalid"); + return token; +} +export async function loadKubernetesCa(config: ClusterServerConfig): Promise { + const ca = await readBoundedRegularFile(config.kubernetesCaPath, 1024 * 1024, "Kubernetes CA"); + if (!ca.includes("BEGIN CERTIFICATE")) throw new Error("Kubernetes CA file is invalid"); + return ca; +} diff --git a/packages/cluster-server/src/gateway.ts b/packages/cluster-server/src/gateway.ts new file mode 100644 index 00000000..76b3f5e0 --- /dev/null +++ b/packages/cluster-server/src/gateway.ts @@ -0,0 +1,395 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + CI_TRIGGER_CAPABILITY, + CLUSTER_OPERATOR_FEATURE, + commandId, + decodeClientFrame, + parseBounded, + requiredCapability, + type CiRunArguments, + type ConfirmFrame, + type ClientFrame, + type ClusterSessionCreateArguments, + type ClusterWorkspaceCreateArguments, + type CommandFrame, + type ResultFrame, + type ServerFrame, +} from "@t4-code/host-wire"; +import type { Revision } from "@t4-code/host-wire"; +import { commandFeature, IdempotencyStore, type CommandOutcome } from "@t4-code/host-service"; +import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; +import { + rewriteClientAddress, + rewriteServerAddress, + type PodHostConnection, + type PodHostConnector, + type PodHostRoute, +} from "./pod-host-router.ts"; +import type { CiProvider } from "./woodpecker.ts"; +import { KubernetesApiError } from "./kubernetes-client.ts"; + +const GATEWAY_CAPABILITIES = [ + "sessions.read", "sessions.prompt", "sessions.control", "sessions.manage", "agents.control", + "bash.run", "files.read", "files.write", "files.list", "files.diff", + "preview.read", "preview.control", "preview.input", "term.open", "term.input", "term.resize", CI_TRIGGER_CAPABILITY, +] as const; +const GATEWAY_FEATURES = [ + "resume", "session.state", "session.delta", "session.observer", "controller.lease", "prompt.lease", "prompt.images", + "transcript.images", "transcript.search", "transcript.page", "agent.lifecycle", "agent.progress", "agent.event", + "agent.transcript", "terminal.io", "files.list", "files.diff", "preview.control", CLUSTER_OPERATOR_FEATURE, +] as const; + +export interface GatewayClient { + send(frame: ServerFrame): void; + close(code?: number, reason?: string): void; +} +export interface GatewayMutationBackend { + createWorkspace(commandId: string, args: ClusterWorkspaceCreateArguments, principal: string): Promise<{ id: string; revision: string }>; + createSession(commandId: string, args: ClusterSessionCreateArguments, principal: string): Promise<{ sessionId: string; revision: string }>; + deleteSession(commandId: string, sessionId: string, principal: string): Promise<{ deleted: true }>; +} +export interface ClusterGatewayOptions { + readonly projection: ClusterInfrastructureProjection; + readonly connector: PodHostConnector; + readonly mutations: GatewayMutationBackend; + readonly ciProvider?: CiProvider; + readonly appserverVersion?: string; + readonly appserverBuild?: string; +} +export interface GatewayConnection { + receive(frame: unknown): Promise; + close(): void; + drain(): void; +} + +function errorResult(command: CommandFrame, code: string, message: string): ResultFrame { + return { + v: "omp-app/1", type: "response", requestId: command.requestId, commandId: command.commandId, + hostId: command.hostId, ...(command.sessionId ? { sessionId: command.sessionId } : {}), ok: false, + error: { code, message }, + }; +} +function successResult(command: CommandFrame, result: unknown): ResultFrame { + return { + v: "omp-app/1", type: "response", requestId: command.requestId, commandId: command.commandId, + hostId: command.hostId, ...(command.sessionId ? { sessionId: command.sessionId } : {}), ok: true, + command: command.command, result, + }; +} +function replayForRequest(frame: ServerFrame, command: CommandFrame): ServerFrame { + return frame.type === "response" ? { ...frame, requestId: command.requestId, commandId: command.commandId } : frame; +} + +interface PendingConfirmation { readonly command: CommandFrame; readonly expiresAt: number; } +interface RoutedPodConnection { readonly route: PodHostRoute; readonly pending: Promise; } + +function validPrincipal(value: string): boolean { + return value.length > 0 && new TextEncoder().encode(value).byteLength <= 256 && !/\p{Cc}/u.test(value) && value === value.trim(); +} +export class ClusterGateway { + readonly #projection: ClusterInfrastructureProjection; + readonly #connector: PodHostConnector; + readonly #mutations: GatewayMutationBackend; + readonly #ci?: CiProvider; + readonly #idempotency = new IdempotencyStore({ maxCompletedEntries: 1_024, completedTtlMs: 5 * 60_000 }); + readonly #connections = new Set(); + readonly #version: string; + readonly #build: string; + #draining = false; + + constructor(options: ClusterGatewayOptions) { + this.#projection = options.projection; + this.#connector = options.connector; + this.#mutations = options.mutations; + this.#ci = options.ciProvider; + this.#version = options.appserverVersion ?? "0.1.30"; + this.#build = options.appserverBuild ?? "cluster"; + } + get connectionCount(): number { return this.#connections.size; } + get draining(): boolean { return this.#draining; } + beginDrain(): void { + this.#draining = true; + for (const connection of this.#connections) connection.drain(); + } + connect(client: GatewayClient, principal: string): GatewayConnection { + if (!validPrincipal(principal)) { + client.close(1008, "authenticated gateway identity required"); + return { receive: async () => undefined, close: () => undefined, drain: () => undefined }; + } + if (this.#draining) { + client.close(1012, "cluster server draining"); + return { receive: async () => undefined, close: () => undefined, drain: () => undefined }; + } + let helloReceived = false; + let operatorEnabled = false; + let grantedCapabilities = new Set(); + let grantedFeatures = new Set(); + let unsubscribeWorkspaces: (() => void) | undefined; + let unsubscribeSessions: (() => void) | undefined; + const challenges = new Map(); + const upstream = new Map(); + const previewOwners = new Map(); + let closed = false; + const sendSessions = (): void => { + const sessions = this.#projection.sessionRefs(principal); + client.send({ + v: "omp-app/1", type: "sessions", hostId: this.#projection.hostId, + cursor: this.#projection.sessionCursor, sessions, + totalCount: sessions.length, truncated: false, + }); + for (const [session, cached] of upstream) { + const current = this.#projection.sessionRoute(session, principal); + if (current?.url === cached.route.url && current.upstreamSessionId === cached.route.upstreamSessionId) continue; + upstream.delete(session); + void cached.pending.then(connection => connection.close(1001, "session route changed"), () => undefined); + } + }; + const close = (): void => { + if (closed) return; + closed = true; + unsubscribeWorkspaces?.(); + unsubscribeSessions?.(); + challenges.clear(); + for (const cached of upstream.values()) void cached.pending.then(value => value.close(1001, "gateway client closed"), () => undefined); + upstream.clear(); + this.#connections.delete(connection); + }; + const idempotent = async (command: CommandFrame, operation: () => Promise): Promise => { + const scopedId = commandId(createHash("sha256").update(principal).update("\u0000").update(command.commandId).digest("base64url")); + const state = this.#idempotency.begin(scopedId, command); + if (state.kind === "conflict") { client.send(errorResult(command, "idempotency_conflict", "command id was reused with another payload")); return; } + if (state.kind === "replay" || state.kind === "pending") { + const outcome = state.kind === "replay" ? state.outcome : await state.outcome; + client.send(replayForRequest(outcome.frame, command)); + return; + } + let output: ServerFrame; + try { output = await operation(); } + catch (error) { + output = error instanceof KubernetesApiError && error.status === 422 + ? errorResult(command, "INVALID_FRAME", "cluster request did not satisfy the Kubernetes API contract") + : errorResult(command, "UPSTREAM_UNAVAILABLE", "cluster operation failed"); + } + const outcome: CommandOutcome = { frame: output }; + this.#idempotency.complete(scopedId, command, outcome); + client.send(output); + }; + const route = async (frame: ClientFrame): Promise => { + const clusterSessionId = "sessionId" in frame && typeof frame.sessionId === "string" ? frame.sessionId : undefined; + if (!clusterSessionId) { if (frame.type === "command") client.send(errorResult(frame, "INVALID_FRAME", "session route is required")); return; } + const selected = this.#projection.sessionRoute(clusterSessionId, principal); + if (!selected) { if (frame.type === "command") client.send(errorResult(frame, "NOT_AUTHORIZED", "session is unavailable for this identity")); return; } + let cached = upstream.get(clusterSessionId); + if (cached && (cached.route.url !== selected.url || cached.route.upstreamSessionId !== selected.upstreamSessionId)) { + upstream.delete(clusterSessionId); + void cached.pending.then(connection => connection.close(1001, "session route changed"), () => undefined); + cached = undefined; + } + if (!cached) { + let pending: Promise; + pending = this.#connector.connect( + selected, + upstreamFrame => { + if (upstreamFrame.type === "sessions") return; + const current = this.#projection.sessionRoute(clusterSessionId, principal); + if (current?.url !== selected.url || current.upstreamSessionId !== selected.upstreamSessionId) return; + const previewId = (upstreamFrame as unknown as Record).previewId; + if (upstreamFrame.type.startsWith("preview.") && typeof previewId === "string") previewOwners.set(previewId, clusterSessionId); + client.send(rewriteServerAddress(upstreamFrame, selected, this.#projection.hostId)); + }, + () => { if (upstream.get(clusterSessionId)?.pending === pending) upstream.delete(clusterSessionId); }, + ); + cached = { route: selected, pending }; + upstream.set(clusterSessionId, cached); + pending.catch(() => { if (upstream.get(clusterSessionId)?.pending === pending) upstream.delete(clusterSessionId); }); + } + try { + const socket = await cached.pending; + const current = this.#projection.sessionRoute(clusterSessionId, principal); + if (!current + || current.clusterSessionId !== selected.clusterSessionId + || current.url !== selected.url + || current.upstreamSessionId !== selected.upstreamSessionId) { + if (upstream.get(clusterSessionId)?.pending === cached.pending) upstream.delete(clusterSessionId); + socket.close(1001, "session route changed"); + if (frame.type === "command") { + client.send(current + ? errorResult(frame, "UPSTREAM_UNAVAILABLE", "session pod route changed while connecting") + : errorResult(frame, "NOT_AUTHORIZED", "session is unavailable for this identity")); + } + return; + } + socket.send(rewriteClientAddress(frame, selected, socket.hostId ?? "upstream")); + } catch { + if (frame.type === "command") client.send(errorResult(frame, "UPSTREAM_UNAVAILABLE", "session pod host connection failed")); + } + }; + const confirm = async (frame: ConfirmFrame): Promise => { + const pending = challenges.get(String(frame.confirmationId)); + if (!pending) return false; + const command = { ...pending.command, requestId: frame.requestId } as CommandFrame; + if (pending.expiresAt < Date.now() || pending.command.commandId !== frame.commandId || pending.command.hostId !== frame.hostId || pending.command.sessionId !== frame.sessionId) { + challenges.delete(String(frame.confirmationId)); + client.send(errorResult(command, "confirmation_invalid", "confirmation is invalid or expired")); + return true; + } + challenges.delete(String(frame.confirmationId)); + if (frame.decision === "deny") { + client.send(errorResult(command, "confirmation_denied", "command was denied")); + return true; + } + await idempotent(command, async () => { + const session = command.sessionId!; + if (!this.#projection.ownsSession(session, principal)) + return this.#projection.sessionExists(session) + ? errorResult(command, "NOT_AUTHORIZED", "session is unavailable for this identity") + : successResult(command, { deleted: true }); + if (this.#projection.sessionRevision(session, principal) !== command.expectedRevision) + return errorResult(command, "stale_revision", "session revision changed before deletion"); + return successResult(command, await this.#mutations.deleteSession(command.commandId, session, principal)); + }); + return true; + }; + const receive = async (input: unknown): Promise => { + if (closed) return; + let frame: ClientFrame; + try { frame = decodeClientFrame(typeof input === "string" || input instanceof Uint8Array ? parseBounded(input) : input); } + catch { client.close(1002, "invalid omp-app frame"); close(); return; } + if (frame.type === "hello") { + if (helloReceived) { client.close(1002, "duplicate hello"); close(); return; } + helloReceived = true; + operatorEnabled = frame.requestedFeatures.includes(CLUSTER_OPERATOR_FEATURE); + const requestedCapabilities = new Set(frame.capabilities?.client ?? []); + grantedCapabilities = new Set(GATEWAY_CAPABILITIES.filter(capability => requestedCapabilities.has(capability) && (capability !== CI_TRIGGER_CAPABILITY || this.#ci !== undefined))); + const requestedFeatures = new Set(frame.requestedFeatures); + grantedFeatures = new Set(GATEWAY_FEATURES.filter(feature => requestedFeatures.has(feature) && (feature !== CLUSTER_OPERATOR_FEATURE || operatorEnabled))); + client.send({ + v: "omp-app/1", type: "welcome", selectedProtocol: "omp-app/1", hostId: this.#projection.hostId, + ompVersion: "17.0.5", ompBuild: "8476f4451ed95c5d5401785d279a93d3c659fac4", + appserverVersion: this.#version, appserverBuild: this.#build, epoch: this.#projection.epoch, + grantedCapabilities: [...grantedCapabilities], grantedFeatures: [...grantedFeatures], + negotiatedLimits: { maxPayloadLength: 1_048_576, maxWorkspaces: 256, maxSessions: 1_000, workspaceReplayFrames: 512 }, + authentication: "paired", + resumed: frame.savedCursors.some(saved => saved.hostId === this.#projection.hostId && saved.cursor.epoch === this.#projection.epoch), + }); + if (!operatorEnabled || !grantedCapabilities.has("sessions.read")) return; + sendSessions(); + unsubscribeWorkspaces = this.#projection.subscribe(value => client.send(value), this.#projection.workspaceCursor, principal); + unsubscribeSessions = this.#projection.subscribeSessions(sendSessions); + return; + } + if (!helloReceived) { client.close(1002, "hello required"); close(); return; } + if (frame.type === "ping") { client.send({ v: "omp-app/1", type: "pong", nonce: frame.nonce, timestamp: frame.timestamp }); return; } + if (frame.type === "confirm" && await confirm(frame)) return; + if (frame.type !== "command") { + if (!operatorEnabled || !("hostId" in frame) || frame.hostId !== this.#projection.hostId) return; + if (frame.type === "terminal.input" && (!grantedFeatures.has("terminal.io") || !grantedCapabilities.has("term.input"))) return; + if (frame.type === "terminal.resize" && (!grantedFeatures.has("terminal.io") || !grantedCapabilities.has("term.resize"))) return; + if (frame.type === "terminal.close" && (!grantedFeatures.has("terminal.io") || !grantedCapabilities.has("term.open"))) return; + if (frame.type !== "confirm" && !frame.type.startsWith("terminal.")) return; + await route(frame); + return; + } + if (!operatorEnabled) { client.send(errorResult(frame, "UNSUPPORTED_FEATURE", "cluster.operator was not negotiated")); return; } + if (frame.hostId !== this.#projection.hostId) { client.send(errorResult(frame, "NOT_FOUND", "cluster host was not found")); return; } + const capability = requiredCapability(frame.command); + if (!capability || !grantedCapabilities.has(capability)) { client.send(errorResult(frame, "NOT_AUTHORIZED", "command capability was not granted")); return; } + if (frame.command === "session.list") { + const sessions = this.#projection.sessionRefs(principal); + client.send(successResult(frame, { cursor: this.#projection.sessionCursor, sessions, totalCount: sessions.length, truncated: false })); + return; + } + if (frame.command === "workspace.list") { client.send(successResult(frame, this.#projection.workspaceList(principal))); return; } + if (frame.command === "workspace.create") { + await idempotent(frame, async () => { + const args = frame.args as unknown as ClusterWorkspaceCreateArguments; + const created = await this.#mutations.createWorkspace(frame.commandId, args, principal); + return successResult(frame, { workspace: { + id: created.id, displayName: args.displayName, phase: "Pending", retentionPolicy: args.retentionPolicy, + capacity: args.capacity, + accessMode: "ReadWriteMany", revision: created.revision, + } }); + }); + return; + } + if (frame.command === "session.create") { + const args = frame.args as unknown as ClusterSessionCreateArguments; + if (!this.#projection.ownsWorkspace(args.workspaceId, principal)) { client.send(errorResult(frame, "NOT_AUTHORIZED", "workspace is unavailable for this identity")); return; } + await idempotent(frame, async () => { + const created = await this.#mutations.createSession(frame.commandId, args, principal); + const session = await this.#projection.waitForSessionAuthority(created.sessionId); + if (!this.#projection.ownsSession(created.sessionId, principal)) + return errorResult(frame, "NOT_AUTHORIZED", "session is unavailable for this identity"); + return successResult(frame, { session }); + }); + return; + } + if (frame.command === "session.delete") { + if (!frame.sessionId) { client.send(errorResult(frame, "INVALID_FRAME", "session route is required")); return; } + if (!this.#projection.ownsSession(frame.sessionId, principal)) { + client.send(this.#projection.sessionExists(frame.sessionId) + ? errorResult(frame, "NOT_AUTHORIZED", "session is unavailable for this identity") + : successResult(frame, { deleted: true })); + return; + } + if (this.#projection.sessionRevision(frame.sessionId, principal) !== frame.expectedRevision) { client.send(errorResult(frame, "stale_revision", "session revision changed before confirmation")); return; } + if (frame.confirmationId !== undefined) { client.send(errorResult(frame, "confirmation_invalid", "command confirmation must use a confirm frame")); return; } + for (const [id, pending] of challenges) if (pending.expiresAt < Date.now()) challenges.delete(id); + if (challenges.size >= 5) { client.send(errorResult(frame, "confirmation_unavailable", "confirmation capacity exceeded")); return; } + const confirmationId = randomUUID(); + const expiresAt = Date.now() + 60_000; + challenges.set(confirmationId, { command: frame, expiresAt }); + client.send({ + v: "omp-app/1", type: "confirmation", confirmationId: confirmationId as never, + commandId: frame.commandId, hostId: this.#projection.hostId, sessionId: frame.sessionId, + commandHash: createHash("sha256").update(JSON.stringify({ ...frame, confirmationId: undefined })).digest("hex"), + revision: frame.expectedRevision as Revision, + expiresAt: new Date(expiresAt).toISOString(), summary: "session.delete", + }); + return; + } + if (frame.command === "ci.run") { + await idempotent(frame, async () => { + if (!frame.sessionId || this.#projection.sessionRevision(frame.sessionId, principal) !== frame.expectedRevision) + return errorResult(frame, "stale_revision", "session revision changed before CI trigger"); + if (!this.#ci) return errorResult(frame, "UNSUPPORTED_FEATURE", "CI provider is unavailable"); + const args = frame.args as unknown as CiRunArguments; + const allowed = this.#projection.sessionCiSelection(frame.sessionId, principal); + if (!allowed || allowed.repositoryId !== args.repositoryId || allowed.ref !== args.ref || allowed.commit !== args.commit) + return errorResult(frame, "NOT_AUTHORIZED", "CI correlation is not declared by this session"); + return successResult(frame, await this.#ci.run({ commandId: frame.commandId, sessionId: frame.sessionId, repositoryId: args.repositoryId, ref: args.ref, commit: args.commit })); + }); + return; + } + const feature = commandFeature(frame.command); + if (feature && !grantedFeatures.has(feature)) { client.send(errorResult(frame, "UNSUPPORTED_FEATURE", "command feature was not negotiated")); return; } + if (frame.command.startsWith("preview.")) { + if (!frame.sessionId) { client.send(errorResult(frame, "INVALID_FRAME", "session route is required")); return; } + const guiState = this.#projection.sessionGuiState(frame.sessionId, principal); + if (guiState === undefined) { client.send(errorResult(frame, "NOT_AUTHORIZED", "session is unavailable for this identity")); return; } + if (guiState !== "Ready") { + client.send(errorResult(frame, guiState === "Unavailable" ? "UNSUPPORTED_FEATURE" : "UPSTREAM_UNAVAILABLE", guiState === "Unavailable" ? "GUI is disabled for this session" : "session GUI is not ready")); + return; + } + const preview = typeof frame.args.previewId === "string" ? frame.args.previewId : undefined; + if (preview && previewOwners.has(preview) && previewOwners.get(preview) !== frame.sessionId) { + client.send(errorResult(frame, "NOT_AUTHORIZED", "preview belongs to another session")); + return; + } + } + await route(frame); + }; + const connection: GatewayConnection = { + receive, + close, + drain: () => { + if (closed) return; + client.send({ v: "omp-app/1", type: "bye", code: "server_restart", reason: "cluster server draining", retryable: true }); + client.close(1012, "cluster server draining"); + close(); + }, + }; + this.#connections.add(connection); + return connection; + } +} diff --git a/packages/cluster-server/src/index.ts b/packages/cluster-server/src/index.ts new file mode 100644 index 00000000..4f81c495 --- /dev/null +++ b/packages/cluster-server/src/index.ts @@ -0,0 +1,14 @@ +export * from "./ci-projection-runner.ts"; +export * from "./config.ts"; +export * from "./gateway.ts"; +export * from "./kubernetes-client.ts"; +export * from "./kubernetes-projection.ts"; +export * from "./kubernetes-runner.ts"; +export * from "./observability.ts"; +export * from "./pod-host-router.ts"; +export * from "./server.ts"; +export * from "./session-host-policy.ts"; +export * from "./session-authority-runner.ts"; +export * from "./woodpecker.ts"; +export { runClusterServer } from "./main.ts"; +export { runSessionHost } from "./session-host-main.ts"; diff --git a/packages/cluster-server/src/kubernetes-client.ts b/packages/cluster-server/src/kubernetes-client.ts new file mode 100644 index 00000000..bd43edc8 --- /dev/null +++ b/packages/cluster-server/src/kubernetes-client.ts @@ -0,0 +1,382 @@ +import { createHash, timingSafeEqual } from "node:crypto"; +import { isAbsolute } from "node:path"; +import type { + ClusterSessionCreateArguments, + ClusterWorkspaceCreateArguments, +} from "@t4-code/host-wire"; +import { + CLUSTER_MAX_SESSIONS, + CLUSTER_MAX_WORKSPACES, + type InfrastructureList, + type KubernetesResource, + type KubernetesWatchEvent, +} from "./kubernetes-projection.ts"; +import { readBoundedRegularFile, readKubernetesToken } from "./config.ts"; + +const API_PREFIX = "/apis/cluster.t4.dev/v1alpha1"; +const TOKEN_REVIEW_PATH = "/apis/authentication.k8s.io/v1/tokenreviews"; +const MAX_WATCH_LINE_BYTES = 1024 * 1024; +export const CLUSTER_INTERNAL_AUDIENCE = "t4-cluster-internal"; + +export interface KubernetesApiClientOptions { + readonly baseUrl: string; + readonly namespace: string; + readonly token?: string; + readonly tokenFile?: string; + readonly ca?: string; + readonly fetch?: typeof globalThis.fetch; +} + +function canonical(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== "object") return value; + return Object.fromEntries(Object.keys(value as Record).sort().map(key => [key, canonical((value as Record)[key])])); +} +export function semanticResourceHash(value: unknown): string { + return `sha256:${createHash("sha256").update(JSON.stringify(canonical(value))).digest("hex")}`; +} +function resourceName(prefix: "workspace" | "session", commandId: string): string { + const digest = createHash("sha256").update(commandId, "utf8").digest("hex").slice(0, 16); + return `${prefix}-${digest}`; +} +function safeNamespace(value: string): string { + if (!/^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u.test(value)) throw new Error("Kubernetes namespace is invalid"); + return value; +} +function exactHttpsBase(value: string): string { + const url = new URL(value); + if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash) throw new Error("Kubernetes base URL must use HTTPS"); + return url.href.replace(/\/$/u, ""); +} +function object(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +export class KubernetesApiError extends Error { + constructor(readonly status: number, message: string, readonly body?: unknown) { + super(message); + this.name = "KubernetesApiError"; + } +} + +export class KubernetesApiClient { + readonly baseUrl: string; + readonly namespace: string; + readonly #token?: string; + readonly #tokenFile?: string; + readonly #ca?: string; + readonly #fetch: typeof globalThis.fetch; + + constructor(options: KubernetesApiClientOptions) { + this.baseUrl = exactHttpsBase(options.baseUrl); + this.namespace = safeNamespace(options.namespace); + const hasToken = options.token !== undefined; + const hasTokenFile = options.tokenFile !== undefined; + if (hasToken === hasTokenFile) throw new Error("Kubernetes API client requires exactly one credential source"); + if (hasToken) { + if (!options.token || new TextEncoder().encode(options.token).byteLength > 16_384 || /\s/u.test(options.token)) + throw new Error("Kubernetes service account token is invalid"); + this.#token = options.token; + } else { + if (!isAbsolute(options.tokenFile!)) throw new Error("Kubernetes service account token file must be absolute"); + this.#tokenFile = options.tokenFile; + } + this.#ca = options.ca; + this.#fetch = options.fetch ?? globalThis.fetch; + } + + async listInfrastructure(hostName?: string, signal?: AbortSignal): Promise { + const [hosts, workspaces, sessions] = await Promise.all([ + this.list("t4clusterhosts", 256, signal), + this.list("t4workspaces", CLUSTER_MAX_WORKSPACES, signal), + this.list("t4sessions", CLUSTER_MAX_SESSIONS, signal), + ]); + const host = hostName ? hosts.items.find(value => value.metadata.name === hostName) : hosts.items[0]; + if (!host) throw new Error("T4ClusterHost is unavailable"); + if (hosts.items.length > 1 && !hostName) throw new Error("T4ClusterHost selection is ambiguous"); + const belongsToHost = (resource: KubernetesResource): boolean => object(resource.spec).hostRef === host.metadata.name; + return { + host, + workspaces: workspaces.items.filter(belongsToHost), + sessions: sessions.items.filter(belongsToHost), + resourceVersion: sessions.resourceVersion || workspaces.resourceVersion || hosts.resourceVersion, + resourceVersions: { t4clusterhosts: hosts.resourceVersion, t4workspaces: workspaces.resourceVersion, t4sessions: sessions.resourceVersion }, + }; + } + + async list(resource: string, limit: number, signal?: AbortSignal): Promise<{ items: KubernetesResource[]; resourceVersion: string }> { + const response = object(await this.request(`${this.#collection(resource)}?limit=${limit}`, { signal })); + const metadata = object(response.metadata); + const items = Array.isArray(response.items) ? response.items as KubernetesResource[] : []; + if (items.length > limit || typeof metadata.continue === "string" && metadata.continue.length > 0) + throw new Error(`${resource} list exceeds limit`); + return { items, resourceVersion: typeof metadata.resourceVersion === "string" ? metadata.resourceVersion : "0" }; + } + + async create(resource: string, body: unknown, signal?: AbortSignal): Promise { + return await this.request(this.#collection(resource), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + signal, + }) as KubernetesResource; + } + + async get(resource: string, name: string, signal?: AbortSignal): Promise { + return await this.request(`${this.#collection(resource)}/${encodeURIComponent(name)}`, { signal }) as KubernetesResource; + } + + async delete(resource: string, name: string, preconditions: { readonly uid: string; readonly resourceVersion: string }, signal?: AbortSignal): Promise { + return await this.request(`${this.#collection(resource)}/${encodeURIComponent(name)}`, { + method: "DELETE", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ apiVersion: "v1", kind: "DeleteOptions", propagationPolicy: "Foreground", preconditions }), + signal, + }); + } + + async watch( + resource: string, + resourceVersion: string, + onEvent: (event: KubernetesWatchEvent) => void, + signal: AbortSignal, + onStarted?: () => void, + ): Promise { + const query = new URLSearchParams({ watch: "1", allowWatchBookmarks: "true", resourceVersion, timeoutSeconds: "300" }); + const response = await this.#raw(`${this.#collection(resource)}?${query}`, { signal }); + if (!response.ok) throw new KubernetesApiError(response.status, `Kubernetes watch failed with ${response.status}`); + if (!response.body) throw new Error("Kubernetes watch body is unavailable"); + const reader = response.body.getReader(); + onStarted?.(); + const decoder = new TextDecoder("utf-8", { fatal: true }); + let buffered = ""; + for (;;) { + const chunk = await reader.read(); + if (chunk.done) break; + buffered += decoder.decode(chunk.value, { stream: true }); + if (new TextEncoder().encode(buffered).byteLength > MAX_WATCH_LINE_BYTES) throw new Error("Kubernetes watch frame exceeds limit"); + for (;;) { + const newline = buffered.indexOf("\n"); + if (newline < 0) break; + const line = buffered.slice(0, newline); + buffered = buffered.slice(newline + 1); + if (!line) continue; + const value = object(JSON.parse(line)); + if (value.type === "BOOKMARK") continue; + if (value.type === "ERROR") throw new KubernetesApiError(Number(object(value.object).code) || 500, "Kubernetes watch error", value.object); + if (value.type === "ADDED" || value.type === "MODIFIED" || value.type === "DELETED") + onEvent({ type: value.type, object: value.object as KubernetesResource }); + } + } + } + + async request(path: string, init: RequestInit = {}): Promise { + const response = await this.#raw(path, init); + const body = await response.json().catch(() => undefined) as unknown; + if (!response.ok) throw new KubernetesApiError(response.status, `Kubernetes API request failed with ${response.status}`, body); + return body; + } + + #collection(resource: string): string { + if (!/^[a-z0-9]+$/u.test(resource)) throw new Error("Kubernetes resource is invalid"); + return `${API_PREFIX}/namespaces/${this.namespace}/${resource}`; + } + async #raw(path: string, init: RequestInit): Promise { + const token = this.#token ?? await readKubernetesToken(this.#tokenFile!); + const headers = new Headers(init.headers); + headers.set("accept", "application/json"); + headers.set("authorization", `Bearer ${token}`); + const request = { ...init, headers } as RequestInit & { tls?: { ca?: string } }; + if (this.#ca) request.tls = { ca: this.#ca }; + return this.#fetch(`${this.baseUrl}${path}`, request); + } +} + +export interface KubernetesTokenReviewerOptions { + readonly baseUrl: string; + readonly tokenPath: string; + readonly caPath: string; + readonly namespacePath: string; + readonly serverServiceAccountName: string; + readonly timeoutMs?: number; + readonly fetch?: typeof globalThis.fetch; +} + +/** Validates a projected server identity through the Kubernetes authentication authority. */ +export class KubernetesTokenReviewer { + readonly #baseUrl: string; + readonly #tokenPath: string; + readonly #caPath: string; + readonly #namespacePath: string; + readonly #serverServiceAccountName: string; + readonly #timeoutMs: number; + readonly #fetch: typeof globalThis.fetch; + + constructor(options: KubernetesTokenReviewerOptions) { + this.#baseUrl = exactHttpsBase(options.baseUrl); + if (![options.tokenPath, options.caPath, options.namespacePath].every(isAbsolute)) throw new Error("Kubernetes projected credential paths must be absolute"); + this.#tokenPath = options.tokenPath; + this.#caPath = options.caPath; + this.#namespacePath = options.namespacePath; + if (!/^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u.test(options.serverServiceAccountName)) throw new Error("cluster server ServiceAccount name is invalid"); + this.#serverServiceAccountName = options.serverServiceAccountName; + this.#timeoutMs = options.timeoutMs ?? 5_000; + if (!Number.isSafeInteger(this.#timeoutMs) || this.#timeoutMs < 100 || this.#timeoutMs > 30_000) throw new Error("TokenReview timeout is invalid"); + this.#fetch = options.fetch ?? globalThis.fetch; + } + + async review(presentedToken: string): Promise { + try { + const presentedBytes = new TextEncoder().encode(presentedToken).byteLength; + if (presentedBytes < 32 || presentedBytes > 16_384 || /\s/u.test(presentedToken)) return false; + const [rawReviewerToken, ca, rawNamespace] = await Promise.all([ + readBoundedRegularFile(this.#tokenPath, 16_384, "Kubernetes reviewer token"), + readBoundedRegularFile(this.#caPath, 1024 * 1024, "Kubernetes CA"), + readBoundedRegularFile(this.#namespacePath, 256, "Kubernetes namespace"), + ]); + const reviewerToken = rawReviewerToken.trim(); + const namespace = safeNamespace(rawNamespace.trim()); + if (!reviewerToken || /\s/u.test(reviewerToken) || !ca.includes("BEGIN CERTIFICATE")) return false; + const headers = new Headers({ accept: "application/json", authorization: `Bearer ${reviewerToken}`, "content-type": "application/json" }); + const request = { + method: "POST", + headers, + body: JSON.stringify({ + apiVersion: "authentication.k8s.io/v1", + kind: "TokenReview", + spec: { token: presentedToken, audiences: [CLUSTER_INTERNAL_AUDIENCE] }, + }), + signal: AbortSignal.timeout(this.#timeoutMs), + tls: { ca }, + } as RequestInit & { tls: { ca: string } }; + const response = await this.#fetch(`${this.#baseUrl}${TOKEN_REVIEW_PATH}`, request); + if (!response.ok) return false; + let body: unknown; + try { body = await response.json(); } catch { return false; } + const root = object(body); + if (root.apiVersion !== "authentication.k8s.io/v1" || root.kind !== "TokenReview") return false; + const status = object(root.status); + if (status.authenticated !== true || Object.hasOwn(status, "error")) return false; + const user = object(status.user); + if (typeof user.username !== "string") return false; + const expectedUsername = `system:serviceaccount:${namespace}:${this.#serverServiceAccountName}`; + const expectedBytes = Buffer.from(expectedUsername, "utf8"); + const actualBytes = Buffer.from(user.username, "utf8"); + if (expectedBytes.length !== actualBytes.length || !timingSafeEqual(expectedBytes, actualBytes)) return false; + return Array.isArray(status.audiences) && status.audiences.length === 1 && status.audiences[0] === CLUSTER_INTERNAL_AUDIENCE; + } catch { + return false; + } + } +} + +export interface KubernetesGatewayMutationBackendOptions { + readonly client: KubernetesApiClient; + readonly hostRef: string; +} +export class KubernetesGatewayMutationBackend { + readonly #client: KubernetesApiClient; + readonly #hostRef: string; + constructor(options: KubernetesGatewayMutationBackendOptions) { + this.#client = options.client; + this.#hostRef = options.hostRef; + } + + async createWorkspace(commandId: string, args: ClusterWorkspaceCreateArguments, principal: string): Promise<{ id: string; revision: string }> { + this.#validatePrincipal(principal); + const identity = `${principal}\u0000${commandId}`; + const name = resourceName("workspace", identity); + const body = { + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { name, annotations: this.#annotations(commandId, args, principal) }, + spec: { + hostRef: this.#hostRef, + owner: principal, + displayName: args.displayName, + retentionPolicy: args.retentionPolicy, + size: args.capacity, + ...(args.repository ? { repository: { repositoryId: args.repository.repositoryId, ...(args.repository.ref ? { ref: args.repository.ref } : {}), ...(args.repository.commit ? { commit: args.repository.commit } : {}) } } : {}), + }, + }; + const resource = await this.#createOrRead("t4workspaces", name, body, commandId, semanticResourceHash({ args, principal }), principal); + return { id: resource.metadata.name, revision: resource.metadata.resourceVersion ?? "0" }; + } + + async createSession(commandId: string, args: ClusterSessionCreateArguments, principal: string): Promise<{ sessionId: string; revision: string }> { + this.#validatePrincipal(principal); + const workspace = await this.#client.get("t4workspaces", args.workspaceId); + this.#assertWorkspaceOwner(workspace, principal); + const identity = `${principal}\u0000${commandId}`; + const name = resourceName("session", identity); + const body = { + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Session", + metadata: { name, annotations: this.#annotations(commandId, args, principal) }, + spec: { + hostRef: this.#hostRef, + workspaceRef: args.workspaceId, + title: args.title ?? "Cluster session", + runtimeProfile: args.runtimeProfile, + guiEnabled: args.guiEnabled, + ...(args.ci ? { ci: { repositoryId: args.ci.repositoryId, ref: args.ci.ref, commit: args.ci.commit } } : {}), + }, + }; + const resource = await this.#createOrRead("t4sessions", name, body, commandId, semanticResourceHash({ args, principal }), principal); + return { sessionId: resource.metadata.name, revision: resource.metadata.resourceVersion ?? "0" }; + } + + async deleteSession(_commandId: string, sessionId: string, principal: string): Promise<{ deleted: true }> { + this.#validatePrincipal(principal); + let session: KubernetesResource; + try { session = await this.#client.get("t4sessions", sessionId); } + catch (error) { + if (error instanceof KubernetesApiError && error.status === 404) return { deleted: true }; + throw error; + } + if (object(session.spec).hostRef !== this.#hostRef) throw new Error("session belongs to another cluster host"); + const workspaceRef = object(session.spec).workspaceRef; + if (typeof workspaceRef !== "string") throw new Error("session workspace reference is invalid"); + const workspace = await this.#client.get("t4workspaces", workspaceRef); + this.#assertWorkspaceOwner(workspace, principal); + if (!session.metadata.uid || !session.metadata.resourceVersion) throw new Error("session delete precondition is unavailable"); + try { await this.#client.delete("t4sessions", sessionId, { uid: session.metadata.uid, resourceVersion: session.metadata.resourceVersion }); } + catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 404) throw error; + } + return { deleted: true }; + } + + #validatePrincipal(principal: string): void { + if (!principal || new TextEncoder().encode(principal).byteLength > 256 || /\p{Cc}/u.test(principal) || principal !== principal.trim()) + throw new Error("gateway principal is invalid"); + } + #assertWorkspaceOwner(workspace: KubernetesResource, principal: string): void { + const spec = object(workspace.spec); + if (workspace.kind !== "T4Workspace" || spec.hostRef !== this.#hostRef || spec.owner !== principal) + throw new Error("workspace is unavailable for this identity"); + } + #annotations(commandId: string, args: unknown, principal: string): Record { + return { + "cluster.t4.dev/command-id": commandId, + "cluster.t4.dev/principal-hash": semanticResourceHash(principal), + "cluster.t4.dev/semantic-hash": semanticResourceHash({ args, principal }), + }; + } + async #createOrRead(resourceType: string, name: string, body: unknown, commandId: string, hash: string, principal: string): Promise { + try { + const created = await this.#client.create(resourceType, body); + return created.metadata ? created : { ...(body as KubernetesResource), metadata: { ...(body as KubernetesResource).metadata, resourceVersion: "0" } }; + } catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 409) throw error; + const existing = await this.#client.get(resourceType, name); + const annotations = existing.metadata.annotations ?? {}; + if ( + annotations["cluster.t4.dev/command-id"] !== commandId || + annotations["cluster.t4.dev/principal-hash"] !== semanticResourceHash(principal) || + annotations["cluster.t4.dev/semantic-hash"] !== hash + ) throw new Error("idempotency conflict for existing Kubernetes resource"); + return existing; + } + } +} diff --git a/packages/cluster-server/src/kubernetes-projection.ts b/packages/cluster-server/src/kubernetes-projection.ts new file mode 100644 index 00000000..fe699e32 --- /dev/null +++ b/packages/cluster-server/src/kubernetes-projection.ts @@ -0,0 +1,577 @@ +import { + CLUSTER_MAX_WORKSPACES as WIRE_MAX_WORKSPACES, + decodeClusterCondition, + decodeSessionCiState, + decodeSessionRef, + decodeWorkspaceInfrastructureProjection, + hostId, + revision, + type ClusterCondition, + type Cursor, + type HostId, + type SessionRef, + type SessionCiState, + type WorkspaceInfrastructureProjection, + type WorkspaceListResult, + type WorkspaceStateFrame, +} from "@t4-code/host-wire"; +import type { PodHostEndpoint, PodHostRoute } from "./pod-host-router.ts"; +export const CLUSTER_MAX_WORKSPACES = WIRE_MAX_WORKSPACES; +export const CLUSTER_MAX_SESSIONS = 1_000; +export const CLUSTER_WORKSPACE_REPLAY_FRAMES = 512; + +export interface KubernetesMetadata { + readonly name: string; + readonly uid?: string; + readonly resourceVersion?: string; + readonly generation?: number; + readonly creationTimestamp?: string; + readonly deletionTimestamp?: string; + readonly annotations?: Readonly>; +} +export interface KubernetesResource { + readonly apiVersion?: string; + readonly kind?: string; + readonly metadata: KubernetesMetadata; + readonly spec?: Readonly>; + readonly status?: Readonly>; +} +export interface KubernetesWatchEvent { + readonly type: "ADDED" | "MODIFIED" | "DELETED"; + readonly object: KubernetesResource; +} +export class KubernetesAuthorityInvalidatedError extends Error { + constructor(message: string) { + super(message); + this.name = "KubernetesAuthorityInvalidatedError"; + } +} +export interface InfrastructureList { + readonly host: KubernetesResource; + readonly workspaces: readonly KubernetesResource[]; + readonly sessions: readonly KubernetesResource[]; + readonly resourceVersion: string; + readonly resourceVersions?: Readonly>; +} +export interface ClusterInfrastructureProjectionOptions { + readonly epoch: string; + readonly namespace: string; + readonly maxWorkspaces?: number; + readonly maxSessions?: number; +} + +type SessionListener = () => void; +interface WorkspaceReplayItem { readonly frame: WorkspaceStateFrame; readonly owner?: string; } +interface WorkspaceSubscription { readonly listener: (frame: WorkspaceStateFrame) => void; readonly principal?: string; } +export interface SessionCiCorrelation { + readonly sessionId: string; + readonly repositoryId: string; + readonly ref: string; + readonly commit: string; +} + +interface SessionAuthorityWaiter { readonly resolve: (value: SessionRef) => void; readonly reject: (reason: Error) => void; readonly timer: ReturnType; } +function record(value: unknown): Record { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} +function text(value: unknown, fallback = ""): string { + return typeof value === "string" ? value : fallback; +} +function number(value: unknown, fallback = 0): number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : fallback; +} +function firstCondition(status: Readonly>): ClusterCondition | undefined { + if (!Array.isArray(status.conditions) || status.conditions.length === 0) return undefined; + const raw = record(status.conditions[0]); + try { + return decodeClusterCondition({ + type: raw.type, + status: raw.status, + reason: raw.reason, + message: raw.message ?? "", + observedGeneration: raw.observedGeneration ?? status.observedGeneration ?? 0, + }); + } catch { + return undefined; + } +} +function categorical(value: unknown, values: T, fallback: T[number]): T[number] { + return typeof value === "string" && (values as readonly string[]).includes(value) ? value as T[number] : fallback; +} +function resourceRevision(resource: KubernetesResource): string { + return text(resource.metadata.resourceVersion, `generation-${number(resource.metadata.generation)}`); +} +function serviceName(value: unknown): string | undefined { + if (typeof value !== "string" || !/^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u.test(value)) return undefined; + return value; +} +function workspaceOwner(resource: KubernetesResource): string | undefined { + const owner = record(resource.spec).owner; + if (typeof owner !== "string" || owner.length === 0 || new TextEncoder().encode(owner).byteLength > 256 || /\p{Cc}/u.test(owner)) return undefined; + return owner; +} +function ciSelection(resource: KubernetesResource): { readonly repositoryId: string; readonly ref: string; readonly commit: string } | undefined { + const ci = record(record(resource.spec).ci); + if (typeof ci.repositoryId !== "string" || typeof ci.ref !== "string" || typeof ci.commit !== "string") return undefined; + return { repositoryId: ci.repositoryId, ref: ci.ref, commit: ci.commit }; +} + +export function clusterHostIdFromUid(uid: string): HostId { + if (!/^[A-Za-z0-9][A-Za-z0-9-]{0,127}$/u.test(uid)) throw new Error("T4ClusterHost UID is invalid"); + return hostId(`cluster:${uid}`); +} + +export class ClusterInfrastructureProjection { + readonly epoch: string; + readonly namespace: string; + readonly maxWorkspaces: number; + readonly maxSessions: number; + #hostId?: HostId; + #host?: KubernetesResource; + #workspaces = new Map(); + #sessions = new Map(); + #authoritativeSessions = new Map(); + #ciStates = new Map(); + #authorityWaiters = new Map>(); + #versions = new Map(); + #workspaceSequence = 0; + #sessionSequence = 0; + #workspaceListeners = new Set(); + #sessionListeners = new Set(); + #replay: WorkspaceReplayItem[] = []; + #resourceVersion = "0"; + #resourceVersions: Readonly> = {}; + #initialized = false; + + constructor(options: ClusterInfrastructureProjectionOptions) { + if (!options.epoch || options.epoch.length > 128) throw new Error("replica epoch is invalid"); + if (!/^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u.test(options.namespace)) throw new Error("namespace is invalid"); + this.epoch = options.epoch; + this.namespace = options.namespace; + this.maxWorkspaces = options.maxWorkspaces ?? CLUSTER_MAX_WORKSPACES; + this.maxSessions = options.maxSessions ?? CLUSTER_MAX_SESSIONS; + if (this.maxWorkspaces < 1 || this.maxWorkspaces > CLUSTER_MAX_WORKSPACES) throw new Error("workspace projection limit is invalid"); + if (this.maxSessions < 1 || this.maxSessions > CLUSTER_MAX_SESSIONS) throw new Error("session projection limit is invalid"); + } + + get hostId(): HostId { + if (!this.#hostId) throw new Error("cluster host projection is not synchronized"); + return this.#hostId; + } + get workspaceCursor(): Cursor { return { epoch: this.epoch, seq: this.#workspaceSequence }; } + get sessionCursor(): Cursor { return { epoch: this.epoch, seq: this.#sessionSequence }; } + get resourceVersion(): string { return this.#resourceVersion; } + resourceVersionFor(resource: string): string { return this.#resourceVersions[resource] ?? this.#resourceVersion; } + + replace(input: InfrastructureList): void { + if (input.workspaces.length > this.maxWorkspaces) throw new Error("workspace projection limit exceeded"); + if (input.sessions.length > this.maxSessions) throw new Error("session projection limit exceeded"); + if (input.host.kind !== "T4ClusterHost" || !input.host.metadata.uid) throw new Error("T4ClusterHost identity is missing"); + const selectedHost = input.host.metadata.name; + for (const resource of input.workspaces) + if (resource.kind !== "T4Workspace" || record(resource.spec).hostRef !== selectedHost) throw new Error("workspace belongs to another cluster host"); + for (const resource of input.sessions) + if (resource.kind !== "T4Session" || record(resource.spec).hostRef !== selectedHost) throw new Error("session belongs to another cluster host"); + const nextHostId = clusterHostIdFromUid(input.host.metadata.uid); + if (this.#hostId && this.#hostId !== nextHostId) throw new Error("T4ClusterHost UID changed within a replica epoch"); + const initialized = this.#initialized; + const previousWorkspaces = this.#workspaces; + const previousSessions = this.#sessions; + const nextWorkspaces = new Map(input.workspaces.map(resource => [resource.metadata.name, resource])); + const nextSessions = new Map(input.sessions.map(resource => [resource.metadata.name, resource])); + this.#hostId = nextHostId; + this.#host = input.host; + this.#initialized = true; + this.#workspaces = nextWorkspaces; + this.#sessions = nextSessions; + for (const name of this.#authoritativeSessions.keys()) if (!nextSessions.has(name)) this.#authoritativeSessions.delete(name); + for (const [name, state] of this.#ciStates) { + const selected = nextSessions.get(name); + const ci = selected ? ciSelection(selected) : undefined; + if (!ci || ci.repositoryId !== state.repositoryId || ci.ref !== state.ref || ci.commit !== state.commit) this.#ciStates.delete(name); + } + this.#versions.clear(); + for (const resource of [input.host, ...input.workspaces, ...input.sessions]) + this.#versions.set(`${resource.kind ?? "unknown"}/${resource.metadata.name}`, text(resource.metadata.resourceVersion)); + this.#resourceVersion = input.resourceVersion; + this.#resourceVersions = input.resourceVersions ?? {}; + if (!initialized) { + this.#workspaceSequence = 1; + this.#sessionSequence = 1; + return; + } + const workspaceNames = [...new Set([...previousWorkspaces.keys(), ...nextWorkspaces.keys()])].sort(); + let workspaceAccessChanged = false; + for (const name of workspaceNames) { + const before = previousWorkspaces.get(name); + const after = nextWorkspaces.get(name); + if (before && after && resourceRevision(before) === resourceRevision(after)) continue; + const beforeOwner = before ? workspaceOwner(before) : undefined; + const afterOwner = after ? workspaceOwner(after) : undefined; + if (before && after && beforeOwner !== afterOwner) { + this.#publishWorkspace(before, true, beforeOwner); + this.#publishWorkspace(after, false, afterOwner); + workspaceAccessChanged = true; + } else if (after) { + this.#publishWorkspace(after, false, afterOwner); + workspaceAccessChanged ||= !before; + } else if (before) { + this.#publishWorkspace(before, true, beforeOwner); + workspaceAccessChanged = true; + } + } + const sessionNames = new Set([...previousSessions.keys(), ...nextSessions.keys()]); + const sessionsChanged = [...sessionNames].some(name => { + const before = previousSessions.get(name); + const after = nextSessions.get(name); + return !before || !after || resourceRevision(before) !== resourceRevision(after); + }); + if (sessionsChanged || workspaceAccessChanged) { + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + } + } + + workspaceList(principal?: string): WorkspaceListResult { + return { + cursor: this.workspaceCursor, + workspaces: [...this.#workspaces.values()] + .filter(resource => principal === undefined || workspaceOwner(resource) === principal) + .sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)) + .map(resource => this.#workspace(resource)), + }; + } + sessionRefs(principal?: string): SessionRef[] { + return [...this.#sessions.values()] + .filter(resource => principal === undefined || this.#ownsSessionResource(resource, principal)) + .sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)) + .flatMap(resource => { + const authoritative = this.#authoritativeSessions.get(resource.metadata.name); + return authoritative ? [this.#session(resource, authoritative)] : []; + }); + } + + allowedOrigins(): readonly string[] { + const values = record(this.#host?.spec).allowedOrigins; + if (!Array.isArray(values) || values.length > 64) return []; + const origins: string[] = []; + for (const value of values) { + if (typeof value !== "string") continue; + try { + const url = new URL(value); + if (url.protocol === "https:" && !url.username && !url.password && url.pathname === "/" && !url.search && !url.hash) + origins.push(url.origin); + } catch {} + } + return origins; + } + + ownsWorkspace(workspace: string, principal: string): boolean { + const resource = this.#workspaces.get(workspace); + return resource !== undefined && workspaceOwner(resource) === principal; + } + ownsSession(session: string, principal: string): boolean { + const resource = this.#sessions.get(session); + return resource !== undefined && this.#ownsSessionResource(resource, principal); + } + sessionExists(session: string): boolean { + return this.#sessions.has(session); + } + sessionGuiState(session: string, principal?: string): "Unavailable" | "Starting" | "Ready" | "Failed" | undefined { + const resource = this.#sessions.get(session); + if (!resource || principal !== undefined && !this.#ownsSessionResource(resource, principal)) return undefined; + if (record(resource.spec).guiEnabled !== true) return "Unavailable"; + const phase = record(resource.status).phase; + if (phase === "Running") return "Ready"; + if (phase === "Failed" || phase === "Terminating") return "Failed"; + return "Starting"; + } + sessionEndpoints(): PodHostEndpoint[] { + return [...this.#sessions.values()].flatMap(resource => { + const service = serviceName(record(resource.status).serviceName); + return service ? [{ clusterSessionId: resource.metadata.name, url: `ws://${service}.${this.namespace}.svc:8787/v1/ws` }] : []; + }); + } + sessionRoute(clusterSessionId: string, principal?: string): PodHostRoute | undefined { + const resource = this.#sessions.get(clusterSessionId); + const authoritative = this.#authoritativeSessions.get(clusterSessionId); + if (!resource || !authoritative || principal !== undefined && !this.#ownsSessionResource(resource, principal)) return undefined; + const service = serviceName(record(resource.status).serviceName); + if (!service) return undefined; + return { clusterSessionId, upstreamSessionId: authoritative.sessionId, url: `ws://${service}.${this.namespace}.svc:8787/v1/ws` }; + } + sessionRevision(session: string, principal?: string): string | undefined { + const resource = this.#sessions.get(session); + const authoritative = this.#authoritativeSessions.get(session); + if (!resource || !authoritative || principal !== undefined && !this.#ownsSessionResource(resource, principal)) return undefined; + return authoritative.revision; + } + sessionRef(session: string, principal?: string): SessionRef | undefined { + const resource = this.#sessions.get(session); + const authoritative = this.#authoritativeSessions.get(session); + if (!resource || !authoritative || principal !== undefined && !this.#ownsSessionResource(resource, principal)) return undefined; + return this.#session(resource, authoritative); + } + setSessionAuthority(clusterSessionId: string, value: SessionRef): void { + const resource = this.#sessions.get(clusterSessionId); + if (!resource) return; + const authoritative = decodeSessionRef(value, `authority.${clusterSessionId}`); + const previous = this.#authoritativeSessions.get(clusterSessionId); + if (previous && JSON.stringify(previous) === JSON.stringify(authoritative)) return; + const projected = this.#session(resource, authoritative); + this.#authoritativeSessions.set(clusterSessionId, authoritative); + this.#sessionSequence++; + for (const waiter of this.#authorityWaiters.get(clusterSessionId) ?? []) { + clearTimeout(waiter.timer); + waiter.resolve(projected); + } + this.#authorityWaiters.delete(clusterSessionId); + for (const listener of this.#sessionListeners) listener(); + } + clearSessionAuthority(clusterSessionId: string): void { + if (!this.#authoritativeSessions.delete(clusterSessionId)) return; + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + } + waitForSessionAuthority(clusterSessionId: string, timeoutMs = 30_000): Promise { + const current = this.sessionRef(clusterSessionId); + if (current) return Promise.resolve(current); + const deferred = Promise.withResolvers(); + let waiter: SessionAuthorityWaiter | undefined; + const timer = setTimeout(() => { + const waiters = this.#authorityWaiters.get(clusterSessionId); + if (waiter) waiters?.delete(waiter); + if (waiters?.size === 0) this.#authorityWaiters.delete(clusterSessionId); + deferred.reject(new Error("authoritative OMP session did not become available")); + }, timeoutMs); + waiter = { resolve: deferred.resolve, reject: deferred.reject, timer }; + const waiters = this.#authorityWaiters.get(clusterSessionId) ?? new Set(); + waiters.add(waiter); + this.#authorityWaiters.set(clusterSessionId, waiters); + return deferred.promise; + } + sessionCiSelection(session: string, principal?: string): { repositoryId: string; ref: string; commit: string } | undefined { + const resource = this.#sessions.get(session); + if (!resource || principal !== undefined && !this.#ownsSessionResource(resource, principal)) return undefined; + return ciSelection(resource); + } + sessionCiCorrelations(): SessionCiCorrelation[] { + return [...this.#sessions.values()].flatMap(resource => { + const selected = ciSelection(resource); + return selected ? [{ sessionId: resource.metadata.name, ...selected }] : []; + }); + } + setSessionCiState(session: string, value: SessionCiState): void { + const resource = this.#sessions.get(session); + const selected = resource ? ciSelection(resource) : undefined; + if (!selected) return; + const state = decodeSessionCiState(value, `ci.${session}`); + if (state.repositoryId !== selected.repositoryId || state.ref !== selected.ref || state.commit !== selected.commit) + throw new Error("CI provider state does not match the declared session correlation"); + const previous = this.#ciStates.get(session); + if (previous && JSON.stringify(previous) === JSON.stringify(state)) return; + this.#ciStates.set(session, state); + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + } + + applyWatch(event: KubernetesWatchEvent): void { + const resource = event.object; + const name = resource.metadata.name; + if (resource.kind === "T4ClusterHost") { + if (!this.#host || name !== this.#host.metadata.name) return; + let authorityValid = false; + try { + authorityValid = event.type !== "DELETED" + && resource.metadata.uid !== undefined + && clusterHostIdFromUid(resource.metadata.uid) === this.#hostId; + } catch { /* An invalid replacement UID invalidates the selected authority too. */ } + if (!authorityValid) { + const error = new KubernetesAuthorityInvalidatedError( + event.type === "DELETED" ? "selected T4ClusterHost was deleted" : "selected T4ClusterHost identity changed", + ); + this.#invalidateAuthority(error); + throw error; + } + } else if (resource.kind === "T4Workspace" || resource.kind === "T4Session") { + if (!this.#host) return; + if (record(resource.spec).hostRef !== this.#host.metadata.name) { + if (resource.kind === "T4Session") { + if (!this.#sessions.delete(name)) return; + this.#authoritativeSessions.delete(name); + this.#ciStates.delete(name); + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + return; + } + const existing = this.#workspaces.get(name); + if (!existing) return; + this.#workspaces.delete(name); + this.#publishWorkspace(existing, true, workspaceOwner(existing)); + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + return; + } + } else return; + const key = `${resource.kind}/${name}`; + const version = text(resource.metadata.resourceVersion); + if (event.type !== "DELETED" && version && this.#versions.get(key) === version) return; + if (version) { + this.#versions.set(key, version); + this.#resourceVersion = version; + const collection = resource.kind === "T4ClusterHost" ? "t4clusterhosts" : resource.kind === "T4Workspace" ? "t4workspaces" : "t4sessions"; + this.#resourceVersions = { ...this.#resourceVersions, [collection]: version }; + } + if (resource.kind === "T4ClusterHost") { this.#host = resource; return; } + if (resource.kind === "T4Session") { + if (event.type === "DELETED") { + if (!this.#sessions.delete(name)) return; + this.#authoritativeSessions.delete(name); + this.#ciStates.delete(name); + } else { + if (!this.#sessions.has(name) && this.#sessions.size >= this.maxSessions) throw new Error("session projection limit exceeded"); + this.#sessions.set(name, resource); + const state = this.#ciStates.get(name); + const selected = ciSelection(resource); + if (state && (!selected || state.repositoryId !== selected.repositoryId || state.ref !== selected.ref || state.commit !== selected.commit)) + this.#ciStates.delete(name); + } + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + return; + } + const existing = this.#workspaces.get(name); + let workspaceAccessChanged = false; + if (event.type === "DELETED") { + if (!existing) return; + this.#workspaces.delete(name); + this.#publishWorkspace(resource, true, workspaceOwner(existing) ?? workspaceOwner(resource)); + workspaceAccessChanged = true; + } else { + if (!existing && this.#workspaces.size >= this.maxWorkspaces) throw new Error("workspace projection limit exceeded"); + this.#workspaces.set(name, resource); + const previousOwner = existing ? workspaceOwner(existing) : undefined; + const nextOwner = workspaceOwner(resource); + if (existing && previousOwner !== nextOwner) this.#publishWorkspace(existing, true, previousOwner); + this.#publishWorkspace(resource, false, nextOwner); + workspaceAccessChanged = !existing || previousOwner !== nextOwner; + } + if (workspaceAccessChanged) { + this.#sessionSequence++; + for (const listener of this.#sessionListeners) listener(); + } + } + + subscribe(listener: (frame: WorkspaceStateFrame) => void, cursor?: Cursor, principal?: string): () => void { + if (cursor && cursor.epoch === this.epoch && cursor.seq < this.#workspaceSequence) + for (const item of this.#replay) if (item.frame.cursor.seq > cursor.seq && (principal === undefined || item.owner === principal)) listener(item.frame); + const subscription: WorkspaceSubscription = { listener, ...(principal === undefined ? {} : { principal }) }; + this.#workspaceListeners.add(subscription); + return () => { this.#workspaceListeners.delete(subscription); }; + } + subscribeSessions(listener: SessionListener): () => void { + this.#sessionListeners.add(listener); + return () => { this.#sessionListeners.delete(listener); }; + } + + #invalidateAuthority(error: KubernetesAuthorityInvalidatedError): void { + const workspaces = [...this.#workspaces.values()]; + const sessionStateChanged = workspaces.length > 0 || this.#sessions.size > 0 || this.#authoritativeSessions.size > 0; + this.#workspaces.clear(); + this.#sessions.clear(); + this.#authoritativeSessions.clear(); + this.#ciStates.clear(); + for (const resource of workspaces) { + try { this.#publishWorkspace(resource, true, workspaceOwner(resource)); } + catch { /* A subscriber cannot veto authority invalidation. */ } + } + this.#host = undefined; + this.#hostId = undefined; + this.#versions.clear(); + this.#resourceVersion = "0"; + this.#resourceVersions = {}; + for (const waiters of this.#authorityWaiters.values()) { + for (const waiter of waiters) { + clearTimeout(waiter.timer); + waiter.reject(error); + } + } + this.#authorityWaiters.clear(); + if (sessionStateChanged) { + this.#sessionSequence++; + for (const listener of this.#sessionListeners) { + try { listener(); } + catch { /* A subscriber cannot veto authority invalidation. */ } + } + } + } + #ownsSessionResource(resource: KubernetesResource, principal: string): boolean { + const workspace = text(record(resource.spec).workspaceRef); + return workspace.length > 0 && this.ownsWorkspace(workspace, principal); + } + #publishWorkspace(resource: KubernetesResource, deleted: boolean, owner?: string): void { + this.#workspaceSequence++; + const name = resource.metadata.name; + const currentRevision = revision(resourceRevision(resource)); + const frame: WorkspaceStateFrame = deleted + ? { v: "omp-app/1", type: "workspace.state", hostId: this.hostId, workspaceId: name, cursor: this.workspaceCursor, revision: currentRevision, remove: name } + : { v: "omp-app/1", type: "workspace.state", hostId: this.hostId, workspaceId: name, cursor: this.workspaceCursor, revision: currentRevision, upsert: this.#workspace(resource) }; + this.#replay.push({ frame, ...(owner === undefined ? {} : { owner }) }); + if (this.#replay.length > CLUSTER_WORKSPACE_REPLAY_FRAMES) this.#replay.shift(); + for (const subscription of this.#workspaceListeners) + if (subscription.principal === undefined || subscription.principal === owner) subscription.listener(frame); + } + #session(resource: KubernetesResource, authoritative: SessionRef): SessionRef { + const spec = record(resource.spec); + const status = record(resource.status); + const workspaceId = text(spec.workspaceRef, "unknown-workspace"); + const selectedCi = ciSelection(resource); + const projectedCi = this.#ciStates.get(resource.metadata.name) ?? (selectedCi ? { + provider: "woodpecker" as const, + correlation: "unknown" as const, + ...selectedCi, + } : undefined); + const condition = firstCondition(status); + const guiEnabled = spec.guiEnabled === true; + const infrastructurePhase = categorical(status.phase, ["Pending", "Running", "Failed", "Terminating", "Unknown"] as const, "Unknown"); + const guiState = !guiEnabled + ? "Unavailable" + : infrastructurePhase === "Running" + ? "Ready" + : infrastructurePhase === "Failed" || infrastructurePhase === "Terminating" + ? "Failed" + : "Starting"; + const { cluster: _upstreamCluster, ci: _upstreamCi, ...ompLiveState } = authoritative.liveState ?? {}; + return decodeSessionRef({ + ...authoritative, + hostId: this.hostId, + sessionId: resource.metadata.name, + liveState: { + ...ompLiveState, + cluster: { + workspaceId, + phase: infrastructurePhase, + ...(condition ? { condition } : {}), + gui: { state: guiState }, + }, + ...(projectedCi ? { ci: projectedCi } : {}), + }, + }, `session.${resource.metadata.name}`); + } + #workspace(resource: KubernetesResource): WorkspaceInfrastructureProjection { + const spec = record(resource.spec); + const status = record(resource.status); + const condition = firstCondition(status); + return decodeWorkspaceInfrastructureProjection({ + id: resource.metadata.name, + displayName: text(spec.displayName, resource.metadata.name).slice(0, 256), + phase: resource.metadata.deletionTimestamp ? "Terminating" : categorical(status.phase, ["Pending", "Ready", "Failed", "Terminating", "Unknown"] as const, "Unknown"), + retentionPolicy: spec.retentionPolicy === "Delete" ? "Delete" : "Retain", + ...(typeof record(this.#host?.spec).storageClassName === "string" ? { storageClass: String(record(this.#host?.spec).storageClassName).slice(0, 128) } : {}), + ...(typeof status.capacity === "string" ? { capacity: status.capacity.slice(0, 64) } : typeof spec.size === "string" ? { capacity: spec.size.slice(0, 64) } : {}), + accessMode: "ReadWriteMany", + revision: resourceRevision(resource), + ...(condition ? { condition } : {}), + }, `workspace.${resource.metadata.name}`); + } +} diff --git a/packages/cluster-server/src/kubernetes-runner.ts b/packages/cluster-server/src/kubernetes-runner.ts new file mode 100644 index 00000000..3f23a5ef --- /dev/null +++ b/packages/cluster-server/src/kubernetes-runner.ts @@ -0,0 +1,129 @@ +import { KubernetesApiClient, KubernetesApiError } from "./kubernetes-client.ts"; +import { ClusterInfrastructureProjection, KubernetesAuthorityInvalidatedError } from "./kubernetes-projection.ts"; + +export interface KubernetesProjectionRunnerOptions { + readonly client: KubernetesApiClient; + readonly projection: ClusterInfrastructureProjection; + readonly hostName: string; + readonly onSynchronized?: () => void; + readonly onError?: (error: unknown) => void; + readonly retryMs?: number; +} + +function delay(milliseconds: number, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const finish = (): void => { signal.removeEventListener("abort", abort); resolve(); }; + const timer = setTimeout(finish, milliseconds); + const abort = (): void => { clearTimeout(timer); reject(signal.reason); }; + signal.addEventListener("abort", abort, { once: true }); + }); +} + +/** Rebuildable list/watch runner. Kubernetes remains the sole infrastructure source. */ +export class KubernetesProjectionRunner { + readonly #options: KubernetesProjectionRunnerOptions; + readonly #synchronized = new Set(); + #abort?: AbortController; + #running?: Promise; + #relist?: Promise; + constructor(options: KubernetesProjectionRunnerOptions) { this.#options = options; } + + async start(): Promise { + if (this.#running) throw new Error("Kubernetes projection runner already started"); + this.#abort = new AbortController(); + const signal = this.#abort.signal; + const initial = await this.#options.client.listInfrastructure(this.#options.hostName, signal); + this.#options.projection.replace(initial); + this.#synchronized.clear(); + for (const resource of ["t4clusterhosts", "t4workspaces", "t4sessions"]) this.#synchronized.add(resource); + this.#options.onSynchronized?.(); + this.#running = this.#run(signal); + } + + async stop(): Promise { + this.#abort?.abort(new Error("Kubernetes projection runner stopped")); + await this.#running?.catch(() => undefined); + this.#running = undefined; + this.#abort = undefined; + this.#relist = undefined; + this.#synchronized.clear(); + } + + async #run(signal: AbortSignal): Promise { + const resources = ["t4clusterhosts", "t4workspaces", "t4sessions"] as const; + while (!signal.aborted) { + const generation = new AbortController(); + const stopGeneration = (): void => generation.abort(signal.reason); + signal.addEventListener("abort", stopGeneration, { once: true }); + await Promise.all(resources.map(resource => this.#watchResource(resource, resources.length, signal, generation))); + signal.removeEventListener("abort", stopGeneration); + } + } + + async #watchResource( + resource: string, + resourceCount: number, + rootSignal: AbortSignal, + generation: AbortController, + ): Promise { + let version = this.#options.projection.resourceVersionFor(resource); + while (!generation.signal.aborted) { + try { + await this.#options.client.watch( + resource, + version, + event => { + this.#options.projection.applyWatch(event); + version = event.object.metadata.resourceVersion ?? version; + }, + generation.signal, + () => { + this.#synchronized.add(resource); + if (this.#synchronized.size === resourceCount) this.#options.onSynchronized?.(); + }, + ); + if (generation.signal.aborted) return; + this.#synchronized.delete(resource); + this.#options.onError?.(new Error(`Kubernetes ${resource} watch ended`)); + } catch (error) { + if (generation.signal.aborted || rootSignal.aborted) return; + this.#synchronized.delete(resource); + this.#options.onError?.(error); + const requiresRelist = error instanceof KubernetesAuthorityInvalidatedError + || error instanceof KubernetesApiError && error.status === 410; + if (requiresRelist) { + try { await this.#restartGeneration(generation, rootSignal); } + catch (relistError) { + if (!rootSignal.aborted) this.#options.onError?.(relistError); + } + return; + } + } + try { await delay(this.#options.retryMs ?? 1_000, generation.signal); } + catch { return; } + } + } + + async #restartGeneration(generation: AbortController, signal: AbortSignal): Promise { + if (!this.#relist) { + this.#synchronized.clear(); + generation.abort(new Error("Kubernetes watch generation relisting")); + this.#relist = (async () => { + while (!signal.aborted) { + try { + const snapshot = await this.#options.client.listInfrastructure(this.#options.hostName, signal); + this.#options.projection.replace(snapshot); + return; + } catch (error) { + if (signal.aborted) throw error; + this.#options.onError?.(error); + await delay(this.#options.retryMs ?? 1_000, signal); + } + } + throw signal.reason; + })().finally(() => { this.#relist = undefined; }); + } + await this.#relist; + } +} diff --git a/packages/cluster-server/src/main.ts b/packages/cluster-server/src/main.ts new file mode 100755 index 00000000..50a9e797 --- /dev/null +++ b/packages/cluster-server/src/main.ts @@ -0,0 +1,116 @@ +#!/usr/bin/env bun +import { CiProjectionRunner } from "./ci-projection-runner.ts"; +import { clusterServerConfigFromEnv, loadKubernetesCa } from "./config.ts"; +import { ClusterGateway } from "./gateway.ts"; +import { KubernetesApiClient, KubernetesGatewayMutationBackend } from "./kubernetes-client.ts"; +import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; +import { KubernetesProjectionRunner } from "./kubernetes-runner.ts"; +import { ClusterMetrics, ClusterServerHealth, JsonLogger } from "./observability.ts"; +import { WebSocketPodHostConnector } from "./pod-host-router.ts"; +import { startClusterHttpServers, type ClusterHttpServers } from "./server.ts"; +import { SessionAuthorityRunner } from "./session-authority-runner.ts"; +import { WoodpeckerProvider } from "./woodpecker.ts"; + +export async function runClusterServer(env: Readonly> = process.env): Promise { + const config = clusterServerConfigFromEnv(env); + const logger = new JsonLogger(undefined, { component: "cluster-server", version: "0.1.30", namespace: config.namespace }); + const health = new ClusterServerHealth(); + const metrics = new ClusterMetrics({ component: "cluster-server", version: "0.1.30", namespace: config.namespace }); + const ca = await loadKubernetesCa(config); + const kubernetes = new KubernetesApiClient({ + baseUrl: config.kubernetesBaseUrl, + namespace: config.namespace, + tokenFile: config.kubernetesTokenPath, + ca, + }); + const projection = new ClusterInfrastructureProjection({ epoch: config.epoch, namespace: config.namespace }); + const runner = new KubernetesProjectionRunner({ + client: kubernetes, + projection, + hostName: config.hostName, + onSynchronized: () => health.markKubernetesSynced(), + onError: error => { + health.markKubernetesUnavailable(); + logger.warn("Kubernetes watch reconnecting", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }); + }, + }); + + const stopped = Promise.withResolvers(); + const stop = (): void => stopped.resolve(); + let authority: SessionAuthorityRunner | undefined; + let ciProjection: CiProjectionRunner | undefined; + let servers: ClusterHttpServers | undefined; + let signalsInstalled = false; + try { + await runner.start(); + const connector = new WebSocketPodHostConnector({ identityTokenFile: config.identityTokenPath }); + authority = new SessionAuthorityRunner({ + projection, + connector, + onError: error => logger.warn("session authority reconnecting", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }), + }); + authority.start(); + const ciProvider = config.woodpecker ? new WoodpeckerProvider(config.woodpecker) : undefined; + if (ciProvider) { + ciProjection = new CiProjectionRunner({ + projection, + provider: ciProvider, + onError: error => logger.warn("CI projection refresh failed", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }), + }); + ciProjection.start(); + } + const gateway = new ClusterGateway({ + projection, + connector, + mutations: new KubernetesGatewayMutationBackend({ client: kubernetes, hostRef: config.hostName }), + ...(ciProvider ? { ciProvider } : {}), + }); + servers = startClusterHttpServers({ + gateway, + projection, + gatewayPort: config.gatewayPort, + adminPort: config.adminPort, + trustedProxyAddresses: config.trustedProxyAddresses, + trustedProxyCidrs: config.trustedProxyCidrs, + health, + metrics, + logger, + }); + process.once("SIGTERM", stop); + process.once("SIGINT", stop); + signalsInstalled = true; + await stopped.promise; + } finally { + if (signalsInstalled) { + process.off("SIGTERM", stop); + process.off("SIGINT", stop); + } + try { + await servers?.drain(); + } finally { + try { + await ciProjection?.stop(); + } finally { + try { + await authority?.stop(); + } finally { + try { + await runner.stop(); + } finally { + await servers?.stop(); + } + } + } + } + } +} + +async function main(): Promise { + try { await runClusterServer(); } + catch (error) { + const logger = new JsonLogger(undefined, { component: "cluster-server", version: "0.1.30" }); + logger.error("cluster server failed", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }); + process.exitCode = 1; + } +} +if (import.meta.main) await main(); diff --git a/packages/cluster-server/src/observability.ts b/packages/cluster-server/src/observability.ts new file mode 100644 index 00000000..9a8b47be --- /dev/null +++ b/packages/cluster-server/src/observability.ts @@ -0,0 +1,111 @@ +const REDACTED = "[REDACTED]"; +const SECRET_MARKERS = ["password", "passwd", "secret", "token", "credential", "apikey", "privatekey", "cookie", "auth", "pairing", "prompt", "transcript", "path"]; +const METRIC_LABELS = new Set(["component", "version", "namespace", "workspace", "session", "condition", "provider", "result"]); +const METRIC_NAME = /^[a-zA-Z_:][a-zA-Z0-9_:]*$/u; +const LABEL_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/u; + +function redactKey(key: string): boolean { + const normalized = key.normalize("NFKC").toLowerCase().replace(/[^a-z0-9]+/gu, ""); + return SECRET_MARKERS.some(marker => normalized.includes(marker)); +} +export function redactStructuredValue(value: unknown, seen = new WeakSet()): unknown { + if (Array.isArray(value)) return value.slice(0, 1_000).map(item => redactStructuredValue(item, seen)); + if (!value || typeof value !== "object") return typeof value === "string" && value.length > 2_048 ? `${value.slice(0, 2_048)}…` : value; + if (seen.has(value)) return "[CYCLE]"; + seen.add(value); + const output: Record = {}; + for (const [key, child] of Object.entries(value as Record).slice(0, 128)) + output[key] = redactKey(key) ? REDACTED : redactStructuredValue(child, seen); + return output; +} + +export class JsonLogger { + constructor( + private readonly write: (line: string) => void = line => process.stdout.write(`${line}\n`), + private readonly base: Readonly> = {}, + ) {} + info(message: string, fields: Readonly> = {}): void { this.#log("info", message, fields); } + warn(message: string, fields: Readonly> = {}): void { this.#log("warn", message, fields); } + error(message: string, fields: Readonly> = {}): void { this.#log("error", message, fields); } + #log(level: "info" | "warn" | "error", message: string, fields: Readonly>): void { + const value = redactStructuredValue({ timestamp: new Date().toISOString(), level, message: message.slice(0, 512), ...this.base, ...fields }); + this.write(JSON.stringify(value)); + } +} + +interface MetricValue { value: number; labels: Readonly>; } +function labelsKey(labels: Readonly>): string { + return Object.entries(labels).sort(([a], [b]) => a.localeCompare(b)).map(([key, value]) => `${key}=${value}`).join("\u0000"); +} +function escaped(value: string): string { return value.replace(/\\/gu, "\\\\").replace(/\n/gu, "\\n").replace(/"/gu, '\\"'); } + +export class ClusterMetrics { + readonly #base: Readonly>; + readonly #metrics = new Map(); + constructor(base: Readonly>) { + this.#base = this.#labels(base); + } + increment(name: string, labels: Readonly>, amount = 1): void { + this.#name(name); + if (!Number.isFinite(amount) || amount < 0) throw new Error("metric increment is invalid"); + const merged = this.#labels({ ...this.#base, ...labels }); + const key = `${name}\u0000${labelsKey(merged)}`; + const current = this.#metrics.get(key); + this.#metrics.set(key, { labels: merged, value: (current?.value ?? 0) + amount }); + } + set(name: string, value: number, labels: Readonly> = {}): void { + this.#name(name); + if (!Number.isFinite(value)) throw new Error("metric value is invalid"); + const merged = this.#labels({ ...this.#base, ...labels }); + this.#metrics.set(`${name}\u0000${labelsKey(merged)}`, { labels: merged, value }); + } + render(): string { + return [...this.#metrics.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([key, metric]) => { + const name = key.slice(0, key.indexOf("\u0000")); + const labels = Object.entries(metric.labels).sort(([a], [b]) => a.localeCompare(b)).map(([label, value]) => `${label}="${escaped(value)}"`).join(","); + return `${name}${labels ? `{${labels}}` : ""} ${metric.value}`; + }).join("\n") + "\n"; + } + #name(name: string): void { if (!METRIC_NAME.test(name)) throw new Error("metric name is invalid"); } + #labels(labels: Readonly>): Readonly> { + if (Object.keys(labels).length > 8) throw new Error("metric label limit exceeded"); + const output: Record = {}; + for (const [key, value] of Object.entries(labels)) { + if (!LABEL_NAME.test(key) || !METRIC_LABELS.has(key) || redactKey(key) || typeof value !== "string" || value.length === 0 || new TextEncoder().encode(value).byteLength > 128) + throw new Error("metric label is invalid"); + output[key] = value; + } + return Object.freeze(output); + } +} + +export class ClusterServerHealth { + #kubernetesSynced = false; + #gatewayListening = false; + #draining = false; + markKubernetesSynced(): void { this.#kubernetesSynced = true; } + markKubernetesUnavailable(): void { this.#kubernetesSynced = false; } + markGatewayListening(): void { this.#gatewayListening = true; } + markGatewayStopped(): void { this.#gatewayListening = false; } + beginDrain(): void { this.#draining = true; } + get healthy(): boolean { return true; } + get ready(): boolean { return this.#kubernetesSynced && this.#gatewayListening && !this.#draining; } + get draining(): boolean { return this.#draining; } +} + +export interface AdminHandlerOptions { + readonly health: ClusterServerHealth; + readonly metrics: ClusterMetrics; +} +function json(value: unknown, status = 200): Response { + return Response.json(value, { status, headers: { "cache-control": "no-store" } }); +} +export function createAdminHandler(options: AdminHandlerOptions): (request: Request) => Promise { + return async request => { + const path = new URL(request.url).pathname; + if (path === "/healthz") return request.method === "GET" ? json({ healthy: options.health.healthy }) : new Response(null, { status: 405 }); + if (path === "/readyz") return request.method === "GET" ? json({ ready: options.health.ready }, options.health.ready ? 200 : 503) : new Response(null, { status: 405 }); + if (path === "/metrics") return request.method === "GET" ? new Response(options.metrics.render(), { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8", "cache-control": "no-store" } }) : new Response(null, { status: 405 }); + return new Response("not found", { status: 404 }); + }; +} diff --git a/packages/cluster-server/src/pod-host-router.ts b/packages/cluster-server/src/pod-host-router.ts new file mode 100644 index 00000000..db26affc --- /dev/null +++ b/packages/cluster-server/src/pod-host-router.ts @@ -0,0 +1,162 @@ +import { + DEVICE_CAPABILITIES, + PROTOCOL_FEATURES, + decodeServerFrame, + parseBounded, + type ClientFrame, + type HostId, + type ServerFrame, +} from "@t4-code/host-wire"; +import { readClusterIdentityToken } from "./config.ts"; + +export interface PodHostEndpoint { + readonly clusterSessionId: string; + readonly url: string; +} +export interface PodHostRoute extends PodHostEndpoint { + readonly upstreamSessionId: string; +} +export interface PodHostConnection { + readonly hostId?: string; + send(frame: ClientFrame): void; + close(code?: number, reason?: string): void; +} +export interface PodHostConnector { + connect(endpoint: PodHostEndpoint, onFrame: (frame: ServerFrame) => void, onClose?: () => void): Promise; +} +export interface WebSocketPodHostConnectorOptions { + readonly identityTokenFile: string; + readonly openTimeoutMs?: number; + readonly keepAliveMs?: number; + readonly webSocketFactory?: (url: string) => WebSocket; +} + +export class WebSocketPodHostConnector implements PodHostConnector { + readonly #identityTokenFile: string; + readonly #timeoutMs: number; + readonly #factory: (url: string) => WebSocket; + readonly #keepAliveMs: number; + constructor(options: WebSocketPodHostConnectorOptions) { + this.#identityTokenFile = options.identityTokenFile; + this.#timeoutMs = options.openTimeoutMs ?? 10_000; + this.#keepAliveMs = options.keepAliveMs ?? 30_000; + if (!Number.isSafeInteger(this.#keepAliveMs) || this.#keepAliveMs < 10 || this.#keepAliveMs > 60_000) + throw new Error("pod host keepalive interval is invalid"); + this.#factory = options.webSocketFactory ?? (url => new WebSocket(url)); + } + connect(endpoint: PodHostEndpoint, onFrame: (frame: ServerFrame) => void, onClose?: () => void): Promise { + if (!endpoint.url.startsWith("ws://") && !endpoint.url.startsWith("wss://")) return Promise.reject(new Error("pod host URL is invalid")); + const socket = this.#factory(endpoint.url); + return new Promise((resolve, reject) => { + let settled = false; + let upstreamHostId: string | undefined; + let heartbeat: ReturnType | undefined; + let heartbeatNonce: string | undefined; + const timer = setTimeout(() => { + if (!settled) { settled = true; socket.close(1013, "upstream timeout"); reject(new Error("pod host handshake timed out")); } + }, this.#timeoutMs); + socket.addEventListener("open", () => { + void readClusterIdentityToken(this.#identityTokenFile).then(token => { + if (settled) return; + socket.send(JSON.stringify({ + v: "omp-app/1", type: "hello", + protocol: { min: "omp-app/1", max: "omp-app/1" }, + client: { name: "cluster-server", version: "0.1.30", build: "cluster", platform: "linux" }, + requestedFeatures: PROTOCOL_FEATURES.filter(feature => feature !== "cluster.operator"), + savedCursors: [], + capabilities: { client: DEVICE_CAPABILITIES.filter(capability => capability !== "ci.trigger") }, + authentication: { deviceId: "cluster-server", deviceToken: token }, + })); + }, () => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.close(1011, "identity unavailable"); + reject(new Error("pod host identity is unavailable")); + }); + }); + socket.addEventListener("message", event => { + try { + const frame = decodeServerFrame(parseBounded(typeof event.data === "string" ? event.data : new Uint8Array(event.data as ArrayBuffer))); + if (frame.type === "welcome") { + if (settled) throw new Error("duplicate pod host welcome"); + if (frame.authentication !== "paired") throw new Error("pod host authentication failed"); + upstreamHostId = frame.hostId; + settled = true; + clearTimeout(timer); + heartbeat = setInterval(() => { + if (socket.readyState !== WebSocket.OPEN) return; + heartbeatNonce = `cluster-${Date.now().toString(36)}`; + try { socket.send(JSON.stringify({ v: "omp-app/1", type: "ping", nonce: heartbeatNonce, timestamp: new Date().toISOString() })); } + catch { socket.close(1011, "upstream keepalive failed"); } + }, this.#keepAliveMs); + resolve({ + get hostId() { return upstreamHostId; }, + send: value => { + if (socket.readyState !== WebSocket.OPEN) throw new Error("pod host connection is closed"); + socket.send(JSON.stringify(value)); + }, + close: (code, reason) => { + clearInterval(heartbeat); + socket.close(code, reason); + }, + }); + return; + } + if (!settled) throw new Error("pod host frame arrived before welcome"); + if (frame.type === "pong" && frame.nonce === heartbeatNonce) return; + onFrame(frame); + } catch (error) { + if (!settled) { + settled = true; + clearTimeout(timer); + socket.close(1002, "invalid upstream frame"); + reject(error); + } else socket.close(1002, "invalid upstream frame"); + } + }); + socket.addEventListener("error", () => { + if (!settled) { settled = true; clearTimeout(timer); reject(new Error("pod host connection failed")); } + }); + socket.addEventListener("close", () => { + clearInterval(heartbeat); + if (!settled) { settled = true; clearTimeout(timer); reject(new Error("pod host connection closed")); } + else onClose?.(); + }); + }); + } +} + +/** Only app-wire address fields are translated; opaque ids and payload values remain byte-semantically unchanged. */ +export function rewriteClientAddress( + frame: ClientFrame, + route: PodHostRoute, + upstreamHostId: string, +): ClientFrame { + if (!("hostId" in frame)) return frame; + return { + ...frame, + hostId: upstreamHostId as HostId, + ...(frame.sessionId === undefined ? {} : { sessionId: route.upstreamSessionId }), + } as ClientFrame; +} + +function rewriteEntry(entry: Record, clusterHostId: string, clusterSessionId: string): Record { + return { ...entry, hostId: clusterHostId, sessionId: clusterSessionId }; +} +export function rewriteServerAddress( + frame: ServerFrame, + route: PodHostRoute, + clusterHostId: string, +): ServerFrame { + let output: Record = { ...frame }; + if ("hostId" in output) output.hostId = clusterHostId; + if ("sessionId" in output) output.sessionId = route.clusterSessionId; + if (frame.type === "entry") output.entry = rewriteEntry(frame.entry as unknown as Record, clusterHostId, route.clusterSessionId); + if (frame.type === "snapshot") output.entries = frame.entries.map(entry => rewriteEntry(entry as unknown as Record, clusterHostId, route.clusterSessionId)); + if (frame.type === "agent.transcript") output.entries = frame.entries.map(entry => rewriteEntry(entry as unknown as Record, clusterHostId, route.clusterSessionId)); + if (frame.type === "session.delta" && frame.upsert) output.upsert = { ...frame.upsert, hostId: clusterHostId, sessionId: route.clusterSessionId }; + if (frame.type === "session.delta" && frame.remove) output.remove = route.clusterSessionId; + if (frame.type === "sessions") output.sessions = frame.sessions.map(ref => ({ ...ref, hostId: clusterHostId, sessionId: route.clusterSessionId })); + return output as unknown as ServerFrame; +} diff --git a/packages/cluster-server/src/server.ts b/packages/cluster-server/src/server.ts new file mode 100644 index 00000000..520b13e1 --- /dev/null +++ b/packages/cluster-server/src/server.ts @@ -0,0 +1,136 @@ +import { BlockList, isIP } from "node:net"; +import { normalizeIpAddress } from "@t4-code/host-service"; +import type { GatewayConnection, ClusterGateway } from "./gateway.ts"; +import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; +import { ClusterMetrics, ClusterServerHealth, JsonLogger, createAdminHandler } from "./observability.ts"; + +interface SocketData { connection?: GatewayConnection; principal: string; } +export interface ClusterHttpServersOptions { + readonly gateway: ClusterGateway; + readonly projection: ClusterInfrastructureProjection; + readonly gatewayPort: number; + readonly adminPort: number; + readonly trustedProxyAddresses?: readonly string[]; + readonly trustedProxyCidrs?: readonly string[]; + readonly health: ClusterServerHealth; + readonly metrics: ClusterMetrics; + readonly logger: JsonLogger; +} +export interface ClusterHttpServers { + drain(): Promise; + stop(): Promise; +} +function trustedProxyMatcher(addresses: readonly string[], cidrs: readonly string[]): (address: string) => boolean { + const exact = new Set(addresses.map(normalizeIpAddress)); + const subnets = new BlockList(); + for (const cidr of cidrs) { + const [address, prefixText] = cidr.split("/"); + const family = isIP(address!); + if ((family !== 4 && family !== 6) || !/^(?:0|[1-9][0-9]*)$/u.test(prefixText ?? "")) + throw new Error("trusted proxy CIDR is invalid"); + subnets.addSubnet(address!, Number(prefixText), family === 4 ? "ipv4" : "ipv6"); + } + return address => { + const normalized = normalizeIpAddress(address); + const family = isIP(normalized); + return exact.has(normalized) || family !== 0 && subnets.check(normalized, family === 4 ? "ipv4" : "ipv6"); + }; +} + +export function isLoopbackAddress(address: string): boolean { + const normalized = normalizeIpAddress(address); + if (normalized === "::1") return true; + if (isIP(normalized) !== 4) return false; + const firstOctet = Number(normalized.split(".", 1)[0]); + return firstOctet === 127; +} + +function gatewayPrincipal(request: Request, remoteAddress: string, trustedSource: (address: string) => boolean): string | undefined { + if (!trustedSource(remoteAddress)) return undefined; + if (request.headers.get("x-forwarded-proto") !== "https") return undefined; + const principal = request.headers.get("tailscale-user-login") ?? request.headers.get("tailscale-user-name"); + if (!principal || principal !== principal.trim() || new TextEncoder().encode(principal).byteLength > 256 || /\p{Cc}/u.test(principal)) return undefined; + return principal; +} + +export function startClusterHttpServers(options: ClusterHttpServersOptions): ClusterHttpServers { + let draining = false; + const trustedSource = trustedProxyMatcher(options.trustedProxyAddresses ?? [], options.trustedProxyCidrs ?? []); + const gatewayServer = Bun.serve({ + hostname: "0.0.0.0", + port: options.gatewayPort, + fetch(request, server) { + const url = new URL(request.url); + if (url.pathname !== "/v1/ws") return new Response("not found", { status: 404 }); + if (request.method !== "GET") return new Response("method not allowed", { status: 405 }); + if (draining) return new Response("draining", { status: 503 }); + const remoteAddress = server.requestIP(request)?.address; + if (!remoteAddress) return new Response("authenticated proxy required", { status: 401 }); + const principal = gatewayPrincipal(request, remoteAddress, trustedSource); + if (!principal) return new Response("authenticated Tailscale proxy identity required", { status: 401 }); + const origin = request.headers.get("origin"); + if (origin && !options.projection.allowedOrigins().includes(origin)) return new Response("forbidden", { status: 403 }); + if (!server.upgrade(request, { data: { principal } })) return new Response("upgrade required", { status: 426 }); + return undefined; + }, + websocket: { + maxPayloadLength: 1_048_576, + idleTimeout: 120, + backpressureLimit: 1_048_576, + closeOnBackpressureLimit: true, + perMessageDeflate: false, + open(socket) { + socket.data.connection = options.gateway.connect({ + send(frame) { + const accepted = socket.send(JSON.stringify(frame)); + if (accepted <= 0) socket.close(1013, "gateway backpressure"); + }, + close(code, reason) { socket.close(code, reason); }, + }, socket.data.principal); + options.metrics.set("t4_cluster_gateway_connections", options.gateway.connectionCount); + }, + message(socket, message) { + const input = typeof message === "string" ? message : new Uint8Array(message); + void socket.data.connection?.receive(input).catch(() => socket.close(1011, "gateway error")); + }, + close(socket) { + socket.data.connection?.close(); + socket.data.connection = undefined; + options.metrics.set("t4_cluster_gateway_connections", options.gateway.connectionCount); + }, + }, + }); + let stopping: Promise | undefined; + const drain = async (): Promise => { + if (stopping) return await stopping; + draining = true; + options.health.beginDrain(); + options.gateway.beginDrain(); + stopping = gatewayServer.stop(false).then(() => undefined); + options.logger.info("gateway drain started", { result: "success" }); + await stopping; + options.health.markGatewayStopped(); + }; + const adminHandler = createAdminHandler({ health: options.health, metrics: options.metrics }); + const adminServer = Bun.serve({ + hostname: "0.0.0.0", + port: options.adminPort, + async fetch(request, server) { + if (new URL(request.url).pathname !== "/drainz") return adminHandler(request); + if (request.method !== "POST") return new Response(null, { status: 405 }); + const source = server.requestIP(request)?.address; + if (!source || !isLoopbackAddress(source)) return new Response("forbidden", { status: 403 }); + await drain(); + return Response.json({ draining: true }, { status: 202, headers: { "cache-control": "no-store" } }); + }, + }); + options.health.markGatewayListening(); + options.logger.info("cluster server listening", { result: "success" }); + return { + drain, + async stop() { + await drain(); + await adminServer.stop(true); + }, + }; +} diff --git a/packages/cluster-server/src/session-authority-runner.ts b/packages/cluster-server/src/session-authority-runner.ts new file mode 100644 index 00000000..307cd21d --- /dev/null +++ b/packages/cluster-server/src/session-authority-runner.ts @@ -0,0 +1,100 @@ +import type { ServerFrame, SessionRef } from "@t4-code/host-wire"; +import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; +import type { PodHostConnection, PodHostConnector, PodHostEndpoint } from "./pod-host-router.ts"; + +export interface SessionAuthorityRunnerOptions { + readonly projection: ClusterInfrastructureProjection; + readonly connector: PodHostConnector; + readonly retryMs?: number; + readonly onError?: (error: unknown) => void; +} + +interface AuthorityConnection { + readonly endpoint: PodHostEndpoint; + readonly pending: Promise; +} + +/** Reconstructs session truth from each pod's authenticated omp-app/1 inventory. */ +export class SessionAuthorityRunner { + readonly #options: SessionAuthorityRunnerOptions; + readonly #connections = new Map(); + #unsubscribe?: () => void; + #retry?: ReturnType; + #started = false; + + constructor(options: SessionAuthorityRunnerOptions) { this.#options = options; } + + start(): void { + if (this.#started) throw new Error("session authority runner already started"); + this.#started = true; + this.#unsubscribe = this.#options.projection.subscribeSessions(() => this.#reconcile()); + this.#reconcile(); + } + + async stop(): Promise { + if (!this.#started) return; + this.#started = false; + this.#unsubscribe?.(); + this.#unsubscribe = undefined; + if (this.#retry) clearTimeout(this.#retry); + this.#retry = undefined; + const pending = [...this.#connections.values()].map(entry => entry.pending); + this.#connections.clear(); + await Promise.allSettled(pending.map(async connection => (await connection).close(1001, "cluster server stopping"))); + } + + #reconcile(): void { + if (!this.#started) return; + const endpoints = new Map(this.#options.projection.sessionEndpoints().map(endpoint => [endpoint.clusterSessionId, endpoint])); + for (const [session, existing] of this.#connections) { + const current = endpoints.get(session); + if (current?.url === existing.endpoint.url) continue; + this.#connections.delete(session); + this.#options.projection.clearSessionAuthority(session); + void existing.pending.then(connection => connection.close(1001, "session endpoint changed"), () => undefined); + } + for (const [session, endpoint] of endpoints) { + if (this.#connections.has(session)) continue; + let pending: Promise; + pending = this.#options.connector.connect( + endpoint, + frame => this.#projectFrame(endpoint, frame), + () => this.#disconnected(session, pending), + ); + this.#connections.set(session, { endpoint, pending }); + pending.catch(error => { + if (this.#connections.get(session)?.pending !== pending) return; + this.#connections.delete(session); + this.#options.projection.clearSessionAuthority(session); + this.#options.onError?.(error); + this.#scheduleRetry(); + }); + } + } + + #projectFrame(endpoint: PodHostEndpoint, frame: ServerFrame): void { + if (frame.type === "sessions") { + if (frame.sessions.length !== 1) throw new Error("session pod must expose exactly one authoritative OMP session"); + this.#options.projection.setSessionAuthority(endpoint.clusterSessionId, frame.sessions[0] as SessionRef); + return; + } + if (frame.type !== "session.delta") return; + if (frame.upsert) this.#options.projection.setSessionAuthority(endpoint.clusterSessionId, frame.upsert); + else if (frame.remove) this.#options.projection.clearSessionAuthority(endpoint.clusterSessionId); + } + + #disconnected(session: string, pending: Promise): void { + if (this.#connections.get(session)?.pending !== pending) return; + this.#connections.delete(session); + this.#options.projection.clearSessionAuthority(session); + this.#scheduleRetry(); + } + + #scheduleRetry(): void { + if (!this.#started || this.#retry) return; + this.#retry = setTimeout(() => { + this.#retry = undefined; + this.#reconcile(); + }, this.#options.retryMs ?? 1_000); + } +} diff --git a/packages/cluster-server/src/session-host-main.ts b/packages/cluster-server/src/session-host-main.ts new file mode 100755 index 00000000..72181224 --- /dev/null +++ b/packages/cluster-server/src/session-host-main.ts @@ -0,0 +1,121 @@ +#!/usr/bin/env bun +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { + OmpAuthorityBridgeClient, + TranscriptSearchIndex, + appserverSupportedCapabilities, + appserverSupportedFeatures, + createAppserver, + type AppserverHandle, +} from "@t4-code/host-service"; +import { hostId } from "@t4-code/host-wire"; +import { ClusterInternalRemotePolicy, sessionHostConfigFromEnv, type SessionHostConfig } from "./session-host-policy.ts"; +import { KubernetesTokenReviewer } from "./kubernetes-client.ts"; + +const OMP_VERSION = "17.0.5"; +const OMP_COMMIT = "8476f4451ed95c5d5401785d279a93d3c659fac4"; + +export async function runSessionHost( + config: SessionHostConfig, + registerSignal: (signal: "SIGINT" | "SIGTERM", listener: () => void) => void = (signal, listener) => process.on(signal, listener), +): Promise { + const home = join(config.stateRoot, "home"); + const runtime = join(config.stateRoot, "run"); + await Promise.all([mkdir(home, { recursive: true, mode: 0o700 }), mkdir(runtime, { recursive: true, mode: 0o700 })]); + const bridge = new OmpAuthorityBridgeClient({ + executable: config.ompExecutable, + cwd: "/workspace", + environment: { + OMP_PROFILE: config.sessionName, + HOME: home, + XDG_CONFIG_HOME: join(home, ".config"), + XDG_STATE_HOME: join(home, ".local", "state"), + XDG_CACHE_HOME: join(home, ".cache"), + XDG_RUNTIME_DIR: runtime, + }, + }); + const ready = await bridge.start(); + if (ready.ompVersion !== OMP_VERSION || ready.ompBuild !== OMP_COMMIT) { + await bridge.stop(); + throw new Error("session pod OMP authority identity does not match the pinned release"); + } + let appserver: AppserverHandle | undefined; + const search = new TranscriptSearchIndex(join(config.stateRoot, "transcript-search.sqlite")); + try { + const authorities = bridge.createAuthorities(); + const existing = await authorities.sessionAuthority.list(); + if (existing.length > 1) throw new Error("session pod contains more than one authoritative OMP session"); + if (existing.length === 0) await authorities.sessionAuthority.create("/workspace", config.sessionName); + const hostInfo = await authorities.hostInfo(); + const base = { + hostId: hostId(`pod:${config.sessionName}`), + ...(process.env.POD_UID ? { epoch: `pod:${process.env.POD_UID}` } : {}), + socketPath: join(runtime, "appserver.sock"), + attentionOutcomePath: join(config.stateRoot, "attention-outcomes.json"), + ompVersion: ready.ompVersion, + ompBuild: ready.ompBuild, + appserverVersion: "0.1.30", + appserverBuild: "cluster-session", + sessionAuthority: authorities.sessionAuthority, + discovery: authorities.discovery, + operationsAuthority: authorities.operationsAuthority, + ...(authorities.usageAuthority ? { usageAuthority: authorities.usageAuthority } : {}), + transcriptSearchAuthority: search, + projectRootForProject: authorities.projectRootForProject, + lockCheck: authorities.lockCheck, + lockStatus: authorities.lockStatus, + transcriptImageRoot: hostInfo.transcriptImageRoot, + rpcChildInvocation: { executable: config.ompExecutable, prefixArgv: [] }, + }; + const policy = new ClusterInternalRemotePolicy({ + reviewer: new KubernetesTokenReviewer({ + baseUrl: config.kubernetesBaseUrl, + tokenPath: config.kubernetesTokenPath, + caPath: config.kubernetesCaPath, + namespacePath: config.kubernetesNamespacePath, + serverServiceAccountName: config.serverServiceAccountName, + }), + supportedCapabilities: appserverSupportedCapabilities(base), + supportedFeatures: appserverSupportedFeatures(base, true), + }); + appserver = createAppserver({ + ...base, + remoteEndpoint: { + address: "0.0.0.0", + port: config.port, + internalPeerNodeId: "cluster-server", + originAllowlist: [], + maxConnections: 8, + maxFrameBytes: 1_048_576, + idleTimeoutSeconds: 120, + backpressureLimit: 1_048_576, + }, + remotePolicy: policy, + }); + await appserver.start(); + const stopped = Promise.withResolvers(); + let stopping = false; + const stop = (): void => { + if (stopping) return; + stopping = true; + void appserver!.stop().then(stopped.resolve, stopped.reject); + }; + registerSignal("SIGINT", stop); + registerSignal("SIGTERM", stop); + await stopped.promise; + } finally { + await appserver?.stop().catch(() => undefined); + await Promise.resolve(search.close()).catch(() => undefined); + await bridge.stop(); + } +} + +async function main(): Promise { + try { await runSessionHost(sessionHostConfigFromEnv(process.env)); } + catch (error) { + process.stderr.write(`${JSON.stringify({ component: "session-host", level: "error", message: error instanceof Error ? error.message : "session host failed" })}\n`); + process.exitCode = 1; + } +} +if (import.meta.main) await main(); diff --git a/packages/cluster-server/src/session-host-policy.ts b/packages/cluster-server/src/session-host-policy.ts new file mode 100644 index 00000000..4c2f3abb --- /dev/null +++ b/packages/cluster-server/src/session-host-policy.ts @@ -0,0 +1,149 @@ +import { isAbsolute } from "node:path"; +import { + decodeClientFrame, + requiredCapability, + type ClientFrame, + type HelloFrame, +} from "@t4-code/host-wire"; +import type { + RemoteAuthorizationContext, + RemoteConnectionPolicy, + RemoteHelloDecision, +} from "@t4-code/host-service"; +import type { RemoteConnection } from "@t4-code/host-service"; + +const SESSION_NAME = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; +const AUDIENCE = /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/u; +const INTERNAL_TOKEN_PLACEHOLDER = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; +const MAX_PROJECTED_TOKEN_BYTES = 16_384; +interface ConnectionGrant { capabilities: Set; features: Set; } +export interface ClusterIdentityReviewer { + review(token: string): Promise; +} +export interface ClusterInternalRemotePolicyOptions { + readonly reviewer: ClusterIdentityReviewer; + readonly supportedCapabilities: readonly string[]; + readonly supportedFeatures: readonly string[]; +} + +function projectedToken(value: unknown): string { + if (typeof value !== "string") throw new Error("cluster identity token is invalid"); + const bytes = new TextEncoder().encode(value).byteLength; + if (bytes < 32 || bytes > MAX_PROJECTED_TOKEN_BYTES || /\s/u.test(value)) throw new Error("cluster identity token is invalid"); + return value; +} +export function decodeClusterInternalClientFrame(input: unknown): ClientFrame { + if (!input || typeof input !== "object" || Array.isArray(input)) return decodeClientFrame(input); + const source = input as Record; + if (source.type !== "hello" || !source.authentication || typeof source.authentication !== "object" || Array.isArray(source.authentication)) + return decodeClientFrame(input); + const authentication = source.authentication as Record; + const token = projectedToken(authentication.deviceToken); + const decoded = decodeClientFrame({ ...source, authentication: { ...authentication, deviceToken: INTERNAL_TOKEN_PLACEHOLDER } }); + if (decoded.type !== "hello" || !decoded.authentication) throw new Error("cluster identity authentication is required"); + return { ...decoded, authentication: { ...decoded.authentication, deviceToken: token } }; +} + +export class ClusterInternalRemotePolicy implements RemoteConnectionPolicy { + readonly #reviewer: ClusterIdentityReviewer; + readonly #capabilities: readonly string[]; + readonly #features: readonly string[]; + readonly #connections = new Map(); + constructor(options: ClusterInternalRemotePolicyOptions) { + this.#reviewer = options.reviewer; + this.#capabilities = [...new Set(options.supportedCapabilities)].filter(value => value !== "ci.trigger"); + this.#features = [...new Set(options.supportedFeatures)].filter(value => value !== "cluster.operator"); + } + decodeClientFrame(input: unknown): ClientFrame { return decodeClusterInternalClientFrame(input); } + async authenticate(connection: RemoteConnection, hello: HelloFrame): Promise { + const authentication = hello.authentication; + if (connection.peer.identity.nodeId !== "cluster-server" || authentication?.deviceId !== "cluster-server") { + this.#connections.delete(connection.connectionId); + return { authenticated: false, authentication: "denied", grantedCapabilities: [], grantedFeatures: [] }; + } + try { + if (!await this.#reviewer.review(authentication.deviceToken)) { + this.#connections.delete(connection.connectionId); + return { authenticated: false, authentication: "denied", grantedCapabilities: [], grantedFeatures: [] }; + } + } catch { + this.#connections.delete(connection.connectionId); + return { authenticated: false, authentication: "denied", grantedCapabilities: [], grantedFeatures: [] }; + } + const requestedCapabilities = new Set(hello.capabilities?.client ?? this.#capabilities); + const requestedFeatures = new Set(hello.requestedFeatures); + const grantedCapabilities = this.#capabilities.filter(value => requestedCapabilities.has(value)); + const grantedFeatures = this.#features.filter(value => requestedFeatures.has(value)); + this.#connections.set(connection.connectionId, { capabilities: new Set(grantedCapabilities), features: new Set(grantedFeatures) }); + return { authenticated: true, authentication: "paired", deviceId: "cluster-server", grantedCapabilities, grantedFeatures }; + } + authorize(connection: RemoteConnection, frame: ClientFrame, _context: RemoteAuthorizationContext): boolean { + const grant = this.#connections.get(connection.connectionId); + if (!grant) return false; + if (frame.type === "confirm") return true; + if (frame.type === "ping") return true; + if (frame.type === "terminal.input") return grant.features.has("terminal.io") && grant.capabilities.has("term.input"); + if (frame.type === "terminal.resize") return grant.features.has("terminal.io") && grant.capabilities.has("term.resize"); + if (frame.type === "terminal.close") return grant.features.has("terminal.io") && grant.capabilities.has("term.open"); + if (frame.type !== "command") return false; + const capability = requiredCapability(frame.command); + return capability !== undefined && grant.capabilities.has(capability); + } + disconnected(connection: RemoteConnection): void { this.#connections.delete(connection.connectionId); } +} + +export interface SessionHostConfig { + readonly kubernetesBaseUrl: string; + readonly kubernetesTokenPath: string; + readonly kubernetesCaPath: string; + readonly kubernetesNamespacePath: string; + readonly kubernetesApiAudience: string; + readonly serverServiceAccountName: string; + readonly sessionName: string; + readonly ompExecutable: string; + readonly stateRoot: string; + readonly port: number; +} +function required(env: Readonly>, name: string): string { + const value = env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} +function dns(value: string, name: string): string { + if (!SESSION_NAME.test(value)) throw new Error(`${name} is invalid`); + return value; +} +function absolutePath(value: string, name: string): string { + if (!isAbsolute(value)) throw new Error(`${name} must be absolute`); + return value; +} +function audience(value: string): string { + if (value.length > 253 || !AUDIENCE.test(value)) throw new Error("T4_KUBERNETES_API_AUDIENCE is invalid"); + return value; +} +export function sessionHostConfigFromEnv(env: Readonly>): SessionHostConfig { + const serviceHost = required(env, "KUBERNETES_SERVICE_HOST"); + const servicePort = Number(env.KUBERNETES_SERVICE_PORT_HTTPS ?? env.KUBERNETES_SERVICE_PORT ?? "443"); + if (!Number.isSafeInteger(servicePort) || servicePort < 1 || servicePort > 65_535) throw new Error("KUBERNETES_SERVICE_PORT is invalid"); + const serverServiceAccountName = dns(required(env, "T4_CLUSTER_SERVER_SERVICE_ACCOUNT"), "T4_CLUSTER_SERVER_SERVICE_ACCOUNT"); + const sessionName = dns(required(env, "T4_SESSION_NAME"), "T4_SESSION_NAME"); + const ompExecutable = env.T4_OMP_EXECUTABLE ?? "/opt/t4/bin/omp"; + if (!isAbsolute(ompExecutable)) throw new Error("T4_OMP_EXECUTABLE must be absolute"); + const stateRoot = absolutePath(required(env, "T4_SESSION_STATE_ROOT"), "T4_SESSION_STATE_ROOT"); + if (!/^\/workspace\/\.t4\/sessions\/[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u.test(stateRoot)) + throw new Error("T4_SESSION_STATE_ROOT must select one isolated session directory"); + const port = Number(env.T4_SESSION_HOST_PORT ?? "8787"); + if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) throw new Error("T4_SESSION_HOST_PORT is invalid"); + return { + kubernetesBaseUrl: `https://${serviceHost}:${servicePort}`, + kubernetesTokenPath: absolutePath(env.T4_KUBERNETES_TOKEN_PATH ?? "/var/run/secrets/kubernetes.io/serviceaccount/token", "T4_KUBERNETES_TOKEN_PATH"), + kubernetesCaPath: absolutePath(env.T4_KUBERNETES_CA_PATH ?? "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", "T4_KUBERNETES_CA_PATH"), + kubernetesNamespacePath: absolutePath(env.T4_KUBERNETES_NAMESPACE_PATH ?? "/var/run/secrets/kubernetes.io/serviceaccount/namespace", "T4_KUBERNETES_NAMESPACE_PATH"), + kubernetesApiAudience: audience(env.T4_KUBERNETES_API_AUDIENCE ?? "https://kubernetes.default.svc"), + serverServiceAccountName, + sessionName, + ompExecutable, + stateRoot, + port, + }; +} diff --git a/packages/cluster-server/src/woodpecker.ts b/packages/cluster-server/src/woodpecker.ts new file mode 100644 index 00000000..2e3a8ad2 --- /dev/null +++ b/packages/cluster-server/src/woodpecker.ts @@ -0,0 +1,246 @@ +import { createHash } from "node:crypto"; +import { isAbsolute } from "node:path"; +import { readBoundedRegularFile } from "./config.ts"; +import type { CiRunResult, SessionCiState } from "@t4-code/host-wire"; + +export const WOODPECKER_MAX_PIPELINES = 100; +export const WOODPECKER_MAX_RESPONSE_BYTES = 1024 * 1024; +export const WOODPECKER_MAX_INFLIGHT_TRIGGERS = 128; +export const WOODPECKER_MAX_STAGES = 128; + +export interface CiCorrelation { + readonly sessionId: string; + readonly repositoryId: string; + readonly ref: string; + readonly commit: string; +} +export interface CiRunCorrelation extends CiCorrelation { readonly commandId: string; } +export interface CiProvider { + readonly name: "woodpecker"; + query(correlation: CiCorrelation, signal?: AbortSignal): Promise; + run(correlation: CiRunCorrelation, signal?: AbortSignal): Promise; +} +export interface WoodpeckerRepositoryConfig { readonly slug: string; } +export interface WoodpeckerProviderOptions { + readonly baseUrl: string; + readonly webBaseUrl?: string; + readonly token?: string; + readonly tokenFile?: string; + readonly repositories: Readonly>; + readonly fetch?: typeof globalThis.fetch; +} +interface WoodpeckerPipeline { + readonly number?: unknown; + readonly status?: unknown; + readonly ref?: unknown; + readonly branch?: unknown; + readonly commit?: unknown; + readonly created?: unknown; + readonly started?: unknown; + readonly finished?: unknown; + readonly variables?: unknown; + readonly stages?: unknown; +} +function httpsBase(value: string): string { + const url = new URL(value); + if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash) + throw new Error("Woodpecker web base URL must use HTTPS without credentials"); + return url.href.replace(/\/$/u, ""); +} +function apiBase(value: string): string { + const url = new URL(value); + const inCluster = url.protocol === "http:" && url.hostname.endsWith(".svc.cluster.local"); + if (!(url.protocol === "https:" || inCluster) || url.username || url.password || url.search || url.hash || !["", "/"].includes(url.pathname)) + throw new Error("Woodpecker API base URL must use HTTPS or an exact in-cluster Service URL without credentials"); + return url.href.replace(/\/$/u, ""); +} +function bounded(value: unknown, name: string, max: number): string { + if (typeof value !== "string" || value.length === 0 || new TextEncoder().encode(value).byteLength > max || /\p{Cc}/u.test(value)) + throw new Error(`${name} is invalid`); + return value; +} +function slug(value: string): string { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value) || value.includes("..")) throw new Error("Woodpecker repository slug is invalid"); + return value; +} +function pipelineStatus(value: unknown): NonNullable { + switch (value) { + case "pending": case "queued": case "blocked": return value === "blocked" ? "unknown" : "queued"; + case "running": return "running"; + case "success": return "success"; + case "failure": case "error": return "failure"; + case "killed": case "canceled": case "cancelled": return "killed"; + default: return "unknown"; + } +} +function positiveInteger(value: unknown): number | undefined { + return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : undefined; +} +function timestamp(value: unknown): string | undefined { + const seconds = typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : undefined; + return seconds === undefined ? undefined : new Date(seconds * 1_000).toISOString(); +} +function object(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} +function exactPipeline(pipeline: WoodpeckerPipeline, correlation: CiCorrelation, idempotencyKey?: string): boolean { + const variables = object(pipeline.variables); + return pipeline.ref === correlation.ref && pipeline.commit === correlation.commit && + variables.T4_SESSION_ID === correlation.sessionId && + (idempotencyKey === undefined || variables.T4_IDEMPOTENCY_KEY === idempotencyKey); +} + +export function mapWoodpeckerPipeline( + pipeline: WoodpeckerPipeline, + correlation: CiCorrelation, + link?: string, +): SessionCiState { + const number = positiveInteger(pipeline.number); + if (Array.isArray(pipeline.stages) && pipeline.stages.length > WOODPECKER_MAX_STAGES) throw new Error("Woodpecker stage response limit exceeded"); + const stages = Array.isArray(pipeline.stages) ? pipeline.stages.map(object) : []; + const current = stages.find(stage => stage.status === "running")?.name; + const repositoryId = bounded(correlation.repositoryId, "CI repository id", 256); + const ref = bounded(correlation.ref, "CI ref", 512); + const commit = bounded(correlation.commit, "CI commit", 512); + return { + provider: "woodpecker", + correlation: "exact", + repositoryId, + ...(typeof pipeline.branch === "string" ? { branch: bounded(pipeline.branch, "Woodpecker branch", 512) } : {}), + ref, + commit, + ...(number === undefined ? {} : { pipelineNumber: number }), + status: pipelineStatus(pipeline.status), + ...(typeof current === "string" ? { currentStage: bounded(current, "Woodpecker stage", 256) } : {}), + ...(timestamp(pipeline.created) ? { createdAt: timestamp(pipeline.created)! } : {}), + ...(timestamp(pipeline.started) ? { startedAt: timestamp(pipeline.started)! } : {}), + ...(timestamp(pipeline.finished) ? { finishedAt: timestamp(pipeline.finished)! } : {}), + ...(link ? { link: httpsBase(link) } : {}), + }; +} + +export class WoodpeckerProvider implements CiProvider { + readonly name = "woodpecker" as const; + readonly #baseUrl: string; + readonly #webBaseUrl: string; + readonly #token?: string; + readonly #tokenFile?: string; + readonly #repositories: Readonly>; + readonly #fetch: typeof globalThis.fetch; + readonly #inflight = new Map>(); + + constructor(options: WoodpeckerProviderOptions) { + this.#baseUrl = apiBase(options.baseUrl); + this.#webBaseUrl = options.webBaseUrl ? httpsBase(options.webBaseUrl) : httpsBase(options.baseUrl); + if (Boolean(options.token) === Boolean(options.tokenFile)) throw new Error("Woodpecker provider requires exactly one credential source"); + if (options.token) this.#token = bounded(options.token, "Woodpecker token", 16_384); + if (options.tokenFile) { + if (!isAbsolute(options.tokenFile)) throw new Error("Woodpecker token file must be absolute"); + this.#tokenFile = options.tokenFile; + } + const repositories: Record = {}; + for (const [id, config] of Object.entries(options.repositories)) { + bounded(id, "Woodpecker repository id", 256); + repositories[id] = { slug: slug(config.slug) }; + } + this.#repositories = Object.freeze(repositories); + this.#fetch = options.fetch ?? globalThis.fetch; + } + async query(correlation: CiCorrelation, signal?: AbortSignal): Promise { + return await this.#queryExact(correlation, signal); + } + + async #queryExact(correlation: CiCorrelation, signal?: AbortSignal, idempotencyKey?: string): Promise { + const repository = this.#repository(correlation.repositoryId); + this.#validateCorrelation(correlation); + const query = new URLSearchParams({ ref: correlation.ref, commit: correlation.commit, limit: String(WOODPECKER_MAX_PIPELINES) }); + const pipelines = await this.#json(`/api/repos/${encodeURIComponent(repository.slug)}/pipelines?${query}`, { signal }); + if (!Array.isArray(pipelines)) throw new Error("Woodpecker pipeline response is invalid"); + if (pipelines.length > WOODPECKER_MAX_PIPELINES) throw new Error("Woodpecker pipeline response limit exceeded"); + const exact = (pipelines as WoodpeckerPipeline[]).find(value => exactPipeline(value, correlation, idempotencyKey)); + if (!exact) return { + provider: "woodpecker", correlation: "unknown", repositoryId: correlation.repositoryId, + ref: correlation.ref, commit: correlation.commit, + }; + const number = positiveInteger(exact.number); + return mapWoodpeckerPipeline(exact, correlation, number ? `${this.#webBaseUrl}/repos/${repository.slug}/pipeline/${number}` : undefined); + } + async run(correlation: CiRunCorrelation, signal?: AbortSignal): Promise { + const key = `${correlation.repositoryId}\u0000${correlation.ref}\u0000${correlation.commit}\u0000${correlation.sessionId}\u0000${correlation.commandId}`; + const pending = this.#inflight.get(key); + if (pending) return await pending; + if (this.#inflight.size >= WOODPECKER_MAX_INFLIGHT_TRIGGERS) throw new Error("Woodpecker trigger concurrency limit exceeded"); + const operation = this.#runExact(correlation, signal); + this.#inflight.set(key, operation); + try { return await operation; } + finally { if (this.#inflight.get(key) === operation) this.#inflight.delete(key); } + } + async #runExact(correlation: CiRunCorrelation, signal?: AbortSignal): Promise { + const idempotencyKey = createHash("sha256") + .update(correlation.sessionId).update("\u0000").update(correlation.commandId) + .digest("base64url"); + const existing = await this.#queryExact(correlation, signal, idempotencyKey); + if (existing.correlation === "exact" && existing.pipelineNumber !== undefined) + return { triggered: false, pipelineNumber: existing.pipelineNumber, ...(existing.status ? { status: existing.status } : {}) }; + const repository = this.#repository(correlation.repositoryId); + const created = object(await this.#json(`/api/repos/${encodeURIComponent(repository.slug)}/pipelines`, { + method: "POST", + headers: { "content-type": "application/json", "idempotency-key": `t4-${idempotencyKey}` }, + body: JSON.stringify({ ref: correlation.ref, commit: correlation.commit, event: "manual", variables: { T4_SESSION_ID: correlation.sessionId, T4_IDEMPOTENCY_KEY: idempotencyKey } }), + signal, + })); + if (created.idempotencyKey !== idempotencyKey) + throw new Error("Woodpecker adapter did not confirm provider-side idempotency"); + const mapped = mapWoodpeckerPipeline(created, correlation); + return { + triggered: true, + ...(mapped.pipelineNumber === undefined ? {} : { pipelineNumber: mapped.pipelineNumber }), + ...(mapped.status === undefined ? {} : { status: mapped.status }), + }; + } + + #repository(repositoryId: string): WoodpeckerRepositoryConfig { + const repository = this.#repositories[repositoryId]; + if (!repository) throw new Error("Woodpecker repository is not allowlisted"); + return repository; + } + + #validateCorrelation(value: CiCorrelation): void { + bounded(value.sessionId, "CI session id", 256); + bounded(value.repositoryId, "CI repository id", 256); + bounded(value.ref, "CI ref", 512); + bounded(value.commit, "CI commit", 512); + if ("commandId" in value) bounded(value.commandId, "CI command id", 256); + } + async #credential(): Promise { + if (this.#token) return this.#token; + const token = (await readBoundedRegularFile(this.#tokenFile!, 16_384, "Woodpecker token")).trim(); + if (new TextEncoder().encode(token).byteLength < 32 || /\s/u.test(token)) throw new Error("Woodpecker token file is invalid"); + return token; + } + async #json(path: string, init: RequestInit): Promise { + const headers = { Authorization: `Bearer ${await this.#credential()}`, ...Object.fromEntries(new Headers(init.headers).entries()) }; + const response = await this.#fetch(`${this.#baseUrl}${path}`, { ...init, headers }); + const declaredLength = Number(response.headers.get("content-length")); + if (Number.isFinite(declaredLength) && declaredLength > WOODPECKER_MAX_RESPONSE_BYTES) throw new Error("Woodpecker response exceeds byte limit"); + if (!response.body) throw new Error("Woodpecker response body is unavailable"); + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let byteLength = 0; + for (;;) { + const chunk = await reader.read(); + if (chunk.done) break; + byteLength += chunk.value.byteLength; + if (byteLength > WOODPECKER_MAX_RESPONSE_BYTES) { await reader.cancel(); throw new Error("Woodpecker response exceeds byte limit"); } + chunks.push(chunk.value); + } + const bytes = new Uint8Array(byteLength); + let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } + let value: unknown; + try { value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); } + catch { throw new Error("Woodpecker response is not JSON"); } + if (!response.ok) throw new Error(`Woodpecker request failed with ${response.status}`); + return value; + } +} diff --git a/packages/cluster-server/test/ci-projection-runner.test.ts b/packages/cluster-server/test/ci-projection-runner.test.ts new file mode 100644 index 00000000..1466eefa --- /dev/null +++ b/packages/cluster-server/test/ci-projection-runner.test.ts @@ -0,0 +1,90 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; +import { hostId, projectId, revision, sessionId, type SessionCiState, type SessionRef } from "@t4-code/host-wire"; +import { CiProjectionRunner } from "../src/ci-projection-runner.ts"; +import { ClusterInfrastructureProjection } from "../src/kubernetes-projection.ts"; +import type { CiProvider } from "../src/woodpecker.ts"; + +const correlation = { + sessionId: "session-one", + repositoryId: "t4-code", + ref: "refs/heads/agent/t4-cluster-operator", + commit: "0123456789abcdef0123456789abcdef01234567", +}; +const exactState: SessionCiState = { + provider: "woodpecker", + correlation: "exact", + repositoryId: correlation.repositoryId, + ref: correlation.ref, + commit: correlation.commit, + pipelineNumber: 42, + status: "running", + currentStage: "cluster-server-tests", +}; +const authority: SessionRef = { + hostId: hostId("pod-host"), + sessionId: sessionId("omp-session"), + project: { projectId: projectId("workspace-one") }, + revision: revision("authority-r1"), + title: "Cluster session", + status: "active", + updatedAt: "2026-07-20T00:00:00.000Z", +}; + +function projection(): ClusterInfrastructureProjection { + const value = new ClusterInfrastructureProjection({ epoch: "replica-one", namespace: "development" }); + value.replace({ + host: { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4ClusterHost", metadata: { name: "primary", uid: "host-uid", resourceVersion: "1" }, spec: {} }, + workspaces: [{ apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", metadata: { name: "workspace-one", resourceVersion: "2" }, spec: { hostRef: "primary", owner: "owner@example.com", displayName: "Workspace", retentionPolicy: "Retain", size: "20Gi" }, status: { phase: "Ready" } }], + sessions: [{ apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", metadata: { name: correlation.sessionId, resourceVersion: "3" }, spec: { hostRef: "primary", workspaceRef: "workspace-one", title: "Session", runtimeProfile: "default", guiEnabled: true, ci: { repositoryId: correlation.repositoryId, ref: correlation.ref, commit: correlation.commit } }, status: { phase: "Running", serviceName: "session-one" } }], + resourceVersion: "3", + }); + value.setSessionAuthority(correlation.sessionId, authority); + return value; +} + +afterEach(() => vi.useRealTimers()); + +describe("CI projection runner", () => { + it("reconstructs exact provider state and refreshes stage transitions", async () => { + vi.useFakeTimers(); + const state = projection(); + let current = exactState; + const provider: CiProvider = { + name: "woodpecker", + query: async () => current, + run: async () => ({ triggered: false }), + }; + const runner = new CiProjectionRunner({ projection: state, provider, pollMs: 10 }); + runner.start(); + await vi.advanceTimersByTimeAsync(0); + expect(state.sessionRef(correlation.sessionId)?.liveState?.ci).toEqual(exactState); + current = { ...exactState, status: "success", currentStage: "publish" }; + await vi.advanceTimersByTimeAsync(10); + expect(state.sessionRef(correlation.sessionId)?.liveState?.ci).toMatchObject({ status: "success", currentStage: "publish" }); + await runner.stop(); + }); + + it("fails closed to unknown correlation when the provider is unavailable", async () => { + vi.useFakeTimers(); + const state = projection(); + state.setSessionCiState(correlation.sessionId, exactState); + const errors: unknown[] = []; + const provider: CiProvider = { + name: "woodpecker", + query: async () => { throw new Error("provider unavailable"); }, + run: async () => ({ triggered: false }), + }; + const runner = new CiProjectionRunner({ projection: state, provider, pollMs: 10, onError: error => errors.push(error) }); + runner.start(); + await vi.advanceTimersByTimeAsync(0); + expect(state.sessionRef(correlation.sessionId)?.liveState?.ci).toEqual({ + provider: "woodpecker", + correlation: "unknown", + repositoryId: correlation.repositoryId, + ref: correlation.ref, + commit: correlation.commit, + }); + expect(errors).toHaveLength(1); + await runner.stop(); + }); +}); diff --git a/packages/cluster-server/test/config.test.ts b/packages/cluster-server/test/config.test.ts new file mode 100644 index 00000000..11ab50c4 --- /dev/null +++ b/packages/cluster-server/test/config.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from "vite-plus/test"; +import { mkdtemp, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clusterServerConfigFromEnv, readClusterIdentityToken } from "../src/config.ts"; + +const BASE_ENV = { + POD_NAMESPACE: "cluster-system", + POD_NAME: "cluster-server-0", + POD_UID: "12345678-abcd", + KUBERNETES_SERVICE_HOST: "10.96.0.1", + KUBERNETES_SERVICE_PORT_HTTPS: "443", + T4_CLUSTER_HOST_NAME: "default", + T4_CLUSTER_IDENTITY_TOKEN_FILE: "/var/run/secrets/t4-cluster-identity/token", + T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "release-t4-cluster-server", +} as const; + +describe("cluster server configuration", () => { + it("selects the projected server identity independently from its Kubernetes watch credentials", () => { + const config = clusterServerConfigFromEnv(BASE_ENV); + expect(config).toMatchObject({ + identityTokenPath: "/var/run/secrets/t4-cluster-identity/token", + serverServiceAccountName: "release-t4-cluster-server", + kubernetesTokenPath: "/var/run/secrets/kubernetes.io/serviceaccount/token", + kubernetesCaPath: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + kubernetesApiAudience: "https://kubernetes.default.svc", + }); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_IDENTITY_TOKEN_FILE: "relative/token" })).toThrow("absolute"); + expect(clusterServerConfigFromEnv({ ...BASE_ENV, T4_KUBERNETES_API_AUDIENCE: "kubernetes.custom.example" }).kubernetesApiAudience).toBe("kubernetes.custom.example"); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_KUBERNETES_API_AUDIENCE: "/invalid" })).toThrow("T4_KUBERNETES_API_AUDIENCE"); + }); + + it("accepts Kubernetes DNS subdomains as cluster host names", () => { + const dottedHostName = "primary.us-west-2.example"; + expect(clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_HOST_NAME: dottedHostName }).hostName).toBe(dottedHostName); + + const maximumLengthHostName = [ + "a".repeat(63), + "b".repeat(63), + "c".repeat(63), + "d".repeat(61), + ].join("."); + expect(maximumLengthHostName).toHaveLength(253); + expect(clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_HOST_NAME: maximumLengthHostName }).hostName).toBe(maximumLengthHostName); + }); + + it("rejects invalid Kubernetes DNS subdomains as cluster host names", () => { + const overlongHostName = [ + "a".repeat(63), + "b".repeat(63), + "c".repeat(63), + "d".repeat(62), + ].join("."); + expect(overlongHostName).toHaveLength(254); + + for (const hostName of [ + "Cluster.example", + "", + overlongHostName, + "-cluster.example", + "cluster-.example", + "cluster..example", + `${"a".repeat(64)}.example`, + ]) { + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_HOST_NAME: hostName })).toThrow("T4_CLUSTER_HOST_NAME"); + } + }); + + it("keeps Kubernetes identity fields restricted to DNS labels", () => { + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, POD_NAMESPACE: "cluster.system" })).toThrow("POD_NAMESPACE"); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, POD_NAME: "cluster.server" })).toThrow("POD_NAME"); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "cluster.server" })).toThrow("T4_CLUSTER_SERVER_SERVICE_ACCOUNT"); + }); + + it("reads only a bounded regular projected identity file", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-cluster-identity-")); + try { + const path = join(directory, "token"); + const nextPath = `${path}.next`; + const token = `header.payload.${"s".repeat(64)}`; + await writeFile(nextPath, token, { mode: 0o400 }); + await rename(nextPath, path); + expect(await readClusterIdentityToken(path)).toBe(token); + await writeFile(nextPath, "x".repeat(16_385), { mode: 0o400 }); + await rename(nextPath, path); + await expect(readClusterIdentityToken(path)).rejects.toThrow("invalid"); + await expect(readClusterIdentityToken(directory)).rejects.toThrow("invalid"); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + it("accepts exactly one server-side Woodpecker credential source", () => { + const common = { + ...BASE_ENV, + T4_WOODPECKER_BASE_URL: "https://ci.example.test", + T4_WOODPECKER_REPOSITORIES: '{"t4-code":{"slug":"owner/t4-code"}}', + }; + expect(clusterServerConfigFromEnv({ ...common, T4_WOODPECKER_TOKEN_FILE: "/var/run/secrets/t4-ci/token" }).woodpecker).toMatchObject({ + tokenFile: "/var/run/secrets/t4-ci/token", + }); + expect(clusterServerConfigFromEnv({ ...common, T4_WOODPECKER_TOKEN: "secret-from-kubernetes" }).woodpecker).toMatchObject({ + token: "secret-from-kubernetes", + }); + expect(() => clusterServerConfigFromEnv(common)).toThrow("complete"); + expect(() => clusterServerConfigFromEnv({ + ...common, + T4_WOODPECKER_TOKEN: "secret-from-kubernetes", + T4_WOODPECKER_TOKEN_FILE: "/var/run/secrets/t4-ci/token", + })).toThrow("exactly one"); + }); + +}); + +describe("trusted cluster gateway proxy sources", () => { + it("accepts bounded canonical IPv4 and IPv6 networks", () => { + const config = clusterServerConfigFromEnv({ + ...BASE_ENV, + T4_CLUSTER_TRUSTED_PROXY_ADDRESSES: "10.42.1.7,fd7a:115c:a1e0::1", + T4_CLUSTER_TRUSTED_PROXY_CIDRS: "10.42.0.0/16,fd7a:115c:a1e0::/48", + }); + expect(config.trustedProxyAddresses).toEqual(["10.42.1.7", "fd7a:115c:a1e0::1"]); + expect(config.trustedProxyCidrs).toEqual(["10.42.0.0/16", "fd7a:115c:a1e0::/48"]); + }); + + it("rejects CIDRs with host bits or non-canonical notation", () => { + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: "10.42.1.7/16" })).toThrow(); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: "fd7a:115c:a1e0:0::/48" })).toThrow(); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: "0.0.0.0/0" })).toThrow(); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: "::/0" })).toThrow(); + }); + + it("bounds the trusted CIDR list", () => { + const cidrs = Array.from({ length: 65 }, (_, index) => `10.${index}.0.0/16`).join(","); + expect(() => clusterServerConfigFromEnv({ ...BASE_ENV, T4_CLUSTER_TRUSTED_PROXY_CIDRS: cidrs })).toThrow(); + }); +}); diff --git a/packages/cluster-server/test/gateway.test.ts b/packages/cluster-server/test/gateway.test.ts new file mode 100644 index 00000000..d507a3ba --- /dev/null +++ b/packages/cluster-server/test/gateway.test.ts @@ -0,0 +1,387 @@ +import { describe, expect, it } from "vite-plus/test"; +import { + agentId, + entryId, + hostId, + projectId, + revision, + sessionId, + type ClientFrame, + type ServerFrame, + type SessionRef, +} from "@t4-code/host-wire"; +import { ClusterGateway, type GatewayClient, type GatewayMutationBackend } from "../src/gateway.ts"; +import { ClusterInfrastructureProjection } from "../src/kubernetes-projection.ts"; +import { KubernetesApiError } from "../src/kubernetes-client.ts"; +import type { PodHostConnection, PodHostConnector, PodHostRoute } from "../src/pod-host-router.ts"; + +const PRINCIPAL = "owner@example.com"; +function authority(upstreamSessionId: string): SessionRef { + return { + hostId: hostId("session-pod"), + sessionId: sessionId(upstreamSessionId), + project: { projectId: projectId("t4-code"), name: "T4 code" }, + revision: revision("authority-r1"), + title: "Authoritative OMP session", + status: "idle", + updatedAt: "2026-07-20T00:00:00.000Z", + }; +} + +const host = { + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4ClusterHost", + metadata: { name: "primary", uid: "host-uid", resourceVersion: "10", generation: 1 }, + status: { observedGeneration: 1, conditions: [] }, +}; +const workspace = { + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { name: "workspace-one", uid: "workspace-uid", resourceVersion: "11", generation: 1 }, + spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "Workspace one", retentionPolicy: "Retain", size: "20Gi" }, + status: { observedGeneration: 1, phase: "Ready", conditions: [] }, +}; +const session = (name: string) => ({ + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Session", + metadata: { name, uid: `${name}-uid`, resourceVersion: name === "session-one" ? "12" : "13", generation: 1 }, + spec: { hostRef: "primary", workspaceRef: "workspace-one", title: name, runtimeProfile: "omp-17.0.5", guiEnabled: true }, + status: { + observedGeneration: 1, + phase: "Running", + serviceName: name, + conditions: [], + }, +}); + +class MemoryClient implements GatewayClient { + readonly frames: ServerFrame[] = []; + readonly closes: Array<[number | undefined, string | undefined]> = []; + send(frame: ServerFrame): void { this.frames.push(frame); } + close(code?: number, reason?: string): void { this.closes.push([code, reason]); } +} + +class MemoryConnector implements PodHostConnector { + readonly routes: PodHostRoute[] = []; + readonly sent: ClientFrame[] = []; + onFrame?: (frame: ServerFrame) => void; + async connect(route: PodHostRoute, onFrame: (frame: ServerFrame) => void): Promise { + this.routes.push(route); + this.onFrame = onFrame; + return { send: frame => { this.sent.push(frame); }, close: () => undefined }; + } +} + +class PendingConnector extends MemoryConnector { + readonly closes: Array<[number | undefined, string | undefined]> = []; + #resolve: (() => void) | undefined; + override connect(route: PodHostRoute, onFrame: (frame: ServerFrame) => void): Promise { + this.routes.push(route); + this.onFrame = onFrame; + const deferred = Promise.withResolvers(); + this.#resolve = () => { + this.#resolve = undefined; + deferred.resolve({ + send: frame => { this.sent.push(frame); }, + close: (code, reason) => { this.closes.push([code, reason]); }, + }); + }; + return deferred.promise; + } + resolveConnection(): void { + if (!this.#resolve) throw new Error("pod connector is not pending"); + this.#resolve(); + } +} + +class MemoryMutations implements GatewayMutationBackend { + workspaceCreates = 0; + sessionCreates = 0; + sessionDeletes = 0; + async createWorkspace() { + this.workspaceCreates++; + return { id: "workspace-created", revision: "workspace-r1" }; + } + async createSession() { + this.sessionCreates++; + return { sessionId: "session-created", revision: "session-r1" }; + } + async deleteSession() { + this.sessionDeletes++; + return { deleted: true as const }; + } +} + +function setup(epoch = "replica-uid-1", connector: MemoryConnector = new MemoryConnector()) { + const projection = new ClusterInfrastructureProjection({ epoch, namespace: "development" }); + projection.replace({ + host, + workspaces: [workspace], + sessions: [session("session-one"), session("session-two")], + resourceVersion: "13", + }); + projection.setSessionAuthority("session-one", authority("omp-private-one")); + projection.setSessionAuthority("session-two", authority("omp-private-two")); + const mutations = new MemoryMutations(); + const gateway = new ClusterGateway({ projection, connector, mutations }); + const client = new MemoryClient(); + const connection = gateway.connect(client, PRINCIPAL); + return { projection, connector, mutations, gateway, client, connection }; +} + +const hello = { + v: "omp-app/1" as const, + type: "hello" as const, + protocol: { min: "omp-app/1", max: "omp-app/1" }, + client: { name: "test", version: "1", build: "test", platform: "linux" }, + requestedFeatures: ["resume", "preview.control", "cluster.operator"], + savedCursors: [], + capabilities: { client: ["sessions.read", "sessions.manage", "preview.read", "preview.control", "ci.trigger"] }, +}; + +describe("stateless omp-app cluster gateway", () => { + it("negotiates cluster.operator, bootstraps one canonical inventory, and changes epoch on replica restart", async () => { + const first = setup("replica-uid-1"); + await first.connection.receive(hello); + expect(first.client.frames.map(frame => frame.type)).toEqual(["welcome", "sessions"]); + expect(first.client.frames[0]).toMatchObject({ + type: "welcome", + hostId: "cluster:host-uid", + epoch: "replica-uid-1", + grantedFeatures: ["resume", "preview.control", "cluster.operator"], + }); + expect((first.client.frames[1] as { sessions: unknown[] }).sessions).toHaveLength(2); + + const second = setup("replica-uid-2"); + await second.connection.receive({ ...hello, savedCursors: [{ hostId: "cluster:host-uid", sessionId: "session-one", cursor: { epoch: "replica-uid-1", seq: 7 } }] }); + expect(second.client.frames[0]).toMatchObject({ type: "welcome", epoch: "replica-uid-2", resumed: false }); + expect((second.client.frames[1] as { sessions: unknown[] }).sessions).toHaveLength(2); + }); + + it("answers the host-scoped session.list bootstrap from the Kubernetes projection", async () => { + const value = setup(); + await value.connection.receive(hello); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "r-sessions", commandId: "c-sessions", hostId: "cluster:host-uid", + command: "session.list", args: {}, + }); + expect(value.client.frames.at(-1)).toMatchObject({ + type: "response", commandId: "c-sessions", ok: true, command: "session.list", + result: { cursor: { epoch: "replica-uid-1" }, totalCount: 2, truncated: false, sessions: [{ sessionId: "session-one" }, { sessionId: "session-two" }] }, + }); + }); + + it("returns workspace bootstrap with its independent cursor and streams each watch revision once", async () => { + const value = setup(); + await value.connection.receive(hello); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "r-list", commandId: "c-list", hostId: "cluster:host-uid", + command: "workspace.list", args: {}, + }); + expect(value.client.frames.at(-1)).toMatchObject({ + type: "response", commandId: "c-list", ok: true, command: "workspace.list", + result: { cursor: { epoch: "replica-uid-1", seq: 1 }, workspaces: [{ id: "workspace-one" }] }, + }); + value.projection.applyWatch({ + type: "MODIFIED", + object: { ...workspace, metadata: { ...workspace.metadata, resourceVersion: "20", generation: 2 }, status: { ...workspace.status, observedGeneration: 2, phase: "Failed" } }, + }); + value.projection.applyWatch({ type: "MODIFIED", object: { ...workspace, metadata: { ...workspace.metadata, resourceVersion: "20" } } }); + expect(value.client.frames.filter(frame => frame.type === "workspace.state")).toHaveLength(1); + }); + + it("routes to exactly one pod host, rewrites only address ids, and preserves attach output order", async () => { + const value = setup(); + await value.connection.receive(hello); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "request-attach", commandId: "command-attach", + hostId: "cluster:host-uid", sessionId: "session-one", command: "session.attach", args: {}, + }); + expect(value.connector.routes).toEqual([{ clusterSessionId: "session-one", upstreamSessionId: "omp-private-one", url: "ws://session-one.development.svc:8787/v1/ws" }]); + expect(value.connector.sent.at(-1)).toMatchObject({ + type: "command", requestId: "request-attach", commandId: "command-attach", + hostId: "upstream", sessionId: "omp-private-one", command: "session.attach", + }); + value.connector.onFrame?.({ + v: "omp-app/1", type: "snapshot", hostId: hostId("upstream"), sessionId: sessionId("omp-private-one"), + cursor: { epoch: "pod-epoch", seq: 1 }, revision: revision("session-r1"), entries: [], + }); + value.connector.onFrame?.({ + v: "omp-app/1", type: "entry", hostId: hostId("upstream"), sessionId: sessionId("omp-private-one"), + cursor: { epoch: "pod-epoch", seq: 2 }, revision: revision("session-r2"), + entry: { id: entryId("entry-one"), parentId: null, hostId: hostId("upstream"), sessionId: sessionId("omp-private-one"), kind: "message", timestamp: "2026-07-20T00:00:00.000Z", data: { text: "hello", correlationId: "omp-private-one" } }, + }); + value.connector.onFrame?.({ + v: "omp-app/1", type: "agent", hostId: hostId("upstream"), sessionId: sessionId("omp-private-one"), agentId: agentId("Main"), state: "running", detail: {}, + }); + const forwarded = value.client.frames.slice(-3); + expect(forwarded.map(frame => frame.type)).toEqual(["snapshot", "entry", "agent"]); + expect(forwarded[0]).toMatchObject({ hostId: "cluster:host-uid", sessionId: "session-one" }); + expect(forwarded[1]).toMatchObject({ + hostId: "cluster:host-uid", sessionId: "session-one", + entry: { hostId: "cluster:host-uid", sessionId: "session-one", data: { correlationId: "omp-private-one" } }, + }); + }); + + it("closes a pending route without dispatch when session ownership changes", async () => { + const scenarios: Array<{ frame: unknown; commandId?: string }> = [ + { + commandId: "command-revoked", + frame: { + v: "omp-app/1", type: "command", requestId: "request-revoked", commandId: "command-revoked", + hostId: "cluster:host-uid", sessionId: "session-one", command: "session.attach", args: {}, + }, + }, + { + commandId: "preview-revoked", + frame: { + v: "omp-app/1", type: "command", requestId: "request-preview-revoked", commandId: "preview-revoked", + hostId: "cluster:host-uid", sessionId: "session-one", command: "preview.state", args: {}, + }, + }, + { + frame: { + v: "omp-app/1", type: "terminal.input", hostId: "cluster:host-uid", sessionId: "session-one", + terminalId: "terminal-revoked", data: "blocked", + }, + }, + ]; + for (const scenario of scenarios) { + const connector = new PendingConnector(); + const value = setup("replica-uid-1", connector); + await value.connection.receive({ + ...hello, + requestedFeatures: [...hello.requestedFeatures, "terminal.io"], + capabilities: { client: [...hello.capabilities.client, "term.input"] }, + }); + const pending = value.connection.receive(scenario.frame); + expect(connector.routes).toHaveLength(1); + value.projection.applyWatch({ + type: "MODIFIED", + object: { + ...workspace, + metadata: { ...workspace.metadata, resourceVersion: "30", generation: 2 }, + spec: { ...workspace.spec, owner: "other@example.com" }, + }, + }); + connector.resolveConnection(); + await pending; + expect(connector.sent).toEqual([]); + expect(connector.closes).toContainEqual([1001, "session route changed"]); + if (scenario.commandId) { + expect(value.client.frames.find(frame => frame.type === "response" && frame.commandId === scenario.commandId)).toMatchObject({ + type: "response", commandId: scenario.commandId, ok: false, error: { code: "NOT_AUTHORIZED" }, + }); + } + } + }); + + it("rejects a pending connection when its authoritative upstream route changes", async () => { + const changes: Array<(projection: ClusterInfrastructureProjection) => void> = [ + projection => projection.setSessionAuthority("session-one", authority("omp-private-moved")), + projection => projection.applyWatch({ + type: "MODIFIED", + object: { + ...session("session-one"), + metadata: { ...session("session-one").metadata, resourceVersion: "31", generation: 2 }, + status: { ...session("session-one").status, serviceName: "session-one-moved" }, + }, + }), + ]; + for (const [index, change] of changes.entries()) { + const connector = new PendingConnector(); + const value = setup("replica-uid-1", connector); + await value.connection.receive(hello); + const commandId = `command-route-changed-${index}`; + const pending = value.connection.receive({ + v: "omp-app/1", type: "command", requestId: `request-route-changed-${index}`, commandId, + hostId: "cluster:host-uid", sessionId: "session-one", command: "session.attach", args: {}, + }); + expect(connector.routes).toHaveLength(1); + change(value.projection); + connector.resolveConnection(); + await pending; + expect(connector.sent).toEqual([]); + expect(connector.closes).toContainEqual([1001, "session route changed"]); + expect(value.client.frames.find(frame => frame.type === "response" && frame.commandId === commandId)).toMatchObject({ + type: "response", commandId, ok: false, error: { code: "UPSTREAM_UNAVAILABLE" }, + }); + } + }); + + it("denies a preview id learned from another session without opening a second upstream socket", async () => { + const value = setup(); + await value.connection.receive(hello); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "request-owner", commandId: "command-owner", + hostId: "cluster:host-uid", sessionId: "session-one", command: "preview.state", args: {}, + }); + value.connector.onFrame?.({ + v: "omp-app/1", type: "preview.state", hostId: "upstream" as never, sessionId: "omp-private-one" as never, + previewId: "preview-one" as never, state: "ready", url: "https://example.test", revision: "preview-r1" as never, + cursor: { epoch: "preview-e1", seq: 1 }, + }); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "request-preview", commandId: "command-preview", + hostId: "cluster:host-uid", sessionId: "session-two", command: "preview.activate", args: { previewId: "preview-one" }, + }); + expect(value.connector.routes).toHaveLength(1); + expect(value.client.frames.at(-1)).toMatchObject({ type: "response", commandId: "command-preview", ok: false, error: { code: "NOT_AUTHORIZED" } }); + }); + + it("uses bounded command idempotency while CR mutation identity survives reconnect", async () => { + const value = setup(); + await value.connection.receive(hello); + const command = { + v: "omp-app/1" as const, type: "command" as const, requestId: "request-create", commandId: "command-create", + hostId: "cluster:host-uid", command: "workspace.create", + args: { displayName: "Created", retentionPolicy: "Retain", capacity: "20Gi" }, + }; + await value.connection.receive(command); + await value.connection.receive({ ...command, requestId: "request-create-retry" }); + expect(value.mutations.workspaceCreates).toBe(1); + expect(value.client.frames.slice(-2)).toEqual([ + expect.objectContaining({ type: "response", commandId: "command-create", ok: true }), + expect.objectContaining({ type: "response", commandId: "command-create", requestId: "request-create-retry", ok: true }), + ]); + }); + + it("fails closed when the session infrastructure does not authorize GUI access", async () => { + const value = setup(); + value.projection.applyWatch({ + type: "MODIFIED", + object: { ...session("session-one"), metadata: { ...session("session-one").metadata, resourceVersion: "30" }, spec: { ...session("session-one").spec, guiEnabled: false } }, + }); + await value.connection.receive(hello); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "request-disabled-gui", commandId: "command-disabled-gui", + hostId: "cluster:host-uid", sessionId: "session-one", command: "preview.state", args: {}, + }); + expect(value.connector.routes).toHaveLength(0); + expect(value.client.frames.at(-1)).toMatchObject({ type: "response", ok: false, error: { code: "UNSUPPORTED_FEATURE", message: "GUI is disabled for this session" } }); + }); + + it("treats an already absent session delete as an idempotent success", async () => { + const value = setup(); + value.projection.applyWatch({ type: "DELETED", object: session("session-one") }); + await value.connection.receive(hello); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "request-delete-replay", commandId: "command-delete-replay", + hostId: "cluster:host-uid", sessionId: "session-one", command: "session.delete", expectedRevision: "authority-r1", args: {}, + }); + expect(value.client.frames.at(-1)).toMatchObject({ type: "response", ok: true, result: { deleted: true } }); + expect(value.mutations.sessionDeletes).toBe(0); + }); + + it("reports Kubernetes schema rejection as a client contract error", async () => { + const value = setup(); + value.mutations.createWorkspace = async () => { throw new KubernetesApiError(422, "invalid"); }; + await value.connection.receive(hello); + await value.connection.receive({ + v: "omp-app/1", type: "command", requestId: "request-invalid", commandId: "command-invalid", + hostId: "cluster:host-uid", command: "workspace.create", + args: { displayName: "Created", retentionPolicy: "Retain", capacity: "20Gi" }, + }); + expect(value.client.frames.at(-1)).toMatchObject({ type: "response", ok: false, error: { code: "INVALID_FRAME" } }); + }); +}); diff --git a/packages/cluster-server/test/kubernetes-client.test.ts b/packages/cluster-server/test/kubernetes-client.test.ts new file mode 100644 index 00000000..7adc0c65 --- /dev/null +++ b/packages/cluster-server/test/kubernetes-client.test.ts @@ -0,0 +1,304 @@ +import { describe, expect, it } from "vite-plus/test"; +import { mkdtemp, rename, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + CLUSTER_INTERNAL_AUDIENCE, + KubernetesApiClient, + KubernetesGatewayMutationBackend, + KubernetesTokenReviewer, + semanticResourceHash, +} from "../src/kubernetes-client.ts"; + +const PRINCIPAL = "owner@example.com"; + +function recordingFetch(responses: unknown[]) { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + return Response.json(responses.shift() ?? {}, { status: init?.method === "POST" ? 201 : 200 }); + }) as typeof globalThis.fetch; + return { requests, fetch }; +} + +function conflictFetch(existing: unknown) { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + return requests.length === 1 + ? Response.json({ reason: "AlreadyExists" }, { status: 409 }) + : Response.json(existing); + }) as typeof globalThis.fetch; + return { requests, fetch }; +} + +describe("namespaced Kubernetes client", () => { + it("lists and watches only the three cluster.t4.dev resources with bounded resource versions", async () => { + const values = recordingFetch([ + { + metadata: { resourceVersion: "20" }, + items: [{ apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4ClusterHost", metadata: { name: "primary", uid: "host-uid", resourceVersion: "20" }, spec: {} }], + }, + { metadata: { resourceVersion: "21" }, items: [] }, + { metadata: { resourceVersion: "22" }, items: [] }, + ]); + const client = new KubernetesApiClient({ + baseUrl: "https://kubernetes.default.svc", + namespace: "development", + token: "service-account-token", + fetch: values.fetch, + }); + const listed = await client.listInfrastructure(); + expect(listed.resourceVersion).toBe("22"); + expect(values.requests.map(request => request.url)).toEqual([ + "https://kubernetes.default.svc/apis/cluster.t4.dev/v1alpha1/namespaces/development/t4clusterhosts?limit=256", + "https://kubernetes.default.svc/apis/cluster.t4.dev/v1alpha1/namespaces/development/t4workspaces?limit=256", + "https://kubernetes.default.svc/apis/cluster.t4.dev/v1alpha1/namespaces/development/t4sessions?limit=1000", + ]); + for (const request of values.requests) { + expect(new Headers(request.init?.headers).get("authorization")).toBe("Bearer service-account-token"); + } + expect(JSON.stringify(listed)).not.toContain("service-account-token"); + }); + + it("observes projected service account token rotation without recreating the client", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-kubernetes-client-token-")); + try { + const tokenFile = join(directory, "token"); + const nextTokenFile = join(directory, "token.next"); + const values = recordingFetch([{}, {}]); + await writeFile(join(directory, "token-one"), "projected-token-one\n", { mode: 0o400 }); + await writeFile(join(directory, "token-two"), "projected-token-two\n", { mode: 0o400 }); + await symlink(join(directory, "token-one"), tokenFile); + const client = new KubernetesApiClient({ + baseUrl: "https://kubernetes.default.svc", + namespace: "development", + tokenFile, + fetch: values.fetch, + }); + + await client.list("t4clusterhosts", 1); + await symlink(join(directory, "token-two"), nextTokenFile); + await rename(nextTokenFile, tokenFile); + await client.list("t4clusterhosts", 1); + + expect(values.requests.map(request => new Headers(request.init?.headers).get("authorization"))).toEqual([ + "Bearer projected-token-one", + "Bearer projected-token-two", + ]); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("requires exactly one bounded valid credential source and fails closed", async () => { + const common = { baseUrl: "https://kubernetes.default.svc", namespace: "development" } as const; + expect(() => new KubernetesApiClient(common)).toThrow("exactly one credential source"); + expect(() => new KubernetesApiClient({ ...common, token: "static-token", tokenFile: "/projected/token" })).toThrow("exactly one credential source"); + expect(() => new KubernetesApiClient({ ...common, tokenFile: "relative/token" })).toThrow("must be absolute"); + expect(() => new KubernetesApiClient({ ...common, token: "malformed token" })).toThrow("token is invalid"); + + const directory = await mkdtemp(join(tmpdir(), "t4-kubernetes-client-invalid-token-")); + try { + const tokenFile = join(directory, "token"); + const nextTokenFile = join(directory, "token.next"); + const values = recordingFetch([]); + const client = new KubernetesApiClient({ ...common, tokenFile, fetch: values.fetch }); + await writeFile(nextTokenFile, "malformed token", { mode: 0o400 }); + await rename(nextTokenFile, tokenFile); + await expect(client.request("/version")).rejects.toThrow("Kubernetes token file is invalid"); + await writeFile(nextTokenFile, "x".repeat(16_385), { mode: 0o400 }); + await rename(nextTokenFile, tokenFile); + await expect(client.request("/version")).rejects.toThrow("Kubernetes token file is invalid"); + expect(values.requests).toHaveLength(0); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("persists idempotent CR identity as command id plus semantic hash without credentials or arbitrary URLs", async () => { + const values = recordingFetch([ + {}, + { kind: "T4Workspace", metadata: { name: "workspace-one" }, spec: { hostRef: "primary", owner: PRINCIPAL } }, + ]); + const client = new KubernetesApiClient({ + baseUrl: "https://kubernetes.default.svc", + namespace: "development", + token: "service-account-token", + fetch: values.fetch, + }); + const backend = new KubernetesGatewayMutationBackend({ client, hostRef: "primary" }); + const workspaceArgs = { + displayName: "Created workspace", + retentionPolicy: "Retain" as const, + capacity: "20Gi", + repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, + }; + await backend.createWorkspace("command-create-workspace", workspaceArgs, PRINCIPAL); + const workspaceBody = JSON.parse(String(values.requests[0]?.init?.body)); + expect(values.requests[0]).toMatchObject({ + url: "https://kubernetes.default.svc/apis/cluster.t4.dev/v1alpha1/namespaces/development/t4workspaces", + init: { method: "POST" }, + }); + expect(workspaceBody).toMatchObject({ + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { + name: expect.stringMatching(/^workspace-[a-f0-9]{16}$/), + annotations: { + "cluster.t4.dev/command-id": "command-create-workspace", + "cluster.t4.dev/principal-hash": semanticResourceHash(PRINCIPAL), + "cluster.t4.dev/semantic-hash": semanticResourceHash({ args: workspaceArgs, principal: PRINCIPAL }), + }, + }, + spec: { + hostRef: "primary", + owner: PRINCIPAL, + displayName: "Created workspace", + retentionPolicy: "Retain", + size: "20Gi", + repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, + }, + }); + expect(JSON.stringify(workspaceBody)).not.toContain("token"); + expect(JSON.stringify(workspaceBody)).not.toContain("url"); + + await backend.createSession("command-create-session", { + workspaceId: "workspace-one", + title: "Task", + runtimeProfile: "omp-17.0.5", + guiEnabled: true, + ci: { provider: "woodpecker", repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, + }, PRINCIPAL); + const sessionBody = JSON.parse(String(values.requests[2]?.init?.body)); + expect(sessionBody).toMatchObject({ + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Session", + metadata: { name: expect.stringMatching(/^session-[a-f0-9]{16}$/) }, + spec: { hostRef: "primary", workspaceRef: "workspace-one", title: "Task", runtimeProfile: "omp-17.0.5", guiEnabled: true }, + }); + }); + + it("reuses exact principal-scoped annotations and rejects semantic conflicts", async () => { + const args = { displayName: "Created", retentionPolicy: "Delete" as const, capacity: "10Gi" }; + const annotations = { + "cluster.t4.dev/command-id": "command-one", + "cluster.t4.dev/principal-hash": semanticResourceHash(PRINCIPAL), + "cluster.t4.dev/semantic-hash": semanticResourceHash({ args, principal: PRINCIPAL }), + }; + const existing = { + metadata: { name: "workspace-existing", resourceVersion: "9", annotations }, + status: { revision: "workspace-r1" }, + }; + const exact = conflictFetch(existing); + const backend = new KubernetesGatewayMutationBackend({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: exact.fetch }), + hostRef: "primary", + }); + expect(await backend.createWorkspace("command-one", args, PRINCIPAL)).toEqual({ id: "workspace-existing", revision: "9" }); + expect(exact.requests.map(request => request.init?.method ?? "GET")).toEqual(["POST", "GET"]); + + const conflicting = conflictFetch({ ...existing, metadata: { ...existing.metadata, annotations: { ...annotations, "cluster.t4.dev/semantic-hash": "wrong" } } }); + const conflictingBackend = new KubernetesGatewayMutationBackend({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: conflicting.fetch }), + hostRef: "primary", + }); + await expect(conflictingBackend.createWorkspace("command-one", args, PRINCIPAL)).rejects.toThrow("idempotency conflict"); + }); + + it("treats an already absent session as a successful idempotent delete", async () => { + const fetch = (async () => Response.json({ reason: "NotFound" }, { status: 404 })) as unknown as typeof globalThis.fetch; + const backend = new KubernetesGatewayMutationBackend({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch }), + hostRef: "primary", + }); + expect(await backend.deleteSession("command-delete", "session-gone", PRINCIPAL)).toEqual({ deleted: true }); + }); +}); + +describe("Kubernetes projected identity review", () => { + it("submits the presented bearer with the fixed audience and requires the exact server ServiceAccount", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-token-review-")); + try { + await writeFile(join(directory, "token"), "reviewer-api-token", { mode: 0o400 }); + await writeFile(join(directory, "ca.crt"), "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n", { mode: 0o400 }); + await writeFile(join(directory, "namespace"), "team\n", { mode: 0o400 }); + const presentedToken = `header.payload.${"s".repeat(64)}`; + const requests: Array<{ url: string; init?: RequestInit }> = []; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + return Response.json({ + apiVersion: "authentication.k8s.io/v1", + kind: "TokenReview", + status: { + authenticated: true, + audiences: [CLUSTER_INTERNAL_AUDIENCE], + user: { username: "system:serviceaccount:team:release-t4-cluster-server" }, + }, + }); + }) as typeof globalThis.fetch; + const reviewer = new KubernetesTokenReviewer({ + baseUrl: "https://kubernetes.default.svc", + tokenPath: join(directory, "token"), + caPath: join(directory, "ca.crt"), + namespacePath: join(directory, "namespace"), + serverServiceAccountName: "release-t4-cluster-server", + fetch, + }); + expect(await reviewer.review(presentedToken)).toBe(true); + expect(requests[0]?.url).toBe("https://kubernetes.default.svc/apis/authentication.k8s.io/v1/tokenreviews"); + expect(new Headers(requests[0]?.init?.headers).get("authorization")).toBe("Bearer reviewer-api-token"); + expect(JSON.parse(String(requests[0]?.init?.body))).toEqual({ + apiVersion: "authentication.k8s.io/v1", + kind: "TokenReview", + spec: { token: presentedToken, audiences: ["t4-cluster-internal"] }, + }); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("denies malformed, rejected, wrong-audience, wrong-username, API-status, and network responses", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-token-review-denied-")); + try { + await writeFile(join(directory, "token"), "reviewer-api-token", { mode: 0o400 }); + await writeFile(join(directory, "ca.crt"), "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n", { mode: 0o400 }); + await writeFile(join(directory, "namespace"), "team", { mode: 0o400 }); + const presentedToken = `header.payload.${"s".repeat(64)}`; + const statuses: unknown[] = [ + { authenticated: false }, + { authenticated: true, audiences: ["other"], user: { username: "system:serviceaccount:team:release-t4-cluster-server" } }, + { authenticated: true, audiences: [CLUSTER_INTERNAL_AUDIENCE], user: { username: "system:serviceaccount:other:release-t4-cluster-server" } }, + { authenticated: true, audiences: [CLUSTER_INTERNAL_AUDIENCE] }, + { authenticated: true, error: "review failed", audiences: [CLUSTER_INTERNAL_AUDIENCE], user: { username: "system:serviceaccount:team:release-t4-cluster-server" } }, + ]; + for (const status of statuses) { + const reviewer = new KubernetesTokenReviewer({ + baseUrl: "https://kubernetes.default.svc", + tokenPath: join(directory, "token"), + caPath: join(directory, "ca.crt"), + namespacePath: join(directory, "namespace"), + serverServiceAccountName: "release-t4-cluster-server", + fetch: (async () => Response.json({ apiVersion: "authentication.k8s.io/v1", kind: "TokenReview", status })) as unknown as typeof globalThis.fetch, + }); + expect(await reviewer.review(presentedToken)).toBe(false); + } + const malformed = new KubernetesTokenReviewer({ + baseUrl: "https://kubernetes.default.svc", + tokenPath: join(directory, "token"), caPath: join(directory, "ca.crt"), namespacePath: join(directory, "namespace"), + serverServiceAccountName: "release-t4-cluster-server", + fetch: (async () => new Response("{", { status: 200, headers: { "content-type": "application/json" } })) as unknown as typeof globalThis.fetch, + }); + expect(await malformed.review(presentedToken)).toBe(false); + const unavailable = new KubernetesTokenReviewer({ + baseUrl: "https://kubernetes.default.svc", + tokenPath: join(directory, "token"), caPath: join(directory, "ca.crt"), namespacePath: join(directory, "namespace"), + serverServiceAccountName: "release-t4-cluster-server", + fetch: (async () => { throw new Error("network unavailable"); }) as unknown as typeof globalThis.fetch, + }); + expect(await unavailable.review(presentedToken)).toBe(false); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cluster-server/test/kubernetes-projection.test.ts b/packages/cluster-server/test/kubernetes-projection.test.ts new file mode 100644 index 00000000..1f9cc48a --- /dev/null +++ b/packages/cluster-server/test/kubernetes-projection.test.ts @@ -0,0 +1,215 @@ +import { describe, expect, it } from "vite-plus/test"; +import { hostId, projectId, revision, sessionId, type SessionRef } from "@t4-code/host-wire"; +import { + CLUSTER_MAX_SESSIONS, + CLUSTER_MAX_WORKSPACES, + ClusterInfrastructureProjection, + KubernetesAuthorityInvalidatedError, + clusterHostIdFromUid, +} from "../src/kubernetes-projection.ts"; + +const PRINCIPAL = "owner@example.com"; +function authority(upstreamSessionId: string): SessionRef { + return { + hostId: hostId("session-pod"), + sessionId: sessionId(upstreamSessionId), + project: { projectId: projectId("t4-code"), name: "T4 code" }, + revision: revision("authority-r1"), + title: "Authoritative OMP session", + status: "idle", + updatedAt: "2026-07-20T00:00:00.000Z", + }; +} + +const host = { + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4ClusterHost", + metadata: { name: "primary", uid: "24e7bcb1-c694-4ba4-85c4-70a829f7996b", resourceVersion: "100", generation: 2 }, + spec: {}, + status: { observedGeneration: 2, conditions: [{ type: "Available", status: "True", reason: "Ready", message: "ready", observedGeneration: 2 }] }, +}; +const workspace = { + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { name: "workspace-one", uid: "workspace-uid", resourceVersion: "101", generation: 3 }, + spec: { + hostRef: "primary", + owner: PRINCIPAL, + displayName: "T4 code", + retentionPolicy: "Retain", + size: "20Gi", + repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, + }, + status: { + observedGeneration: 3, + phase: "Ready", + capacity: "20Gi", + conditions: [{ type: "StorageReady", status: "True", reason: "Bound", message: "PVC is bound", observedGeneration: 3 }], + pvcRef: "workspace-one", + }, +}; +const session = { + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Session", + metadata: { name: "session-one", uid: "session-uid", resourceVersion: "102", generation: 5 }, + spec: { + hostRef: "primary", + workspaceRef: "workspace-one", + title: "Cluster task", + runtimeProfile: "omp-17.0.5", + guiEnabled: true, + ci: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, + }, + status: { + observedGeneration: 5, + phase: "Running", + podName: "session-one-pod", + serviceName: "session-one", + conditions: [{ type: "Available", status: "True", reason: "PodReady", message: "ready", observedGeneration: 5 }], + }, +}; + +describe("Kubernetes infrastructure projection", () => { + it("derives a stable host id from the T4ClusterHost UID and bounded list state", () => { + expect(clusterHostIdFromUid(host.metadata.uid)).toBe("cluster:24e7bcb1-c694-4ba4-85c4-70a829f7996b"); + expect(CLUSTER_MAX_WORKSPACES).toBe(256); + expect(CLUSTER_MAX_SESSIONS).toBe(1_000); + const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development" }); + projection.replace({ host, workspaces: [workspace], sessions: [session], resourceVersion: "102" }); + expect(projection.hostId).toBe(clusterHostIdFromUid(host.metadata.uid)); + expect(projection.workspaceList()).toEqual({ + cursor: { epoch: "replica-uid-1", seq: 1 }, + workspaces: [ + expect.objectContaining({ + id: "workspace-one", + phase: "Ready", + accessMode: "ReadWriteMany", + retentionPolicy: "Retain", + }), + ], + }); + expect(JSON.stringify(projection.workspaceList())).not.toContain("credentialPath"); + expect(JSON.stringify(projection.workspaceList())).not.toContain("pvcRef"); + expect(JSON.stringify(projection.workspaceList())).not.toContain("repositoryId"); + }); + + it("keeps workspace cursors separate and reconnect replacement idempotent", () => { + const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development" }); + projection.replace({ host, workspaces: [workspace], sessions: [session], resourceVersion: "102" }); + const seen: unknown[] = []; + const stop = projection.subscribe(frame => seen.push(frame), projection.workspaceCursor); + projection.applyWatch({ type: "MODIFIED", object: { ...session, metadata: { ...session.metadata, resourceVersion: "103" } } }); + expect(seen).toHaveLength(0); + projection.applyWatch({ + type: "MODIFIED", + object: { + ...workspace, + metadata: { ...workspace.metadata, resourceVersion: "104", generation: 4 }, + status: { ...workspace.status, observedGeneration: 4, phase: "Failed" }, + }, + }); + expect(seen).toHaveLength(1); + expect(seen[0]).toMatchObject({ type: "workspace.state", workspaceId: "workspace-one", cursor: { seq: 2 } }); + projection.applyWatch({ type: "MODIFIED", object: { ...workspace, metadata: { ...workspace.metadata, resourceVersion: "104" } } }); + expect(seen).toHaveLength(1); + stop(); + }); + + it("projects routable pod authority and removes deleted sessions without local truth", () => { + const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development" }); + projection.replace({ host, workspaces: [workspace], sessions: [session], resourceVersion: "102" }); + projection.setSessionAuthority("session-one", authority("omp-session-private")); + expect(projection.sessionRoute("session-one")).toEqual({ + clusterSessionId: "session-one", + upstreamSessionId: "omp-session-private", + url: "ws://session-one.development.svc:8787/v1/ws", + }); + expect(projection.sessionRefs()).toEqual([ + expect.objectContaining({ + hostId: clusterHostIdFromUid(host.metadata.uid), + sessionId: "session-one", + liveState: expect.objectContaining({ + cluster: expect.objectContaining({ workspaceId: "workspace-one", phase: "Running" }), + }), + }), + ]); + projection.applyWatch({ type: "DELETED", object: session }); + expect(projection.sessionRoute("session-one")).toBeUndefined(); + expect(projection.sessionRefs()).toEqual([]); + }); + + it("accepts only HTTPS browser origins and exposes GUI infrastructure authorization", () => { + const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development" }); + projection.replace({ + host: { ...host, spec: { allowedOrigins: ["https://t4.tailnet.example", "http://insecure.example", "https://user:secret@t4.example"] } }, + workspaces: [workspace], + sessions: [session], + resourceVersion: "102", + }); + expect(projection.allowedOrigins()).toEqual(["https://t4.tailnet.example"]); + expect(projection.sessionGuiState("session-one", PRINCIPAL)).toBe("Ready"); + projection.applyWatch({ + type: "MODIFIED", + object: { ...session, metadata: { ...session.metadata, resourceVersion: "105" }, spec: { ...session.spec, guiEnabled: false } }, + }); + expect(projection.sessionGuiState("session-one", PRINCIPAL)).toBe("Unavailable"); + expect(projection.sessionGuiState("session-one", "other@example.com")).toBeUndefined(); + }); + + it("removes cached resources immediately if a legacy watch migrates them to another host", () => { + const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development" }); + projection.replace({ host, workspaces: [workspace], sessions: [session], resourceVersion: "102" }); + projection.setSessionAuthority("session-one", authority("omp-session-private")); + projection.applyWatch({ + type: "MODIFIED", + object: { ...session, metadata: { ...session.metadata, resourceVersion: "106" }, spec: { ...session.spec, hostRef: "another-host" } }, + }); + expect(projection.sessionRoute("session-one")).toBeUndefined(); + projection.applyWatch({ + type: "MODIFIED", + object: { ...workspace, metadata: { ...workspace.metadata, resourceVersion: "107" }, spec: { ...workspace.spec, hostRef: "another-host" } }, + }); + expect(projection.workspaceList(PRINCIPAL).workspaces).toEqual([]); + }); + + it.each([ + ["deletion", { type: "DELETED" as const, object: host }], + ["UID replacement", { + type: "MODIFIED" as const, + object: { ...host, metadata: { ...host.metadata, uid: "replacement-host-uid", resourceVersion: "108" } }, + }], + ])("invalidates all selected authority immediately on host %s", (_name, event) => { + const projection = new ClusterInfrastructureProjection({ epoch: "replica-uid-1", namespace: "development" }); + projection.replace({ host, workspaces: [workspace], sessions: [session], resourceVersion: "102" }); + projection.setSessionAuthority("session-one", authority("omp-session-private")); + const workspaceSequence = projection.workspaceCursor.seq; + expect(() => projection.applyWatch(event)).toThrow(KubernetesAuthorityInvalidatedError); + expect(() => projection.hostId).toThrow("not synchronized"); + expect(projection.workspaceList().workspaces).toEqual([]); + expect(projection.workspaceCursor.seq).toBeGreaterThan(workspaceSequence); + expect(projection.sessionRefs()).toEqual([]); + expect(projection.sessionRoute("session-one")).toBeUndefined(); + + const replacementHost = { ...host, metadata: { ...host.metadata, uid: "replacement-host-uid", resourceVersion: "200" } }; + projection.replace({ + host: replacementHost, + workspaces: [{ ...workspace, metadata: { ...workspace.metadata, resourceVersion: "201" } }], + sessions: [{ ...session, metadata: { ...session.metadata, resourceVersion: "202" } }], + resourceVersion: "202", + resourceVersions: { t4clusterhosts: "200", t4workspaces: "201", t4sessions: "202" }, + }); + expect(projection.hostId).toBe(clusterHostIdFromUid("replacement-host-uid")); + expect(projection.sessionRoute("session-one")).toBeUndefined(); + }); + + it("fails closed at explicit projection limits", () => { + const projection = new ClusterInfrastructureProjection({ + epoch: "replica-uid-1", + namespace: "development", + maxWorkspaces: 1, + maxSessions: 1, + }); + expect(() => projection.replace({ host, workspaces: [workspace, { ...workspace, metadata: { ...workspace.metadata, name: "two" } }], sessions: [], resourceVersion: "1" })).toThrow("workspace projection limit"); + expect(() => projection.replace({ host, workspaces: [], sessions: [session, { ...session, metadata: { ...session.metadata, name: "two" } }], resourceVersion: "1" })).toThrow("session projection limit"); + }); +}); diff --git a/packages/cluster-server/test/kubernetes-runner.test.ts b/packages/cluster-server/test/kubernetes-runner.test.ts new file mode 100644 index 00000000..04e0ff71 --- /dev/null +++ b/packages/cluster-server/test/kubernetes-runner.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it, vi } from "vite-plus/test"; +import { hostId, projectId, revision, sessionId, type SessionRef } from "@t4-code/host-wire"; +import { KubernetesApiError, type KubernetesApiClient } from "../src/kubernetes-client.ts"; +import { + ClusterInfrastructureProjection, + clusterHostIdFromUid, + type InfrastructureList, + type KubernetesWatchEvent, +} from "../src/kubernetes-projection.ts"; +import { KubernetesProjectionRunner } from "../src/kubernetes-runner.ts"; + +const AUTHORITY: SessionRef = { + hostId: hostId("pod-host"), + sessionId: sessionId("omp-session-old"), + project: { projectId: projectId("workspace-one") }, + revision: revision("authority-r1"), + title: "Cluster session", + status: "active", + updatedAt: "2026-07-20T00:00:00.000Z", +}; + +function snapshot(hostUid: string, versions: readonly [string, string, string]): InfrastructureList { + return { + host: { + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4ClusterHost", + metadata: { name: "primary", uid: hostUid, resourceVersion: versions[0] }, + spec: {}, + }, + workspaces: [{ + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { name: "workspace-one", uid: "workspace-uid", resourceVersion: versions[1] }, + spec: { hostRef: "primary", owner: "owner@example.com", displayName: "Workspace", retentionPolicy: "Retain", size: "20Gi" }, + status: { phase: "Ready" }, + }], + sessions: [{ + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Session", + metadata: { name: "session-one", uid: "session-uid", resourceVersion: versions[2] }, + spec: { hostRef: "primary", workspaceRef: "workspace-one", title: "Session", runtimeProfile: "default", guiEnabled: true }, + status: { phase: "Running", serviceName: "session-one" }, + }], + resourceVersion: versions[2], + resourceVersions: { t4clusterhosts: versions[0], t4workspaces: versions[1], t4sessions: versions[2] }, + }; +} + +interface WatchCall { + readonly resource: string; + readonly resourceVersion: string; + readonly signal: AbortSignal; + emit(event: KubernetesWatchEvent): void; + fail(error: unknown): void; +} + +function watchMock(calls: WatchCall[], freshGenerationStarted: PromiseWithResolvers): KubernetesApiClient["watch"] { + return (resource, resourceVersion, onEvent, signal, onStarted) => new Promise((_resolve, reject) => { + let settled = false; + const fail = (error: unknown): void => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", abort); + reject(error); + }; + const abort = (): void => fail(signal.reason); + signal.addEventListener("abort", abort, { once: true }); + calls.push({ + resource, + resourceVersion, + signal, + emit: event => { + if (settled) return; + try { onEvent(event); } + catch (error) { fail(error); } + }, + fail, + }); + onStarted?.(); + if (calls.length === 6) freshGenerationStarted.resolve(); + }); +} + +const invalidatingEvents = [ + ["deletion", (listed: InfrastructureList) => ({ type: "DELETED" as const, object: listed.host })], + ["UID replacement", (listed: InfrastructureList) => ({ + type: "MODIFIED" as const, + object: { ...listed.host, metadata: { ...listed.host.metadata, uid: "host-uid-new", resourceVersion: "103" } }, + })], +] as const; + +describe("Kubernetes projection runner", () => { + it.each(invalidatingEvents)("fails closed and relists the whole generation after selected host %s", async (_name, event) => { + const initial = snapshot("host-uid-old", ["100", "101", "102"]); + const fresh = snapshot("host-uid-new", ["200", "201", "202"]); + const absentListStarted = Promise.withResolvers(); + const freshListStarted = Promise.withResolvers(); + const freshList = Promise.withResolvers(); + let listAttempt = 0; + const listInfrastructure = vi.fn(() => { + listAttempt++; + if (listAttempt === 1) return Promise.resolve(initial); + if (listAttempt === 2) { + absentListStarted.resolve(); + return Promise.reject(new Error("T4ClusterHost is unavailable")); + } + if (listAttempt === 3) { + freshListStarted.resolve(); + return freshList.promise; + } + return Promise.reject(new Error("unexpected infrastructure list")); + }); + const calls: WatchCall[] = []; + const freshGenerationStarted = Promise.withResolvers(); + const watch = vi.fn(watchMock(calls, freshGenerationStarted)); + const client = { listInfrastructure, watch } as unknown as KubernetesApiClient; + const projection = new ClusterInfrastructureProjection({ epoch: "replica-one", namespace: "development" }); + const errors: unknown[] = []; + const runner = new KubernetesProjectionRunner({ client, projection, hostName: "primary", retryMs: 0, onError: error => errors.push(error) }); + + await runner.start(); + projection.setSessionAuthority("session-one", AUTHORITY); + expect(calls.map(call => [call.resource, call.resourceVersion])).toEqual([ + ["t4clusterhosts", "100"], + ["t4workspaces", "101"], + ["t4sessions", "102"], + ]); + expect(projection.sessionRoute("session-one")?.upstreamSessionId).toBe("omp-session-old"); + + calls[0]!.emit(event(initial)); + expect(projection.sessionRoute("session-one")).toBeUndefined(); + expect(projection.workspaceList().workspaces).toEqual([]); + await absentListStarted.promise; + expect(calls).toHaveLength(3); + expect(calls.every(call => call.signal.aborted)).toBe(true); + await freshListStarted.promise; + expect(projection.sessionRoute("session-one")).toBeUndefined(); + expect(calls).toHaveLength(3); + + freshList.resolve(fresh); + await freshGenerationStarted.promise; + expect(calls.slice(3).map(call => [call.resource, call.resourceVersion])).toEqual([ + ["t4clusterhosts", "200"], + ["t4workspaces", "201"], + ["t4sessions", "202"], + ]); + expect(projection.hostId).toBe(clusterHostIdFromUid("host-uid-new")); + expect(projection.sessionRoute("session-one")).toBeUndefined(); + projection.setSessionAuthority("session-one", { ...AUTHORITY, sessionId: sessionId("omp-session-fresh") }); + expect(projection.sessionRoute("session-one")?.upstreamSessionId).toBe("omp-session-fresh"); + expect(errors).toHaveLength(2); + await runner.stop(); + }); + + it("keeps 410 handling generation-wide and resumes every watch from relisted versions", async () => { + const initial = snapshot("host-uid", ["100", "101", "102"]); + const fresh = snapshot("host-uid", ["200", "201", "202"]); + const relistStarted = Promise.withResolvers(); + const relist = Promise.withResolvers(); + let listAttempt = 0; + const listInfrastructure = vi.fn(() => { + listAttempt++; + if (listAttempt === 1) return Promise.resolve(initial); + if (listAttempt === 2) { relistStarted.resolve(); return relist.promise; } + return Promise.reject(new Error("unexpected infrastructure list")); + }); + const calls: WatchCall[] = []; + const freshGenerationStarted = Promise.withResolvers(); + const watch = vi.fn(watchMock(calls, freshGenerationStarted)); + const client = { listInfrastructure, watch } as unknown as KubernetesApiClient; + const projection = new ClusterInfrastructureProjection({ epoch: "replica-one", namespace: "development" }); + const runner = new KubernetesProjectionRunner({ client, projection, hostName: "primary", retryMs: 0 }); + + await runner.start(); + calls[2]!.fail(new KubernetesApiError(410, "resource version expired")); + await relistStarted.promise; + expect(calls.every(call => call.signal.aborted)).toBe(true); + expect(calls).toHaveLength(3); + relist.resolve(fresh); + await freshGenerationStarted.promise; + expect(calls.slice(3).map(call => call.resourceVersion)).toEqual(["200", "201", "202"]); + await runner.stop(); + }); +}); diff --git a/packages/cluster-server/test/observability.test.ts b/packages/cluster-server/test/observability.test.ts new file mode 100644 index 00000000..40816ab8 --- /dev/null +++ b/packages/cluster-server/test/observability.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vite-plus/test"; +import { + ClusterMetrics, + ClusterServerHealth, + JsonLogger, + createAdminHandler, + redactStructuredValue, +} from "../src/observability.ts"; +import { isLoopbackAddress } from "../src/server.ts"; + +describe("cluster-server observability", () => { + it("redacts credentials, prompt content, pairing data, and private paths recursively", () => { + const redacted = redactStructuredValue({ + component: "cluster-server", + session: "session-one", + token: "top-secret", + request: { + deviceToken: "paired-secret", + prompt: "private prompt", + path: "/workspace/private/file", + result: "success", + }, + }); + expect(redacted).toEqual({ + component: "cluster-server", + session: "session-one", + token: "[REDACTED]", + request: { deviceToken: "[REDACTED]", prompt: "[REDACTED]", path: "[REDACTED]", result: "success" }, + }); + const lines: string[] = []; + new JsonLogger(line => lines.push(line), { component: "cluster-server", version: "test" }).info("ci lookup", { + provider: "woodpecker", + token: "secret", + }); + expect(JSON.parse(lines[0] ?? "{}")).toMatchObject({ level: "info", message: "ci lookup", provider: "woodpecker", token: "[REDACTED]" }); + }); + + it("serves separate health/readiness and bounded metrics without a remote drain route", async () => { + const health = new ClusterServerHealth(); + const metrics = new ClusterMetrics({ component: "cluster-server", version: "test" }); + const handler = createAdminHandler({ health, metrics }); + expect((await handler(new Request("http://admin/healthz"))).status).toBe(200); + expect((await handler(new Request("http://admin/readyz"))).status).toBe(503); + health.markKubernetesSynced(); + health.markGatewayListening(); + expect(await (await handler(new Request("http://admin/readyz"))).json()).toEqual({ ready: true }); + + metrics.increment("t4_cluster_gateway_commands_total", { result: "success", provider: "woodpecker" }); + metrics.set("t4_cluster_gateway_connections", 2); + const metricResponse = await handler(new Request("http://admin/metrics")); + expect(metricResponse.headers.get("content-type")).toContain("text/plain"); + const body = await metricResponse.text(); + expect(body).toContain('t4_cluster_gateway_commands_total{component="cluster-server",provider="woodpecker",result="success",version="test"} 1'); + expect(body).not.toContain("token"); + + expect((await handler(new Request("http://admin/drainz", { method: "POST" }))).status).toBe(404); + expect((await handler(new Request("http://admin/readyz"))).status).toBe(200); + }); + + it("recognizes only kernel loopback sources for the preStop drain route", () => { + expect(isLoopbackAddress("127.0.0.1")).toBe(true); + expect(isLoopbackAddress("127.42.0.7")).toBe(true); + expect(isLoopbackAddress("::1")).toBe(true); + expect(isLoopbackAddress("10.42.0.7")).toBe(false); + expect(isLoopbackAddress("fd7a:115c:a1e0::1")).toBe(false); + }); + + it("rejects unbounded or secret-like metric labels", () => { + const metrics = new ClusterMetrics({ component: "cluster-server", version: "test" }); + expect(() => metrics.increment("bad-name", {})).toThrow("metric name"); + expect(() => metrics.increment("valid_metric", { token: "secret" })).toThrow("metric label"); + expect(() => metrics.increment("valid_metric", { result: "x".repeat(129) })).toThrow("metric label"); + }); +}); diff --git a/packages/cluster-server/test/pod-host-router.test.ts b/packages/cluster-server/test/pod-host-router.test.ts new file mode 100644 index 00000000..2c6b05f3 --- /dev/null +++ b/packages/cluster-server/test/pod-host-router.test.ts @@ -0,0 +1,115 @@ +import { expect, it, vi } from "vite-plus/test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { WebSocketPodHostConnector } from "../src/pod-host-router.ts"; + +const welcome = { + v: "omp-app/1", + type: "welcome", + selectedProtocol: "omp-app/1", + hostId: "host-a", + ompVersion: "17.0.5", + ompBuild: "test", + appserverVersion: "0.1.30", + appserverBuild: "cluster-session", + epoch: "pod-epoch", + grantedCapabilities: [], + grantedFeatures: [], + negotiatedLimits: {}, + authentication: "paired", + resumed: false, +} as const; +class MemoryWebSocket { + readyState: number = WebSocket.CONNECTING; + readonly sent = Promise.withResolvers(); + readonly sentValues: string[] = []; + readonly closes: Array<{ readonly code?: number; readonly reason?: string }> = []; + readonly #listeners: Record void>> = {}; + addEventListener(type: string, listener: EventListenerOrEventListenerObject): void { + const callback = typeof listener === "function" + ? (event: unknown) => listener(event as Event) + : (event: unknown) => listener.handleEvent(event as Event); + (this.#listeners[type] ??= []).push(callback); + } + send(value: string | ArrayBufferLike | Blob | ArrayBufferView): void { + this.sent.resolve(String(value)); + this.sentValues.push(String(value)); + } + close(code?: number, reason?: string): void { this.closes.push({ ...(code === undefined ? {} : { code }), ...(reason === undefined ? {} : { reason }) }); this.readyState = WebSocket.CLOSED; } + emit(type: string, event: unknown = {}): void { + if (type === "open") this.readyState = WebSocket.OPEN; + for (const listener of this.#listeners[type] ?? []) listener(event); + } +} + +it("pod connector reads the current projected identity and presents it in the existing hello authentication field", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-pod-identity-")); + try { + const path = join(directory, "token"); + const token = `header.payload.${"s".repeat(64)}`; + await writeFile(path, token, { mode: 0o400 }); + const socket = new MemoryWebSocket(); + const connector = new WebSocketPodHostConnector({ + identityTokenFile: path, + webSocketFactory: () => socket as unknown as WebSocket, + }); + const pending = connector.connect({ clusterSessionId: "session-one", url: "ws://session-one:8787/v1/ws" }, () => undefined); + socket.emit("open"); + const hello = JSON.parse(await socket.sent.promise); + expect(hello.authentication).toEqual({ deviceId: "cluster-server", deviceToken: token }); + socket.emit("message", { data: JSON.stringify(welcome) }); + const connection = await pending; + expect(connection.hostId).toBe("host-a"); + connection.close(); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +it("closes and rejects a malformed frame before the authenticated welcome", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-pod-invalid-")); + try { + const path = join(directory, "token"); + await writeFile(path, `header.payload.${"s".repeat(64)}`, { mode: 0o400 }); + const socket = new MemoryWebSocket(); + const connector = new WebSocketPodHostConnector({ identityTokenFile: path, webSocketFactory: () => socket as unknown as WebSocket }); + const pending = connector.connect({ clusterSessionId: "session-one", url: "ws://session-one:8787/v1/ws" }, () => undefined); + socket.emit("open"); + await socket.sent.promise; + socket.emit("message", { data: "{" }); + await expect(pending).rejects.toThrow(); + expect(socket.closes).toContainEqual({ code: 1002, reason: "invalid upstream frame" }); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +it("keeps idle authority sockets alive without forwarding heartbeat pongs", async () => { + vi.useFakeTimers(); + const directory = await mkdtemp(join(tmpdir(), "t4-pod-heartbeat-")); + try { + const path = join(directory, "token"); + await writeFile(path, `header.payload.${"s".repeat(64)}`, { mode: 0o400 }); + const socket = new MemoryWebSocket(); + const frames: unknown[] = []; + const connector = new WebSocketPodHostConnector({ identityTokenFile: path, keepAliveMs: 10, webSocketFactory: () => socket as unknown as WebSocket }); + const pending = connector.connect({ clusterSessionId: "session-one", url: "ws://session-one:8787/v1/ws" }, frame => frames.push(frame)); + socket.emit("open"); + await socket.sent.promise; + socket.emit("message", { data: JSON.stringify(welcome) }); + const connection = await pending; + await vi.advanceTimersByTimeAsync(10); + const heartbeat = JSON.parse(socket.sentValues.at(-1) ?? "{}"); + expect(heartbeat).toMatchObject({ v: "omp-app/1", type: "ping" }); + socket.emit("message", { data: JSON.stringify({ ...heartbeat, type: "pong" }) }); + expect(frames).toEqual([]); + connection.close(); + const sent = socket.sentValues.length; + await vi.advanceTimersByTimeAsync(20); + expect(socket.sentValues).toHaveLength(sent); + } finally { + vi.useRealTimers(); + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/packages/cluster-server/test/session-host-policy.test.ts b/packages/cluster-server/test/session-host-policy.test.ts new file mode 100644 index 00000000..85d7bb6d --- /dev/null +++ b/packages/cluster-server/test/session-host-policy.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "vite-plus/test"; +import type { RemoteConnection } from "@t4-code/host-service"; +import { commandId, hostId, requestId, sessionId } from "@t4-code/host-wire"; +import { + ClusterInternalRemotePolicy, + decodeClusterInternalClientFrame, + sessionHostConfigFromEnv, + type ClusterIdentityReviewer, +} from "../src/session-host-policy.ts"; + +const SERVER_TOKEN = `header.payload.${"s".repeat(64)}`; +const connection = { + connectionId: "connection-one", + peer: { + identity: { nodeId: "cluster-server", addresses: ["10.42.0.10"], source: "direct" }, + address: "10.42.0.10", + source: "direct", + }, + socket: { connectionId: "connection-one", peer: {} as never, send: () => true, close: () => undefined }, +} as RemoteConnection; +const hello = { + v: "omp-app/1" as const, + type: "hello" as const, + protocol: { min: "omp-app/1", max: "omp-app/1" }, + client: { name: "cluster-server", version: "1", build: "test", platform: "linux" }, + requestedFeatures: ["resume", "session.state", "cluster.operator"], + savedCursors: [], + capabilities: { client: ["sessions.read", "sessions.prompt", "preview.control", "ci.trigger"] }, + authentication: { deviceId: "cluster-server", deviceToken: SERVER_TOKEN }, +}; +class MemoryReviewer implements ClusterIdentityReviewer { + allowed = true; + failure = false; + readonly tokens: string[] = []; + async review(token: string): Promise { + this.tokens.push(token); + if (this.failure) throw new Error("unavailable"); + return this.allowed; + } +} +function policy(reviewer: ClusterIdentityReviewer): ClusterInternalRemotePolicy { + return new ClusterInternalRemotePolicy({ + reviewer, + supportedCapabilities: ["sessions.read", "sessions.prompt", "preview.control"], + supportedFeatures: ["resume", "session.state"], + }); +} + +describe("one-session pod host authority", () => { + it("preserves the existing hello field while admitting a bounded projected bearer only on the internal policy", () => { + expect(decodeClusterInternalClientFrame(hello)).toMatchObject({ authentication: { deviceId: "cluster-server", deviceToken: SERVER_TOKEN } }); + expect(() => decodeClusterInternalClientFrame({ ...hello, authentication: { ...hello.authentication, deviceToken: "short" } })).toThrow("token"); + }); + + it("accepts only the TokenReview-authorized fixed server peer and never grants cluster-server-only names upstream", async () => { + const reviewer = new MemoryReviewer(); + const remotePolicy = policy(reviewer); + expect(await remotePolicy.authenticate(connection, hello)).toEqual({ + authenticated: true, + authentication: "paired", + deviceId: "cluster-server", + grantedCapabilities: ["sessions.read", "sessions.prompt", "preview.control"], + grantedFeatures: ["resume", "session.state"], + }); + expect(reviewer.tokens).toEqual([SERVER_TOKEN]); + reviewer.allowed = false; + expect(await remotePolicy.authenticate(connection, hello)).toMatchObject({ authenticated: false, authentication: "denied" }); + reviewer.failure = true; + expect(await remotePolicy.authenticate(connection, hello)).toMatchObject({ authenticated: false, authentication: "denied" }); + expect(await remotePolicy.authenticate({ ...connection, peer: { ...connection.peer, identity: { ...connection.peer.identity, nodeId: "other" } } }, hello)).toMatchObject({ authenticated: false }); + }); + + it("authorizes only negotiated command capabilities on an authenticated connection", async () => { + const remotePolicy = policy(new MemoryReviewer()); + await remotePolicy.authenticate(connection, hello); + expect(await remotePolicy.authorize(connection, { v: "omp-app/1", type: "ping", nonce: "one", timestamp: "2026-07-20T00:00:00.000Z" }, { connectionId: "connection-one", peer: connection.peer })).toBe(true); + expect(await remotePolicy.authorize(connection, { + v: "omp-app/1", type: "command", requestId: requestId("r1"), commandId: commandId("c1"), hostId: hostId("pod-host"), + sessionId: sessionId("private-session"), command: "session.attach", args: {}, + }, { connectionId: "connection-one", peer: connection.peer })).toBe(true); + expect(await remotePolicy.authorize(connection, { + v: "omp-app/1", type: "command", requestId: requestId("r2"), commandId: commandId("c2"), hostId: hostId("pod-host"), + sessionId: sessionId("private-session"), command: "session.prompt", args: { message: "hello" }, + }, { connectionId: "connection-one", peer: connection.peer })).toBe(true); + expect(await remotePolicy.authorize(connection, { + v: "omp-app/1", type: "command", requestId: requestId("r3"), commandId: commandId("c3"), hostId: hostId("pod-host"), + sessionId: sessionId("private-session"), command: "preview.click", args: { previewId: "preview-one", x: 10, y: 20 }, + }, { connectionId: "connection-one", peer: connection.peer })).toBe(false); + }); + + it("parses fixed Kubernetes reviewer and isolated session paths", () => { + expect(sessionHostConfigFromEnv({ + KUBERNETES_SERVICE_HOST: "10.96.0.1", + KUBERNETES_SERVICE_PORT_HTTPS: "443", + T4_KUBERNETES_API_AUDIENCE: "kubernetes.custom.example", + T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "release-t4-cluster-server", + T4_SESSION_NAME: "session-one", + T4_OMP_EXECUTABLE: "/opt/t4/bin/omp", + T4_SESSION_STATE_ROOT: "/workspace/.t4/sessions/a1b2c3d4", + T4_SESSION_HOST_PORT: "8787", + })).toEqual({ + kubernetesBaseUrl: "https://10.96.0.1:443", + kubernetesTokenPath: "/var/run/secrets/kubernetes.io/serviceaccount/token", + kubernetesCaPath: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + kubernetesNamespacePath: "/var/run/secrets/kubernetes.io/serviceaccount/namespace", + kubernetesApiAudience: "kubernetes.custom.example", + serverServiceAccountName: "release-t4-cluster-server", + sessionName: "session-one", + ompExecutable: "/opt/t4/bin/omp", + stateRoot: "/workspace/.t4/sessions/a1b2c3d4", + port: 8787, + }); + expect(() => sessionHostConfigFromEnv({ KUBERNETES_SERVICE_HOST: "10.96.0.1", T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "server", T4_SESSION_NAME: "bad/name" })).toThrow("T4_SESSION_NAME"); + expect(() => sessionHostConfigFromEnv({ KUBERNETES_SERVICE_HOST: "10.96.0.1", T4_CLUSTER_SERVER_SERVICE_ACCOUNT: "server", T4_SESSION_NAME: "session", T4_SESSION_STATE_ROOT: "/workspace/.t4/sessions/session", T4_KUBERNETES_API_AUDIENCE: "/invalid" })).toThrow("T4_KUBERNETES_API_AUDIENCE"); + }); +}); diff --git a/packages/cluster-server/test/woodpecker.test.ts b/packages/cluster-server/test/woodpecker.test.ts new file mode 100644 index 00000000..c7fd33a0 --- /dev/null +++ b/packages/cluster-server/test/woodpecker.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, it } from "vite-plus/test"; +import { createHash } from "node:crypto"; +import { mkdtemp, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { WoodpeckerProvider, mapWoodpeckerPipeline } from "../src/woodpecker.ts"; + +const correlation = { + sessionId: "session-one", + repositoryId: "t4-code", + ref: "refs/heads/agent/t4-cluster-operator", + commit: "0123456789abcdef0123456789abcdef01234567", +}; +const runCorrelation = { ...correlation, commandId: "command-one" }; +const runKey = createHash("sha256").update(correlation.sessionId).update("\u0000").update(runCorrelation.commandId).digest("base64url"); +const pipeline = { + number: 42, + status: "running", + ref: correlation.ref, + branch: "agent/t4-cluster-operator", + commit: correlation.commit, + created: 1_773_964_800, + started: 1_773_964_810, + finished: 0, + variables: { T4_SESSION_ID: correlation.sessionId, T4_IDEMPOTENCY_KEY: runKey }, + stages: [ + { name: "clone", status: "success" }, + { name: "test", status: "running" }, + ], +}; + +function provider(fetch: typeof globalThis.fetch) { + return new WoodpeckerProvider({ + baseUrl: "https://ci.example.test", + token: "secret-from-kubernetes", + repositories: { "t4-code": { slug: "owner/t4-code" } }, + fetch, + }); +} + +describe("bounded Woodpecker provider", () => { + it("maps only allowlisted categorical pipeline state and canonical HTTPS links", () => { + expect(mapWoodpeckerPipeline(pipeline, correlation, "https://ci.example.test/repos/owner/t4-code/pipeline/42")).toEqual({ + provider: "woodpecker", + correlation: "exact", + repositoryId: "t4-code", + ref: correlation.ref, + branch: "agent/t4-cluster-operator", + commit: correlation.commit, + pipelineNumber: 42, + status: "running", + currentStage: "test", + createdAt: "2026-03-20T00:00:00.000Z", + startedAt: "2026-03-20T00:00:10.000Z", + link: "https://ci.example.test/repos/owner/t4-code/pipeline/42", + }); + for (const [raw, mapped] of [ + ["pending", "queued"], ["queued", "queued"], ["running", "running"], ["success", "success"], + ["failure", "failure"], ["error", "failure"], ["killed", "killed"], ["blocked", "unknown"], + ] as const) expect(mapWoodpeckerPipeline({ ...pipeline, status: raw }, correlation).status).toBe(mapped); + }); + + it("queries exact repository/ref/commit/session correlation and deduplicates before trigger", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + return Response.json([pipeline]); + }) as typeof globalThis.fetch; + const result = await provider(fetch).run(runCorrelation); + expect(result).toMatchObject({ triggered: false, pipelineNumber: 42, status: "running" }); + expect(requests).toHaveLength(1); + expect(requests[0]?.url).toBe("https://ci.example.test/api/repos/owner%2Ft4-code/pipelines?ref=refs%2Fheads%2Fagent%2Ft4-cluster-operator&commit=0123456789abcdef0123456789abcdef01234567&limit=100"); + expect(requests[0]?.init?.headers).toMatchObject({ Authorization: "Bearer secret-from-kubernetes" }); + expect(JSON.stringify(result)).not.toContain("secret-from-kubernetes"); + }); + + it("ignores approximate matches, triggers once with server-resolved URL, and re-queries before retry", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + let query = 0; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + if (!init?.method || init.method === "GET") { + query++; + return Response.json(query === 1 ? [{ ...pipeline, variables: { T4_SESSION_ID: "another-session" } }] : [pipeline]); + } + return Response.json({ ...pipeline, idempotencyKey: runKey }, { status: 201 }); + }) as typeof globalThis.fetch; + const woodpecker = provider(fetch); + const first = await woodpecker.run(runCorrelation); + expect(first.triggered).toBe(true); + const post = requests.find(request => request.init?.method === "POST"); + expect(post?.url).toBe("https://ci.example.test/api/repos/owner%2Ft4-code/pipelines"); + const posted = JSON.parse(String(post?.init?.body)); + expect(posted).toMatchObject({ + ref: correlation.ref, + commit: correlation.commit, + event: "manual", + variables: { T4_SESSION_ID: correlation.sessionId }, + }); + expect(posted.variables.T4_IDEMPOTENCY_KEY).toMatch(/^[A-Za-z0-9_-]{43}$/u); + expect(new Headers(post?.init?.headers).get("idempotency-key")).toBe(`t4-${posted.variables.T4_IDEMPOTENCY_KEY}`); + const second = await woodpecker.run(runCorrelation); + expect(second).toMatchObject({ triggered: false, pipelineNumber: 42 }); + expect(requests.filter(request => request.init?.method === "POST")).toHaveLength(1); + }); + + it("requires an adapter idempotency receipt before reporting a trigger", async () => { + let posts = 0; + const fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + if (!init?.method || init.method === "GET") return Response.json([]); + posts++; + return Response.json(pipeline, { status: 201 }); + }) as typeof globalThis.fetch; + await expect(provider(fetch).run(runCorrelation)).rejects.toThrow("provider-side idempotency"); + expect(posts).toBe(1); + }); + + it("uses a distinct provider idempotency key for each intentional command", async () => { + const keys: string[] = []; + const fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + if (!init?.method || init.method === "GET") return Response.json([]); + const body = JSON.parse(String(init.body)) as { variables: { T4_IDEMPOTENCY_KEY: string } }; + keys.push(body.variables.T4_IDEMPOTENCY_KEY); + return Response.json({ ...pipeline, idempotencyKey: body.variables.T4_IDEMPOTENCY_KEY }, { status: 201 }); + }) as typeof globalThis.fetch; + const woodpecker = provider(fetch); + await woodpecker.run(runCorrelation); + await woodpecker.run({ ...runCorrelation, commandId: "command-two" }); + expect(keys).toHaveLength(2); + expect(new Set(keys).size).toBe(2); + }); + + it("re-reads a bounded projected credential for every provider request", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-woodpecker-token-")); + try { + const tokenFile = join(directory, "token"); + const seen: string[] = []; + const fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + seen.push(new Headers(init?.headers).get("authorization") ?? ""); + return Response.json([]); + }) as typeof globalThis.fetch; + await writeFile(`${tokenFile}.next`, "a".repeat(40), { mode: 0o400 }); + await rename(`${tokenFile}.next`, tokenFile); + const woodpecker = new WoodpeckerProvider({ + baseUrl: "https://ci.example.test", + tokenFile, + repositories: { "t4-code": { slug: "owner/t4-code" } }, + fetch, + }); + await woodpecker.query(correlation); + await writeFile(`${tokenFile}.next`, "b".repeat(40), { mode: 0o400 }); + await rename(`${tokenFile}.next`, tokenFile); + await woodpecker.query(correlation); + expect(seen).toEqual([`Bearer ${"a".repeat(40)}`, `Bearer ${"b".repeat(40)}`]); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("permits only an exact in-cluster HTTP adapter while keeping deep links on HTTPS", async () => { + const woodpecker = new WoodpeckerProvider({ + baseUrl: "http://woodpecker-ci-trigger.linkedin-ci.svc.cluster.local:8080", + webBaseUrl: "https://woodpecker-ci-dev.tail.example.test", + token: "server-side-token", + repositories: { "t4-code": { slug: "owner/t4-code" } }, + fetch: (async () => Response.json([pipeline])) as unknown as typeof globalThis.fetch, + }); + expect((await woodpecker.query(correlation)).link).toBe("https://woodpecker-ci-dev.tail.example.test/repos/owner/t4-code/pipeline/42"); + expect(() => new WoodpeckerProvider({ + baseUrl: "http://ci.example.test", + webBaseUrl: "https://ci.example.test", + token: "token", + repositories: {}, + })).toThrow("in-cluster"); + }); + + it("fails closed for unconfigured repositories, insecure provider URLs, oversized replies, and unknown correlation", async () => { + expect(() => new WoodpeckerProvider({ baseUrl: "http://ci.example.test", token: "secret", repositories: {} })).toThrow("HTTPS"); + expect(() => new WoodpeckerProvider({ baseUrl: "https://ci.example.test", token: "secret", tokenFile: "/run/token", repositories: {} })).toThrow("exactly one"); + const fetch = (async () => Response.json(Array.from({ length: 101 }, () => pipeline))) as unknown as typeof globalThis.fetch; + await expect(provider(fetch).query(correlation)).rejects.toThrow("pipeline response limit"); + await expect(provider(fetch).query({ ...correlation, repositoryId: "not-allowed" })).rejects.toThrow("not allowlisted"); + const unknown = await provider((async () => Response.json([])) as unknown as typeof globalThis.fetch).query(correlation); + expect(unknown).toEqual({ + provider: "woodpecker", + correlation: "unknown", + repositoryId: "t4-code", + ref: correlation.ref, + commit: correlation.commit, + }); + }); +}); diff --git a/packages/cluster-server/tsconfig.json b/packages/cluster-server/tsconfig.json new file mode 100644 index 00000000..fd155a9e --- /dev/null +++ b/packages/cluster-server/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "target": "ES2024", + "module": "ESNext", + "moduleResolution": "Bundler", + "lib": ["ES2024", "DOM", "DOM.AsyncIterable"], + "types": ["bun", "node"], + "noEmit": true, + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": false, + "exactOptionalPropertyTypes": false, + "noUncheckedIndexedAccess": false, + "noImplicitOverride": false + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/packages/host-service/src/remote/listener.ts b/packages/host-service/src/remote/listener.ts index acea98ca..a976ff21 100644 --- a/packages/host-service/src/remote/listener.ts +++ b/packages/host-service/src/remote/listener.ts @@ -50,6 +50,15 @@ export function createListenerPlan(config: RemoteListenerConfig): ListenerPlan { throw new Error("listener port is invalid"); return { mode: "direct", address: config.address, port: config.port, path: "/v1/ws", trustedServeProxy: false }; } +export function createInternalListenerPlan(config: RemoteListenerConfig): ListenerPlan { + if (config.address !== "0.0.0.0" && config.address !== "::") + throw new Error("internal listener must bind an unspecified pod address"); + if (!config.internalPeerNodeId || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u.test(config.internalPeerNodeId)) + throw new Error("internal listener peer id is invalid"); + if (!Number.isInteger(config.port) || config.port < 1 || config.port > 65535) + throw new Error("listener port is invalid"); + return { mode: "direct", address: config.address, port: config.port, path: "/v1/ws", trustedServeProxy: false }; +} export function originAllowed(origin: string | null, allowlist: readonly string[] = []): boolean { return origin === null || allowlist.includes(origin); } @@ -138,9 +147,14 @@ export class BunRemoteListener { const address = normalizeIpAddress(requested); let peer: ListenerPeerContext | undefined; if (this.plan.mode === "direct") { - if (!isTailnetAddress(address) || !this.resolver) - return new Response("Unauthorized", { status: 401 }); - peer = { address, source: "direct", identity: await this.resolver.resolve(address) }; + if (this.config.internalPeerNodeId) { + const normalized = normalizeIpAddress(address); + peer = { address: normalized, source: "direct", identity: { nodeId: this.config.internalPeerNodeId, addresses: [normalized], source: "direct" } }; + } + else { + if (!isTailnetAddress(address) || !this.resolver) return new Response("Unauthorized", { status: 401 }); + peer = { address, source: "direct", identity: await this.resolver.resolve(address) }; + } } else { peer = resolveServePeer(address, request.headers, this.plan.trustedServeProxy); if (!peer) return new Response("Forbidden", { status: 403 }); diff --git a/packages/host-service/src/remote/types.ts b/packages/host-service/src/remote/types.ts index 11fc4b7d..09069c8e 100644 --- a/packages/host-service/src/remote/types.ts +++ b/packages/host-service/src/remote/types.ts @@ -38,6 +38,8 @@ export interface RemoteListenerConfig { port: number; trustedServeProxy?: boolean; serveProxy?: boolean; + /** Fixed peer identity for a pod-network listener whose omp-app hello is authenticated by a dedicated policy. */ + internalPeerNodeId?: string; originAllowlist?: readonly string[]; maxConnections?: number; maxFrameBytes?: number; diff --git a/packages/host-service/src/server.ts b/packages/host-service/src/server.ts index 2eff0633..e4f488f6 100644 --- a/packages/host-service/src/server.ts +++ b/packages/host-service/src/server.ts @@ -95,7 +95,7 @@ import { } from "./ownership.ts"; import { OfficialOmpCapabilityAdapter, OfficialOmpOperationError } from "./official-omp-capabilities.ts"; import { SessionProjection } from "./projection.ts"; -import { BunRemoteListener, createListenerPlan, createServeProxyPlan } from "./remote/listener.ts"; +import { BunRemoteListener, createInternalListenerPlan, createListenerPlan, createServeProxyPlan } from "./remote/listener.ts"; import type { RemoteConnection, RemoteListenerConfig } from "./remote/types.ts"; import { BunRpcChildFactory, RpcChildSupervisor } from "./rpc-child.ts"; import type { @@ -690,6 +690,7 @@ export function appserverSupportedFeatures( const unsupportedAdditiveFeatures = new Set(["host.watch", "session.watch"]); const implementedFeatures = new Set([ "resume", + "session.delta", "prompt.images", "agent.transcript", "session.observer", @@ -1079,9 +1080,11 @@ export class LocalAppserver implements AppserverHandle { const listener = this.#remoteListener ?? new BunRemoteListener( - this.#remoteEndpoint.serveProxy === true - ? createServeProxyPlan(this.#remoteEndpoint) - : createListenerPlan(this.#remoteEndpoint), + this.#remoteEndpoint.internalPeerNodeId + ? createInternalListenerPlan(this.#remoteEndpoint) + : this.#remoteEndpoint.serveProxy === true + ? createServeProxyPlan(this.#remoteEndpoint) + : createListenerPlan(this.#remoteEndpoint), { connected: connection => this.#remoteConnected(connection), message: (connection, message) => this.#remoteMessage(connection, message), @@ -3545,7 +3548,10 @@ export class LocalAppserver implements AppserverHandle { return; } if (typeof raw !== "string") throw new Error("binary websocket frames are not supported"); - const frame = decodeClientFrame(parseBounded(raw)); + const input = parseBounded(raw); + const frame = ws.remote && this.#remotePolicy?.decodeClientFrame + ? this.#remotePolicy.decodeClientFrame(input) + : decodeClientFrame(input); if (frame.type === "command" && frame.command === "session.attach") attachingSessionId = frame.sessionId; if (frame.type === "hello") { if (this.#hello.has(ws)) throw new Error("hello already received"); @@ -4474,7 +4480,12 @@ export class LocalAppserver implements AppserverHandle { private async broadcastIndex(frame: ServerFrame): Promise { const sends: Array> = []; for (const client of this.#clients) { - if (!this.#hello.has(client) || !this.#clientCapabilities.get(client)?.has("sessions.read")) continue; + if ( + !this.#hello.has(client) || + !this.#clientCapabilities.get(client)?.has("sessions.read") || + !this.#clientFeatures.get(client)?.has("session.delta") + ) + continue; sends.push(this.#sendFrame(client, frame)); } await Promise.all(sends); diff --git a/packages/host-service/src/types.ts b/packages/host-service/src/types.ts index bce7a9a6..c4c44283 100644 --- a/packages/host-service/src/types.ts +++ b/packages/host-service/src/types.ts @@ -55,6 +55,8 @@ export interface RemoteAuthorizationContext { readonly sessionRevision?: Revision; } export interface RemoteConnectionPolicy { + /** Optional decoder used only by a policy-bound internal listener; public remote listeners retain the canonical wire decoder. */ + decodeClientFrame?(input: unknown): ClientFrame; authenticate(connection: RemoteConnection, hello: HelloFrame): RemoteHelloDecision | Promise; pairStart?( connection: RemoteConnection, diff --git a/packages/host-service/test/appserver.test.ts b/packages/host-service/test/appserver.test.ts index 908955bf..a0be4b0e 100644 --- a/packages/host-service/test/appserver.test.ts +++ b/packages/host-service/test/appserver.test.ts @@ -239,6 +239,7 @@ describe("appserver lifecycle", () => { test("advertises the exact default implemented feature set", () => { expect(appserverSupportedFeatures({})).toEqual([ "resume", + "session.delta", "prompt.images", "agent.transcript", "session.observer", diff --git a/packages/host-service/test/cluster-default-off.test.ts b/packages/host-service/test/cluster-default-off.test.ts new file mode 100644 index 00000000..036c1535 --- /dev/null +++ b/packages/host-service/test/cluster-default-off.test.ts @@ -0,0 +1,16 @@ +import { expect, test } from "bun:test"; +import { CI_TRIGGER_CAPABILITY, CLUSTER_OPERATOR_FEATURE } from "@t4-code/host-wire"; +import { appserverSupportedCapabilities, appserverSupportedFeatures } from "../src/server.ts"; + +/** Local and ordinary paired hosts know the additive wire names but stay behaviorally unchanged. */ +test("ordinary appservers keep the cluster operator and CI mutation default-off", () => { + expect(appserverSupportedFeatures({})).not.toContain(CLUSTER_OPERATOR_FEATURE); + expect(appserverSupportedFeatures({}, true)).not.toContain(CLUSTER_OPERATOR_FEATURE); + expect(appserverSupportedCapabilities({})).not.toContain(CI_TRIGGER_CAPABILITY); + + // Merely naming the feature/capability in an override cannot turn on an + // implementation that has no explicit cluster authority. + expect(appserverSupportedFeatures({ supportedFeatures: [CLUSTER_OPERATOR_FEATURE] })).not.toContain( + CLUSTER_OPERATOR_FEATURE, + ); +}); diff --git a/packages/host-service/test/remote-transport.test.ts b/packages/host-service/test/remote-transport.test.ts index 4c083ed1..e0a381a3 100644 --- a/packages/host-service/test/remote-transport.test.ts +++ b/packages/host-service/test/remote-transport.test.ts @@ -184,7 +184,7 @@ function peerIdentity(nodeId: string): RemotePeerIdentity { source: "tailscale", }; } -function hello(requestedFeatures: string[] = ["resume"]): string { +function hello(requestedFeatures: string[] = ["resume"], requestedCapabilities?: string[]): string { return JSON.stringify({ v: "omp-app/1", type: "hello", @@ -192,6 +192,7 @@ function hello(requestedFeatures: string[] = ["resume"]): string { client: { name: "test", version: "1", build: "b", platform: "linux" }, requestedFeatures, savedCursors: [], + ...(requestedCapabilities === undefined ? {} : { capabilities: { client: requestedCapabilities } }), }); } function listCommand(requestId: string): string { @@ -418,6 +419,82 @@ describe("remote appserver policy transport", () => { } }); + for (const { name, features, capabilities, forwards } of [ + { + name: "does not forward session deltas without negotiated session.delta", + features: ["resume"], + capabilities: ["sessions.read"], + forwards: false, + }, + { + name: "does not forward negotiated session deltas without sessions.read", + features: ["resume", "session.delta"], + capabilities: [] as string[], + forwards: false, + }, + { + name: "grants and forwards requested session deltas with sessions.read", + features: ["resume", "session.delta"], + capabilities: ["sessions.read"], + forwards: true, + }, + ]) { + test(name, async () => { + const harness = new FakeBunHarness(); + harness.install(); + let currentTitle = "Session"; + try { + const appserver = createAppserver({ + hostId: hostId("host"), + socketPath: join(mkdtempSync(join(tmpdir(), "omp-delta-capability-")), "app.sock"), + discovery: { + list: async () => [ + { + ...leaseSessionRecord(), + title: currentTitle, + updatedAt: + currentTitle === "Session" ? "2026-01-01T00:00:00.000Z" : "2026-01-01T00:00:01.000Z", + }, + ], + }, + remoteEndpoint: { address: "100.64.0.1", port: 1 }, + remoteResolver: { resolve: async () => peerIdentity("node") }, + remotePolicy: { + authenticate: async () => ({ authenticated: true, grantedCapabilities: capabilities }), + authorize: async () => true, + }, + }); + await appserver.start(); + const remote = harness.remote(); + const socket = await openRemote(remote); + await remote.config.websocket?.message?.(socket, hello(features, capabilities)); + await flush(); + expect(sentFrames(socket).find(frame => frame.type === "welcome")).toMatchObject({ + grantedCapabilities: capabilities, + grantedFeatures: features, + }); + socket.sends.length = 0; + + currentTitle = "Renamed session"; + const refreshSocket = await openRemote(remote); + await remote.config.websocket?.message?.(refreshSocket, hello(features, capabilities)); + await flush(); + const deltas = sentFrames(socket).filter(frame => frame.type === "session.delta"); + if (forwards) { + expect(deltas).toHaveLength(1); + expect(deltas[0]).toMatchObject({ + hostId: "host", + sessionId: "session", + upsert: { title: "Renamed session" }, + }); + } else expect(deltas).toEqual([]); + await appserver.stop(); + } finally { + harness.restore(); + } + }); + } + test("a delayed remote transform cannot let a later response overtake an earlier session delta", async () => { const harness = new FakeBunHarness(); harness.install(); @@ -452,7 +529,7 @@ describe("remote appserver policy transport", () => { await appserver.start(); const remote = harness.remote(); const socket = await openRemote(remote); - await remote.config.websocket?.message?.(socket, hello()); + await remote.config.websocket?.message?.(socket, hello(["resume", "session.delta"], ["sessions.read", "sessions.prompt"])); await flush(); socket.sends.length = 0; holdActive = true; diff --git a/packages/host-wire/fixtures/v1/sessions-cluster.json b/packages/host-wire/fixtures/v1/sessions-cluster.json new file mode 100644 index 00000000..f5f4937a --- /dev/null +++ b/packages/host-wire/fixtures/v1/sessions-cluster.json @@ -0,0 +1 @@ +{"v":"omp-app/1","type":"sessions","hostId":"cluster:host-uid","cursor":{"epoch":"replica-uid-1","seq":1},"sessions":[{"hostId":"cluster:host-uid","sessionId":"session-one","project":{"projectId":"workspace-one","name":"T4 code"},"revision":"session-r5","title":"Cluster task","status":"idle","updatedAt":"2026-07-20T12:00:00.000Z","liveState":{"cluster":{"workspaceId":"workspace-one","phase":"Running","condition":{"type":"Available","status":"True","reason":"PodReady","message":"The session pod is ready","observedGeneration":5},"gui":{"state":"Ready","previewId":"preview-one"}},"ci":{"provider":"woodpecker","correlation":"exact","repositoryId":"t4-code","ref":"refs/heads/agent/t4-cluster-operator","commit":"0123456789abcdef0123456789abcdef01234567","pipelineNumber":42,"status":"running","currentStage":"test","createdAt":"2026-07-20T12:00:00.000Z","startedAt":"2026-07-20T12:00:10.000Z","link":"https://ci.example.test/repos/t4-code/pipeline/42"}}}],"totalCount":1,"truncated":false} diff --git a/packages/host-wire/fixtures/v1/workspace-state.json b/packages/host-wire/fixtures/v1/workspace-state.json new file mode 100644 index 00000000..0288d397 --- /dev/null +++ b/packages/host-wire/fixtures/v1/workspace-state.json @@ -0,0 +1 @@ +{"v":"omp-app/1","type":"workspace.state","hostId":"cluster:host-uid","workspaceId":"workspace-one","cursor":{"epoch":"replica-uid-1","seq":2},"revision":"workspace-r2","upsert":{"id":"workspace-one","displayName":"T4 code","phase":"Ready","retentionPolicy":"Retain","storageClass":"t4-workspaces-rwx","capacity":"20Gi","accessMode":"ReadWriteMany","revision":"workspace-r2","condition":{"type":"StorageReady","status":"True","reason":"Bound","message":"PVC is bound","observedGeneration":2}}} diff --git a/packages/host-wire/src/additive.ts b/packages/host-wire/src/additive.ts index b325c244..823d7ad6 100644 --- a/packages/host-wire/src/additive.ts +++ b/packages/host-wire/src/additive.ts @@ -1,3 +1,4 @@ +import { decodeWorkspaceState, type WorkspaceStateFrame } from "./cluster.js"; import { type Cursor, decodeCursor } from "./cursor.js"; import { type DurableEntry, decodeEntry } from "./entry.js"; import { fail } from "./errors.js"; @@ -75,6 +76,7 @@ export const ADDITIVE_FEATURES = [ "catalog.metadata", "settings.metadata", "preview.control", + "cluster.operator", ] as const; export type AdditiveFeature = (typeof ADDITIVE_FEATURES)[number]; export type WireFeature = AdditiveFeature | "resume"; @@ -1113,10 +1115,12 @@ export type AdditiveServerFrame = | AuditEventFrame | CatalogFrame | SettingsFrame + | WorkspaceStateFrame | PreviewFrame; export function decodeAdditiveServerFrame(input: unknown): AdditiveServerFrame { const type = inputObject(input).type; if (typeof type !== "string") fail("INVALID_FRAME", "frame type must be string", "type"); + if (type === "workspace.state") return decodeWorkspaceState(input); if (["host.watch", "session.watch", "session.state", "session.delta"].includes(type)) return decodeWatch(input); if (type === "lease" || type === "prompt.lease") return decodeLease(input); if (type.startsWith("agent.")) return decodeAgentAdditive(input); diff --git a/packages/host-wire/src/capabilities.ts b/packages/host-wire/src/capabilities.ts index 6670f62e..2b7aabc6 100644 --- a/packages/host-wire/src/capabilities.ts +++ b/packages/host-wire/src/capabilities.ts @@ -23,6 +23,7 @@ export const DEVICE_CAPABILITIES = [ "preview.read", "preview.control", "preview.input", + "ci.trigger", ] as const; export type DeviceCapability = (typeof DEVICE_CAPABILITIES)[number]; export const PROTOCOL_FEATURES = [ @@ -53,6 +54,7 @@ export const PROTOCOL_FEATURES = [ "preview.control", "runtime.adapters", "workspace.lifecycle", + "cluster.operator", ] as const; export type ProtocolFeature = (typeof PROTOCOL_FEATURES)[number]; export const REMOTE_DEFAULT_CAPABILITIES = [ diff --git a/packages/host-wire/src/cluster.ts b/packages/host-wire/src/cluster.ts new file mode 100644 index 00000000..1585eb5a --- /dev/null +++ b/packages/host-wire/src/cluster.ts @@ -0,0 +1,381 @@ +import { type Cursor, decodeCursor } from "./cursor.js"; +import { fail } from "./errors.js"; +import { boundedMap, controlFree, inputObject, safeSeq } from "./guards.js"; +import { type HostId, hostId, type Revision, revision } from "./ids.js"; +import { PROTOCOL_VERSION } from "./limits.js"; + +export const CLUSTER_OPERATOR_FEATURE = "cluster.operator" as const; +export const CI_TRIGGER_CAPABILITY = "ci.trigger" as const; +export const CLUSTER_MAX_WORKSPACES = 256; +export const CLUSTER_MAX_CONDITION_MESSAGE_BYTES = 2_048; +export const CLUSTER_MAX_REFERENCE_BYTES = 256; +export const CLUSTER_MAX_REPOSITORY_ID_BYTES = 128; + +export const CLUSTER_CONDITION_STATUSES = ["True", "False", "Unknown"] as const; +export type ClusterConditionStatus = (typeof CLUSTER_CONDITION_STATUSES)[number]; +export interface ClusterCondition { + readonly type: string; + readonly status: ClusterConditionStatus; + readonly reason: string; + readonly message: string; + readonly observedGeneration: number; +} + +export const WORKSPACE_PHASES = ["Pending", "Ready", "Failed", "Terminating", "Unknown"] as const; +export type WorkspacePhase = (typeof WORKSPACE_PHASES)[number]; +export const WORKSPACE_RETENTION_POLICIES = ["Retain", "Delete"] as const; +export type WorkspaceRetentionPolicy = (typeof WORKSPACE_RETENTION_POLICIES)[number]; +export interface WorkspaceInfrastructureProjection { + readonly id: string; + readonly displayName: string; + readonly phase: WorkspacePhase; + readonly retentionPolicy: WorkspaceRetentionPolicy; + readonly storageClass?: string; + readonly capacity?: string; + readonly accessMode: "ReadWriteMany"; + readonly revision: Revision; + readonly condition?: ClusterCondition; +} + +interface WorkspaceStateBase { + readonly v: typeof PROTOCOL_VERSION; + readonly type: "workspace.state"; + readonly hostId: HostId; + readonly workspaceId: string; + readonly cursor: Cursor; + readonly revision: Revision; +} +export interface WorkspaceStateUpsertFrame extends WorkspaceStateBase { + readonly upsert: WorkspaceInfrastructureProjection; + readonly remove?: never; +} +export interface WorkspaceStateRemoveFrame extends WorkspaceStateBase { + readonly upsert?: never; + readonly remove: string; +} +export type WorkspaceStateFrame = WorkspaceStateUpsertFrame | WorkspaceStateRemoveFrame; + +export const SESSION_INFRASTRUCTURE_PHASES = ["Pending", "Running", "Failed", "Terminating", "Unknown"] as const; +export type SessionInfrastructurePhase = (typeof SESSION_INFRASTRUCTURE_PHASES)[number]; +export const SESSION_GUI_STATES = ["Unavailable", "Starting", "Ready", "Failed"] as const; +export type SessionGuiState = (typeof SESSION_GUI_STATES)[number]; +export interface SessionClusterState { + readonly workspaceId: string; + readonly phase: SessionInfrastructurePhase; + readonly condition?: ClusterCondition; + readonly gui: { + readonly state: SessionGuiState; + readonly previewId?: string; + readonly reason?: string; + }; +} + +export const CI_PIPELINE_STATUSES = ["queued", "running", "success", "failure", "killed", "unknown"] as const; +export type CiPipelineStatus = (typeof CI_PIPELINE_STATUSES)[number]; +export interface SessionCiState { + readonly provider: "woodpecker"; + readonly correlation: "exact" | "unknown"; + readonly repositoryId: string; + readonly branch?: string; + readonly ref: string; + readonly commit: string; + readonly pipelineNumber?: number; + readonly status?: CiPipelineStatus; + readonly currentStage?: string; + readonly createdAt?: string; + readonly startedAt?: string; + readonly finishedAt?: string; + readonly link?: string; +} + +export interface LegacyWorkspaceResultItem { + readonly repositoryId: string; + readonly instanceId: string; + readonly ownership: "managed" | "imported-user" | "detected-external" | "repository-root"; + readonly branch: string; + readonly sourceCommit: string; + readonly expectedHead: string; + readonly lifecycle: "creating" | "active" | "archiving" | "archived" | "recovery-required"; + readonly createdAt: number; + readonly updatedAt: number; + readonly archivedAt?: number; +} +export type WorkspaceListItem = LegacyWorkspaceResultItem | WorkspaceInfrastructureProjection; +export interface WorkspaceListResult { + readonly workspaces: readonly WorkspaceListItem[]; + readonly cursor?: Cursor; +} + +export interface ClusterRepositorySelection { + readonly repositoryId: string; + readonly ref?: string; + readonly commit?: string; +} +export interface ClusterWorkspaceCreateArguments { + readonly displayName: string; + readonly retentionPolicy: WorkspaceRetentionPolicy; + readonly capacity: string; + readonly repository?: ClusterRepositorySelection; +} +export interface ClusterSessionCiSelection { + readonly provider: "woodpecker"; + readonly repositoryId: string; + readonly ref: string; + readonly commit: string; +} +export interface ClusterSessionCreateArguments { + readonly workspaceId: string; + readonly title?: string; + readonly runtimeProfile: string; + readonly guiEnabled: boolean; + readonly ci?: ClusterSessionCiSelection; +} +export interface CiRunArguments { + readonly provider: "woodpecker"; + readonly action: "run"; + readonly repositoryId: string; + readonly ref: string; + readonly commit: string; +} +export interface CiRunResult { + readonly triggered: boolean; + readonly pipelineNumber?: number; + readonly status?: CiPipelineStatus; +} + +function exact(value: unknown, path: string, keys: readonly string[]): Record { + const result = boundedMap(value, path); + const allowed = new Set(keys); + for (const key of Object.keys(result)) + if (!allowed.has(key)) fail("INVALID_FRAME", "unknown cluster field", `${path}.${key}`); + return result; +} +function oneOf(value: unknown, path: string, values: T): T[number] { + if (typeof value !== "string" || !(values as readonly string[]).includes(value)) fail("INVALID_FRAME", "unknown categorical state", path); + return value as T[number]; +} +function identifier(value: unknown, path: string, max = 256): string { + const result = controlFree(value, path, max); + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/u.test(result)) fail("INVALID_FRAME", "invalid cluster identifier", path); + return result; +} +function kubernetesName(value: unknown, path: string, max = 253): string { + const result = controlFree(value, path, max); + if (!/^[a-z0-9](?:[-a-z0-9.]*[a-z0-9])?$/u.test(result)) fail("INVALID_FRAME", "invalid Kubernetes resource name", path); + return result; +} +function runtimeProfile(value: unknown, path: string): string { + const result = controlFree(value, path, 64); + if (!/^[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$/u.test(result)) fail("INVALID_FRAME", "invalid runtime profile", path); + return result; +} +function nonEmptyText(value: unknown, path: string, max: number): string { + const result = controlFree(value, path, max); + if (result.length === 0) fail("INVALID_FRAME", "value must not be empty", path); + return result; +} +function repositoryId(value: unknown, path: string): string { + const result = nonEmptyText(value, path, CLUSTER_MAX_REPOSITORY_ID_BYTES); + if (!/^[A-Za-z0-9][A-Za-z0-9._:/-]*$/u.test(result) || result.includes("..") || result.includes("://")) + fail("INVALID_FRAME", "invalid repository identifier", path); + return result; +} +function reference(value: unknown, path: string): string { + return nonEmptyText(value, path, CLUSTER_MAX_REFERENCE_BYTES); +} +function commit(value: unknown, path: string): string { + const result = nonEmptyText(value, path, 64); + if (!/^[0-9a-fA-F]{7,64}$/u.test(result)) fail("INVALID_FRAME", "commit must be 7 to 64 hexadecimal characters", path); + return result; +} +function quantity(value: unknown, path: string): string { + const result = nonEmptyText(value, path, 32); + if (!/^[1-9][0-9]*(?:Ei|Pi|Ti|Gi|Mi|Ki|E|P|T|G|M|K)$/u.test(result)) fail("INVALID_FRAME", "invalid positive Kubernetes storage quantity", path); + return result; +} +function canonicalTimestamp(value: unknown, path: string): string { + const result = controlFree(value, path, 128); + const milliseconds = Date.parse(result); + if (!Number.isFinite(milliseconds) || new Date(milliseconds).toISOString() !== result) + fail("INVALID_FRAME", "timestamp must be canonical ISO", path); + return result; +} +function httpsUrl(value: unknown, path: string): string { + const text = controlFree(value, path, 2_048); + let url: URL; + try { + url = new URL(text); + } catch { + fail("INVALID_FRAME", "invalid HTTPS URL", path); + } + if (url!.protocol !== "https:" || url!.username || url!.password) + fail("INVALID_FRAME", "URL must use HTTPS without credentials", path); + return url!.href; +} + +export function decodeClusterCondition(value: unknown, path = "condition"): ClusterCondition { + const input = exact(value, path, ["type", "status", "reason", "message", "observedGeneration"]); + return { + type: identifier(input.type, `${path}.type`, 128), + status: oneOf(input.status, `${path}.status`, CLUSTER_CONDITION_STATUSES), + reason: identifier(input.reason, `${path}.reason`, 128), + message: controlFree(input.message, `${path}.message`, CLUSTER_MAX_CONDITION_MESSAGE_BYTES), + observedGeneration: safeSeq(input.observedGeneration, `${path}.observedGeneration`), + }; +} + +export function decodeWorkspaceInfrastructureProjection( + value: unknown, + path = "workspace", +): WorkspaceInfrastructureProjection { + const input = exact(value, path, [ + "id", "displayName", "phase", "retentionPolicy", "storageClass", "capacity", "accessMode", "revision", "condition", + ]); + const currentRevision = revision(input.revision, `${path}.revision`); + if (input.accessMode !== "ReadWriteMany") fail("INVALID_FRAME", "workspace access mode must be ReadWriteMany", `${path}.accessMode`); + return { + id: kubernetesName(input.id, `${path}.id`), + displayName: nonEmptyText(input.displayName, `${path}.displayName`, 128), + phase: oneOf(input.phase, `${path}.phase`, WORKSPACE_PHASES), + retentionPolicy: oneOf(input.retentionPolicy, `${path}.retentionPolicy`, WORKSPACE_RETENTION_POLICIES), + ...(input.storageClass === undefined ? {} : { storageClass: kubernetesName(input.storageClass, `${path}.storageClass`, 63) }), + ...(input.capacity === undefined ? {} : { capacity: quantity(input.capacity, `${path}.capacity`) }), + accessMode: "ReadWriteMany", + revision: currentRevision, + ...(input.condition === undefined ? {} : { condition: decodeClusterCondition(input.condition, `${path}.condition`) }), + }; +} + +export function decodeWorkspaceState(value: unknown): WorkspaceStateFrame { + const input = inputObject(value); + if (input.v !== PROTOCOL_VERSION) fail("MISSING_VERSION", `expected ${PROTOCOL_VERSION}`, "v"); + if (input.type !== "workspace.state") fail("INVALID_FRAME", "expected workspace.state frame", "type"); + const keys = ["v", "type", "hostId", "workspaceId", "cursor", "revision", "upsert", "remove"]; + for (const key of Object.keys(input)) if (!keys.includes(key)) fail("INVALID_FRAME", "unknown workspace state field", key); + const host = hostId(input.hostId); + const workspace = identifier(input.workspaceId, "workspaceId"); + const cursor = decodeCursor(input.cursor); + const currentRevision = revision(input.revision); + if ((input.upsert === undefined) === (input.remove === undefined)) + fail("INVALID_FRAME", "workspace state requires exactly one upsert or remove", "frame"); + if (input.upsert !== undefined) { + const upsert = decodeWorkspaceInfrastructureProjection(input.upsert, "upsert"); + if (upsert.id !== workspace || upsert.revision !== currentRevision) + fail("INVALID_FRAME", "workspace state address mismatch", "upsert"); + return { v: PROTOCOL_VERSION, type: "workspace.state", hostId: host, workspaceId: workspace, cursor, revision: currentRevision, upsert }; + } + if (input.remove !== workspace) fail("INVALID_FRAME", "workspace removal must match workspaceId", "remove"); + return { v: PROTOCOL_VERSION, type: "workspace.state", hostId: host, workspaceId: workspace, cursor, revision: currentRevision, remove: workspace }; +} + +export function decodeSessionClusterState(value: unknown, path = "cluster"): SessionClusterState { + const input = exact(value, path, ["workspaceId", "phase", "condition", "gui"]); + const gui = exact(input.gui, `${path}.gui`, ["state", "previewId", "reason"]); + return { + workspaceId: identifier(input.workspaceId, `${path}.workspaceId`), + phase: oneOf(input.phase, `${path}.phase`, SESSION_INFRASTRUCTURE_PHASES), + ...(input.condition === undefined ? {} : { condition: decodeClusterCondition(input.condition, `${path}.condition`) }), + gui: { + state: oneOf(gui.state, `${path}.gui.state`, SESSION_GUI_STATES), + ...(gui.previewId === undefined ? {} : { previewId: identifier(gui.previewId, `${path}.gui.previewId`) }), + ...(gui.reason === undefined ? {} : { reason: controlFree(gui.reason, `${path}.gui.reason`, 512) }), + }, + }; +} + +export function decodeSessionCiState(value: unknown, path = "ci"): SessionCiState { + const input = exact(value, path, [ + "provider", "correlation", "repositoryId", "branch", "ref", "commit", "pipelineNumber", "status", "currentStage", + "createdAt", "startedAt", "finishedAt", "link", + ]); + if (input.provider !== "woodpecker") fail("INVALID_FRAME", "unknown CI provider", `${path}.provider`); + if (input.correlation !== "exact" && input.correlation !== "unknown") + fail("INVALID_FRAME", "unknown CI correlation", `${path}.correlation`); + const output: SessionCiState = { + provider: "woodpecker", + correlation: input.correlation, + repositoryId: repositoryId(input.repositoryId, `${path}.repositoryId`), + ...(input.branch === undefined ? {} : { branch: reference(input.branch, `${path}.branch`) }), + ref: reference(input.ref, `${path}.ref`), + commit: reference(input.commit, `${path}.commit`), + ...(input.pipelineNumber === undefined ? {} : { pipelineNumber: safeSeq(input.pipelineNumber, `${path}.pipelineNumber`) }), + ...(input.status === undefined ? {} : { status: oneOf(input.status, `${path}.status`, CI_PIPELINE_STATUSES) }), + ...(input.currentStage === undefined ? {} : { currentStage: controlFree(input.currentStage, `${path}.currentStage`, 256) }), + ...(input.createdAt === undefined ? {} : { createdAt: canonicalTimestamp(input.createdAt, `${path}.createdAt`) }), + ...(input.startedAt === undefined ? {} : { startedAt: canonicalTimestamp(input.startedAt, `${path}.startedAt`) }), + ...(input.finishedAt === undefined ? {} : { finishedAt: canonicalTimestamp(input.finishedAt, `${path}.finishedAt`) }), + ...(input.link === undefined ? {} : { link: httpsUrl(input.link, `${path}.link`) }), + }; + if ( + output.correlation === "unknown" && + (output.pipelineNumber !== undefined || output.status !== undefined || output.currentStage !== undefined || + output.createdAt !== undefined || output.startedAt !== undefined || output.finishedAt !== undefined || output.link !== undefined) + ) + fail("INVALID_FRAME", "unknown CI correlation cannot carry pipeline state", path); + return output; +} + +function decodeRepository(value: unknown, path: string, requireRefCommit: boolean): ClusterRepositorySelection { + const input = exact(value, path, ["repositoryId", "ref", "commit"]); + if (requireRefCommit && (input.ref === undefined || input.commit === undefined)) + fail("INVALID_FRAME", "CI repository requires ref and commit", path); + if (input.commit !== undefined && input.ref === undefined) + fail("INVALID_FRAME", "repository commit requires an explicit ref", path); + return { + repositoryId: repositoryId(input.repositoryId, `${path}.repositoryId`), + ...(input.ref === undefined ? {} : { ref: reference(input.ref, `${path}.ref`) }), + ...(input.commit === undefined ? {} : { commit: commit(input.commit, `${path}.commit`) }), + }; +} + +export function decodeClusterWorkspaceCreateArguments(value: unknown): ClusterWorkspaceCreateArguments { + const input = exact(value, "args", ["displayName", "retentionPolicy", "capacity", "repository"]); + return { + displayName: nonEmptyText(input.displayName, "args.displayName", 128), + retentionPolicy: oneOf(input.retentionPolicy, "args.retentionPolicy", WORKSPACE_RETENTION_POLICIES), + capacity: quantity(input.capacity, "args.capacity"), + ...(input.repository === undefined ? {} : { repository: decodeRepository(input.repository, "args.repository", false) }), + }; +} + +export function decodeClusterSessionCreateArguments(value: unknown): ClusterSessionCreateArguments { + const input = exact(value, "args", ["workspaceId", "title", "runtimeProfile", "guiEnabled", "ci"]); + let ci: ClusterSessionCiSelection | undefined; + if (input.ci !== undefined) { + const raw = exact(input.ci, "args.ci", ["provider", "repositoryId", "ref", "commit"]); + if (raw.provider !== "woodpecker") fail("INVALID_FRAME", "unknown CI provider", "args.ci.provider"); + const repository = decodeRepository({ repositoryId: raw.repositoryId, ref: raw.ref, commit: raw.commit }, "args.ci", true); + ci = { provider: "woodpecker", repositoryId: repository.repositoryId, ref: repository.ref!, commit: repository.commit! }; + } + if (typeof input.guiEnabled !== "boolean") fail("INVALID_FRAME", "guiEnabled must be boolean", "args.guiEnabled"); + return { + workspaceId: kubernetesName(input.workspaceId, "args.workspaceId"), + ...(input.title === undefined ? {} : { title: nonEmptyText(input.title, "args.title", 128) }), + runtimeProfile: runtimeProfile(input.runtimeProfile, "args.runtimeProfile"), + guiEnabled: input.guiEnabled, + ...(ci === undefined ? {} : { ci }), + }; +} + +export function decodeCiRunArguments(value: unknown): CiRunArguments { + const input = exact(value, "args", ["provider", "action", "repositoryId", "ref", "commit"]); + if (input.provider !== "woodpecker") fail("INVALID_FRAME", "unknown CI provider", "args.provider"); + if (input.action !== "run") fail("INVALID_FRAME", "unknown CI action", "args.action"); + return { + provider: "woodpecker", + action: "run", + repositoryId: repositoryId(input.repositoryId, "args.repositoryId"), + ref: reference(input.ref, "args.ref"), + commit: commit(input.commit, "args.commit"), + }; +} + +export function decodeCiRunResult(value: unknown): CiRunResult { + const input = exact(value, "result", ["triggered", "pipelineNumber", "status"]); + if (typeof input.triggered !== "boolean") fail("INVALID_FRAME", "triggered must be boolean", "result.triggered"); + return { + triggered: input.triggered, + ...(input.pipelineNumber === undefined ? {} : { pipelineNumber: safeSeq(input.pipelineNumber, "result.pipelineNumber") }), + ...(input.status === undefined ? {} : { status: oneOf(input.status, "result.status", CI_PIPELINE_STATUSES) }), + }; +} diff --git a/packages/host-wire/src/command.ts b/packages/host-wire/src/command.ts index 87026ee0..ed35373b 100644 --- a/packages/host-wire/src/command.ts +++ b/packages/host-wire/src/command.ts @@ -8,6 +8,13 @@ import { type PreviewAction, type PreviewSnapshot, } from "./additive.js"; +import { + decodeCiRunArguments, + decodeCiRunResult, + decodeClusterSessionCreateArguments, + decodeClusterWorkspaceCreateArguments, + decodeWorkspaceInfrastructureProjection, +} from "./cluster.js"; import { decodeBrokerStatusResult } from "./broker.js"; import type { DeviceCapability } from "./capabilities.js"; import { decodeCursor } from "./cursor.js"; @@ -720,6 +727,13 @@ export const COMMAND_DESCRIPTORS: Readonly> = revisionOwner: "session", confirmation: "none", }, + "ci.run": { + capability: "ci.trigger", + scope: "session", + revision: "required", + revisionOwner: "session", + confirmation: "none", + }, }; export const DESKTOP_CATALOG_COMMANDS: readonly string[] = Object.freeze( Object.entries(COMMAND_DESCRIPTORS) @@ -1020,7 +1034,10 @@ function decodeRuntimeResultItem(value: unknown, path: string): Record { - const item = strictMap(value, path, [ + const candidate = boundedMap(value, path); + if (Object.hasOwn(candidate, "id")) + return decodeWorkspaceInfrastructureProjection(candidate, path) as unknown as Record; + const item = strictMap(candidate, path, [ "repositoryId", "instanceId", "ownership", @@ -1593,7 +1610,10 @@ export const COMMAND_ARGUMENT_DECODERS: Readonly { - const x = strictArgs(value, ["projectId", "name", "branch", "sourceCommit"]); + const candidate = boundedMap(value, "args"); + if (Object.hasOwn(candidate, "displayName")) + return decodeClusterWorkspaceCreateArguments(candidate) as unknown as CommandArguments; + const x = strictArgs(candidate, ["projectId", "name", "branch", "sourceCommit"]); projectId(x.projectId, "args.projectId"); controlFree(x.name, "args.name", 128); controlFree(x.branch, "args.branch", 256); @@ -1623,7 +1643,10 @@ export const COMMAND_ARGUMENT_DECODERS: Readonly { - const x = strictArgs(value, ["projectId", "title", "runtimeId", "workspaceInstanceId"]); + const candidate = boundedMap(value, "args"); + if (Object.hasOwn(candidate, "workspaceId")) + return decodeClusterSessionCreateArguments(candidate) as unknown as CommandArguments; + const x = strictArgs(candidate, ["projectId", "title", "runtimeId", "workspaceInstanceId"]); projectId(x.projectId, "args.projectId"); if (x.title !== undefined) boundedText(x.title, "args.title", 512); const runtimeId = x.runtimeId === undefined ? undefined : controlFree(x.runtimeId, "args.runtimeId", 64); @@ -1848,6 +1871,7 @@ export const COMMAND_ARGUMENT_DECODERS: Readonly decodeCiRunArguments(value) as unknown as CommandArguments, }; export const COMMAND_RESULT_DECODERS: Readonly CommandResult>> = { "runtime.list": value => { @@ -1859,11 +1883,12 @@ export const COMMAND_RESULT_DECODERS: Readonly { - const x = strictResult(value, ["workspaces"]); + const x = strictResult(value, ["workspaces", "cursor"]); return { workspaces: boundedArray(x.workspaces, "result.workspaces", 256).map((workspace, index) => decodeWorkspaceResultItem(workspace, `result.workspaces[${index}]`), ), + ...(x.cursor === undefined ? {} : { cursor: decodeCursor(x.cursor, "result.cursor") }), }; }, "workspace.create": value => { @@ -2005,6 +2030,7 @@ export const COMMAND_RESULT_DECODERS: Readonly decodeCiRunResult(value) as unknown as CommandResult, }; export function decodeCommandArguments(command: string, value: unknown): CommandArguments { const decoder = COMMAND_ARGUMENT_DECODERS[command]; diff --git a/packages/host-wire/src/index.ts b/packages/host-wire/src/index.ts index d46423f9..0531e397 100644 --- a/packages/host-wire/src/index.ts +++ b/packages/host-wire/src/index.ts @@ -3,6 +3,7 @@ export * from "./agents.js"; export * from "./audit.js"; export * from "./broker.js"; export * from "./capabilities.js"; +export * from "./cluster.js"; export * from "./command.js"; export * from "./cursor.js"; export * from "./entry.js"; diff --git a/packages/host-wire/src/session-index.ts b/packages/host-wire/src/session-index.ts index 955013b2..da7fa369 100644 --- a/packages/host-wire/src/session-index.ts +++ b/packages/host-wire/src/session-index.ts @@ -1,3 +1,9 @@ +import { + decodeSessionCiState, + decodeSessionClusterState, + type SessionCiState, + type SessionClusterState, +} from "./cluster.js"; import { type Cursor, decodeCursor } from "./cursor.js"; import { fail } from "./errors.js"; import { @@ -117,6 +123,8 @@ export type SessionControlState = export interface SessionLiveState { sessionControl?: SessionControlState; providerTransport?: ProviderTransportState; + cluster?: SessionClusterState; + ci?: SessionCiState; [key: string]: unknown; } export interface SessionRef { @@ -363,6 +371,9 @@ export function decodeSessionRef(value: unknown, path: string): SessionRef { decodeSessionControl(liveState.sessionControl, `${path}.liveState.sessionControl`); if (liveState.providerTransport !== undefined) decodeProviderTransportState(liveState.providerTransport, `${path}.liveState.providerTransport`); + if (liveState.cluster !== undefined) + decodeSessionClusterState(liveState.cluster, `${path}.liveState.cluster`); + if (liveState.ci !== undefined) decodeSessionCiState(liveState.ci, `${path}.liveState.ci`); } if (session.model !== undefined) controlFree(session.model, `${path}.model`, 256); if (session.thinking !== undefined) controlFree(session.thinking, `${path}.thinking`, 256); diff --git a/packages/host-wire/test/app-wire.test.ts b/packages/host-wire/test/app-wire.test.ts index b4626b85..18354ed1 100644 --- a/packages/host-wire/test/app-wire.test.ts +++ b/packages/host-wire/test/app-wire.test.ts @@ -1179,6 +1179,7 @@ describe("app-wire authority", () => { "preview.lease.renew": "session", "preview.lease.release": "session", "preview.handoff": "session", + "ci.run": "session", "files.read": "authority", "files.write": "authority", "files.patch": "authority", diff --git a/packages/host-wire/test/cluster-operator.test.ts b/packages/host-wire/test/cluster-operator.test.ts new file mode 100644 index 00000000..7c8d2c7d --- /dev/null +++ b/packages/host-wire/test/cluster-operator.test.ts @@ -0,0 +1,249 @@ +import { describe, expect, test } from "bun:test"; +import { + AppWireError, + CI_TRIGGER_CAPABILITY, + CLUSTER_OPERATOR_FEATURE, + COMMAND_DESCRIPTORS, + DEVICE_CAPABILITIES, + PROTOCOL_FEATURES, + decodeCiRunArguments, + decodeCiRunResult, + decodeClusterSessionCreateArguments, + decodeClusterWorkspaceCreateArguments, + decodeCommand, + decodeCommandResult, + decodeServerFrame, + decodeSessionRef, + decodeWelcome, + decodeWorkspaceState, +} from "../src/index.js"; + +const session = { + hostId: "cluster-host-uid-1", + sessionId: "session-one", + project: { projectId: "workspace-one" }, + revision: "session-r7", + title: "Cluster task", + status: "idle", + updatedAt: "2026-07-20T12:00:00.000Z", +}; + +const condition = { + type: "Available", + status: "True", + reason: "PodReady", + message: "The session pod is ready", + observedGeneration: 7, +}; + +describe("cluster operator wire contract", () => { + test("negotiates one additive operator feature and one CI mutation capability", () => { + expect(CLUSTER_OPERATOR_FEATURE).toBe("cluster.operator"); + expect(CI_TRIGGER_CAPABILITY).toBe("ci.trigger"); + expect(PROTOCOL_FEATURES).toContain(CLUSTER_OPERATOR_FEATURE); + expect(DEVICE_CAPABILITIES).toContain(CI_TRIGGER_CAPABILITY); + expect( + decodeWelcome({ + v: "omp-app/1", + type: "welcome", + selectedProtocol: "omp-app/1", + hostId: "cluster-host-uid-1", + ompVersion: "17.0.5", + ompBuild: "8476f4451ed95c5d5401785d279a93d3c659fac4", + appserverVersion: "0.1.30", + appserverBuild: "cluster", + epoch: "replica-pod-uid-1", + grantedCapabilities: ["sessions.read", "ci.trigger"], + grantedFeatures: ["resume", "cluster.operator"], + negotiatedLimits: {}, + authentication: "paired", + resumed: false, + }), + ).toMatchObject({ grantedCapabilities: ["sessions.read", "ci.trigger"], grantedFeatures: ["resume", "cluster.operator"] }); + }); + + test("decodes bounded workspace upsert and removal frames in their own cursor domain", () => { + const upsert = decodeWorkspaceState({ + v: "omp-app/1", + type: "workspace.state", + hostId: "cluster-host-uid-1", + workspaceId: "workspace-one", + cursor: { epoch: "replica-pod-uid-1", seq: 19 }, + revision: "workspace-r3", + upsert: { + id: "workspace-one", + displayName: "T4 code", + phase: "Ready", + retentionPolicy: "Retain", + storageClass: "t4-workspaces-rwx", + capacity: "20Gi", + accessMode: "ReadWriteMany", + revision: "workspace-r3", + condition, + }, + }); + expect(upsert.cursor).toEqual({ epoch: "replica-pod-uid-1", seq: 19 }); + expect(upsert.upsert.id).toBe("workspace-one"); + expect(decodeServerFrame(upsert).type).toBe("workspace.state"); + + const remove = decodeWorkspaceState({ + v: "omp-app/1", + type: "workspace.state", + hostId: "cluster-host-uid-1", + workspaceId: "workspace-one", + cursor: { epoch: "replica-pod-uid-1", seq: 20 }, + revision: "workspace-r4", + remove: "workspace-one", + }); + expect(remove).toMatchObject({ workspaceId: "workspace-one", remove: "workspace-one" }); + + for (const malformed of [ + { ...upsert, remove: true }, + { ...upsert, upsert: undefined }, + { ...upsert, workspaceId: "other" }, + { ...upsert, upsert: { ...upsert.upsert, accessMode: "ReadWriteOnce" } }, + { ...upsert, upsert: { ...upsert.upsert, condition: { ...condition, token: "secret" } } }, + ]) expect(() => decodeWorkspaceState(malformed)).toThrow(AppWireError); + }); + + test("strictly decodes cluster and Woodpecker live state without credentials or private paths", () => { + const decoded = decodeSessionRef( + { + ...session, + liveState: { + cluster: { + workspaceId: "workspace-one", + phase: "Running", + condition, + gui: { state: "Ready", previewId: "preview-one" }, + }, + ci: { + provider: "woodpecker", + correlation: "exact", + repositoryId: "t4-code", + ref: "refs/heads/agent/t4-cluster-operator", + commit: "0123456789abcdef0123456789abcdef01234567", + pipelineNumber: 42, + status: "running", + currentStage: "test", + startedAt: "2026-07-20T12:00:00.000Z", + link: "https://ci.example.test/repos/t4-code/pipeline/42", + }, + }, + }, + "session", + ); + expect(decoded.liveState?.cluster?.gui.state).toBe("Ready"); + expect(decoded.liveState?.ci?.pipelineNumber).toBe(42); + + for (const liveState of [ + { cluster: { ...decoded.liveState?.cluster, phase: "Succeeded" } }, + { cluster: { ...decoded.liveState?.cluster, podPath: "/var/run/private" } }, + { ci: { ...decoded.liveState?.ci, provider: "github" } }, + { ci: { ...decoded.liveState?.ci, correlation: "approximate" } }, + { ci: { ...decoded.liveState?.ci, token: "secret" } }, + { ci: { ...decoded.liveState?.ci, link: "http://ci.example.test/run/42" } }, + ]) expect(() => decodeSessionRef({ ...session, liveState }, "session")).toThrow(AppWireError); + }); + + test("types ci.run as a session-revision-gated allowlist with no endpoint, token, or shell fields", () => { + expect(COMMAND_DESCRIPTORS["ci.run"]).toEqual({ + capability: "ci.trigger", + scope: "session", + revision: "required", + revisionOwner: "session", + confirmation: "none", + }); + const args = { + provider: "woodpecker", + action: "run", + repositoryId: "t4-code", + ref: "refs/heads/agent/t4-cluster-operator", + commit: "0123456789abcdef0123456789abcdef01234567", + }; + expect(decodeCiRunArguments(args)).toEqual(args); + expect( + decodeCommand({ + v: "omp-app/1", + type: "command", + requestId: "request-ci-1", + commandId: "command-ci-1", + hostId: "cluster-host-uid-1", + sessionId: "session-one", + command: "ci.run", + expectedRevision: "session-r7", + args, + }).args, + ).toEqual(args); + expect(decodeCiRunResult({ triggered: false, pipelineNumber: 42, status: "running" })).toEqual({ + triggered: false, + pipelineNumber: 42, + status: "running", + }); + for (const extra of [ + { url: "https://ci.example.test" }, + { token: "secret" }, + { shell: "curl ci" }, + { action: "restart" }, + ]) expect(() => decodeCiRunArguments({ ...args, ...extra })).toThrow(AppWireError); + }); + + test("keeps legacy workspace.list fixtures compatible while adding cluster bootstrap and create shapes", () => { + const legacy = { + repositoryId: "project-one", + instanceId: "instance-one", + ownership: "managed", + branch: "main", + sourceCommit: "abc", + expectedHead: "abc", + lifecycle: "active", + createdAt: 1, + updatedAt: 2, + }; + expect(decodeCommandResult("workspace.list", { workspaces: [legacy] })).toEqual({ workspaces: [legacy] }); + const clusterWorkspace = { + id: "workspace-one", + displayName: "T4 code", + phase: "Pending", + retentionPolicy: "Delete", + accessMode: "ReadWriteMany", + revision: "workspace-r1", + }; + expect( + decodeCommandResult("workspace.list", { + workspaces: [clusterWorkspace], + cursor: { epoch: "replica-pod-uid-1", seq: 4 }, + }), + ).toEqual({ workspaces: [clusterWorkspace], cursor: { epoch: "replica-pod-uid-1", seq: 4 } }); + expect( + decodeClusterWorkspaceCreateArguments({ + displayName: "T4 code", + retentionPolicy: "Retain", + capacity: "20Gi", + repository: { repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, + }), + ).toMatchObject({ displayName: "T4 code", capacity: "20Gi" }); + expect( + decodeClusterSessionCreateArguments({ + workspaceId: "workspace-one", + title: "Agent task", + runtimeProfile: "omp-17.0.5", + guiEnabled: true, + ci: { provider: "woodpecker", repositoryId: "t4-code", ref: "refs/heads/main", commit: "abcdef0" }, + }), + ).toMatchObject({ workspaceId: "workspace-one", runtimeProfile: "omp-17.0.5" }); + for (const malformed of [ + { displayName: "x".repeat(129), retentionPolicy: "Retain", capacity: "20Gi" }, + { displayName: "T4 code", retentionPolicy: "Retain", capacity: "0Gi" }, + { displayName: "T4 code", retentionPolicy: "Retain", capacity: "20Gi", storageClass: "client-owned" }, + { displayName: "T4 code", retentionPolicy: "Retain", capacity: "20Gi", repository: { repositoryId: "t4-code", commit: "abcdef0" } }, + { displayName: "T4 code", retentionPolicy: "Retain", capacity: "20Gi", repository: { repositoryId: "t4-code", ref: "main", commit: "abc" } }, + ]) expect(() => decodeClusterWorkspaceCreateArguments(malformed)).toThrow(AppWireError); + for (const malformed of [ + { workspaceId: "Workspace:one", title: "Agent", runtimeProfile: "default", guiEnabled: true }, + { workspaceId: "workspace-one", title: "x".repeat(129), runtimeProfile: "default", guiEnabled: true }, + { workspaceId: "workspace-one", title: "Agent", runtimeProfile: "Default:unsafe", guiEnabled: true }, + { workspaceId: "workspace-one", title: "Agent", runtimeProfile: "default", guiEnabled: true, ci: { provider: "woodpecker", repositoryId: "t4-code", ref: "main", commit: "abc" } }, + ]) expect(() => decodeClusterSessionCreateArguments(malformed)).toThrow(AppWireError); + }); +}); diff --git a/packages/protocol/src/server-event.ts b/packages/protocol/src/server-event.ts index 914083d9..7f6d0941 100644 --- a/packages/protocol/src/server-event.ts +++ b/packages/protocol/src/server-event.ts @@ -26,6 +26,7 @@ const OMP_SERVER_EVENT_KIND_MEMBERS = { "session.watch": true, "session.state": true, "session.delta": true, + "workspace.state": true, lease: true, "prompt.lease": true, "agent.state": true, diff --git a/packages/protocol/test/distribution.test.ts b/packages/protocol/test/distribution.test.ts index ca0857a5..637f2655 100644 --- a/packages/protocol/test/distribution.test.ts +++ b/packages/protocol/test/distribution.test.ts @@ -114,13 +114,16 @@ function sha256(path: string): string { } /** Sorted POSIX fixture path + NUL + raw bytes + NUL, hashed as one stream. */ -function goldenCorpusSha256(root: string): string { +function goldenCorpusSha256(root: string, excluded: ReadonlySet = new Set()): string { const paths: string[] = []; function visit(directory: string): void { for (const entry of readdirSync(directory, { withFileTypes: true })) { const absolute = join(directory, entry.name); if (entry.isDirectory()) visit(absolute); - else if (entry.isFile()) paths.push(relative(root, absolute).split(sep).join("/")); + else if (entry.isFile()) { + const fixturePath = relative(root, absolute).split(sep).join("/"); + if (!excluded.has(fixturePath)) paths.push(fixturePath); + } } } visit(root); @@ -149,9 +152,16 @@ describe("T4 host-wire distribution", () => { }); expect(manifest.createdAt).toBe("2026-07-20T03:17:05Z"); expect(sha256(tarballPath)).toBe(manifest.tarballSha256); - expect(goldenCorpusSha256(join(installedRoot, "fixtures", "v1"))).toBe( - manifest.goldenCorpusSha256, + const installedFixtures = join(installedRoot, "fixtures", "v1"); + expect(goldenCorpusSha256(installedFixtures)).toBe( + "8987b23e778ae5d4aee9b58d430cff007194ba47c8313d78db678f7a418b88bc", ); + expect( + goldenCorpusSha256( + installedFixtures, + new Set(["sessions-cluster.json", "workspace-state.json"]), + ), + ).toBe(manifest.goldenCorpusSha256); const installedPackage = JSON.parse( readFileSync(join(installedRoot, "package.json"), "utf8"), ) as Record; diff --git a/packages/protocol/test/import-boundary.test.ts b/packages/protocol/test/import-boundary.test.ts index c0b28ed9..1afdf6c4 100644 --- a/packages/protocol/test/import-boundary.test.ts +++ b/packages/protocol/test/import-boundary.test.ts @@ -5,7 +5,11 @@ import { describe, expect, it } from "vite-plus/test"; const repoRoot = resolve(import.meta.dirname, "../../.."); const protocolRoot = join(repoRoot, "packages", "protocol"); -const hostRoots = [join(repoRoot, "packages", "host-wire"), join(repoRoot, "packages", "host-service")]; +const hostRoots = [ + join(repoRoot, "packages", "host-wire"), + join(repoRoot, "packages", "host-service"), + join(repoRoot, "packages", "cluster-server"), +]; const scannedRoots = [join(repoRoot, "apps"), join(repoRoot, "packages")]; function sourceFiles(directory: string): string[] { diff --git a/packages/protocol/test/server-event.test.ts b/packages/protocol/test/server-event.test.ts index be278b81..5f264c5f 100644 --- a/packages/protocol/test/server-event.test.ts +++ b/packages/protocol/test/server-event.test.ts @@ -49,7 +49,7 @@ function pairOk(): PairOkFrame { describe("shared server events", () => { it("publishes the complete immutable server event vocabulary", () => { - expect(OMP_SERVER_EVENT_KINDS).toHaveLength(45); + expect(OMP_SERVER_EVENT_KINDS).toHaveLength(46); expect(new Set(OMP_SERVER_EVENT_KINDS).size).toBe(OMP_SERVER_EVENT_KINDS.length); expect(Object.isFrozen(OMP_SERVER_EVENT_KINDS)).toBe(true); expect(OMP_SERVER_EVENT_KINDS).toContain("welcome"); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7b3246bb..b0d6bb7c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -261,6 +261,28 @@ importers: specifier: 'catalog:' version: 0.2.2(@types/node@26.1.1)(jiti@2.7.0)(typescript@6.0.3) + packages/cluster-server: + dependencies: + '@t4-code/host-service': + specifier: workspace:* + version: link:../host-service + '@t4-code/host-wire': + specifier: workspace:* + version: link:../host-wire + devDependencies: + '@types/bun': + specifier: 1.3.5 + version: 1.3.5 + '@types/node': + specifier: 24.12.4 + version: 24.12.4 + typescript: + specifier: ~6.0.3 + version: 6.0.3 + vite-plus: + specifier: 0.2.2 + version: 0.2.2(@types/node@24.12.4)(jiti@2.7.0)(typescript@6.0.3) + packages/client: dependencies: '@t4-code/protocol': diff --git a/scripts/check-release-consistency.mjs b/scripts/check-release-consistency.mjs index 14afbcad..dd787772 100644 --- a/scripts/check-release-consistency.mjs +++ b/scripts/check-release-consistency.mjs @@ -1,4 +1,4 @@ -import { readFileSync, readdirSync } from "node:fs"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; import { resolve } from "node:path"; import { isDeepStrictEqual } from "node:util"; import { fileURLToPath } from "node:url"; @@ -69,7 +69,10 @@ export function discoverReleasePackagePaths(repoRoot) { for (const parent of ["apps", "packages"]) { const entries = readdirSync(resolve(repoRoot, parent), { withFileTypes: true }); for (const entry of entries) { - if (entry.isDirectory()) paths.push(`${parent}/${entry.name}/package.json`); + if (entry.isDirectory()) { + const manifestPath = `${parent}/${entry.name}/package.json`; + if (existsSync(resolve(repoRoot, manifestPath))) paths.push(manifestPath); + } } } return paths.sort((a, b) => a.localeCompare(b)); @@ -811,6 +814,11 @@ export function collectReleaseConsistencyErrors(files, releaseTag) { "path: artifacts/legacy-bridge-continuity/", "if-no-files-found: error", "tooling:", + "cluster:", + "actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16", + "run: pnpm test:cluster:ci", + "run: go test ./...", + "run: helm lint deploy/charts/t4-cluster", "android-debug:", "flutter:", "flutter-android:", @@ -823,7 +831,7 @@ export function collectReleaseConsistencyErrors(files, releaseTag) { "test -x apps/flutter/build/macos/Build/Products/Debug/t4code.app/Contents/Resources/runtime/t4-host", "name: verify", "if: ${{ always() }}", - "needs: [changes, core, legacy-bridge-continuity, tooling, android-debug, flutter, flutter-android, flutter-apple]", + "needs: [changes, core, legacy-bridge-continuity, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", 'test "$CHANGES_RESULT" = success', 'test "$CORE_RESULT" = success', "for result in \\", diff --git a/scripts/check-release-consistency.test.mjs b/scripts/check-release-consistency.test.mjs index 4796e65b..8acccbed 100644 --- a/scripts/check-release-consistency.test.mjs +++ b/scripts/check-release-consistency.test.mjs @@ -7,6 +7,7 @@ import { load as parseYaml } from "js-yaml"; import { collectReleaseConsistencyErrors, + discoverReleasePackagePaths, loadReleaseContractFiles, } from "./check-release-consistency.mjs"; @@ -65,6 +66,10 @@ test("current source tree has one consistent release version", () => { assert.deepEqual(collectReleaseConsistencyErrors(files), []); }); +test("release package discovery ignores non-Node package directories", () => { + assert.equal(discoverReleasePackagePaths(repoRoot).includes("packages/cluster-operator/package.json"), false); +}); + test("rejects duplicate keys in JSON release contracts", () => { const duplicated = changed("compat/omp-app-matrix.json", (text) => text.replace( @@ -245,7 +250,7 @@ test("rejects updater channel, stable manifest, and publication-contract drift", ".github/workflows/ci.yml", (text) => text.replace( - "needs: [changes, core, legacy-bridge-continuity, tooling, android-debug, flutter, flutter-android, flutter-apple]", + "needs: [changes, core, legacy-bridge-continuity, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", "needs: [changes, core, tooling, android-debug]", ), ], @@ -524,6 +529,11 @@ test("deploys release site source only after artifact publication", () => { assert.ok(ciWorkflow.includes("path: artifacts/legacy-bridge-continuity/")); assert.ok(ciWorkflow.includes("if-no-files-found: error")); assert.ok(ciWorkflow.includes("tooling:")); + assert.ok(ciWorkflow.includes("cluster:")); + assert.ok(ciWorkflow.includes("actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16")); + assert.ok(ciWorkflow.includes("run: pnpm test:cluster:ci")); + assert.ok(ciWorkflow.includes("run: go test ./...")); + assert.ok(ciWorkflow.includes("run: helm lint deploy/charts/t4-cluster")); assert.ok(ciWorkflow.includes("flutter:")); assert.ok(ciWorkflow.includes("flutter-android:")); assert.ok(ciWorkflow.includes("flutter-apple:")); @@ -545,7 +555,7 @@ test("deploys release site source only after artifact publication", () => { assert.ok(ciWorkflow.includes("if: ${{ always() }}")); assert.ok( ciWorkflow.includes( - "needs: [changes, core, legacy-bridge-continuity, tooling, android-debug, flutter, flutter-android, flutter-apple]", + "needs: [changes, core, legacy-bridge-continuity, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", ), ); assert.ok(ciWorkflow.includes('test "$CHANGES_RESULT" = success')); diff --git a/scripts/ci-paths.mjs b/scripts/ci-paths.mjs index 52a53dbc..1f4dc3af 100755 --- a/scripts/ci-paths.mjs +++ b/scripts/ci-paths.mjs @@ -18,6 +18,16 @@ const GROUP_PATTERNS = Object.freeze({ /^provenance\/omp-host-migration\.json$/u, /^scripts\/legacy-bridge-continuity(?:\.test)?\.mjs$/u, ], + cluster: [ + /^\.github\/workflows\/ci\.yml$/u, + /^\.woodpecker\.yml$/u, + /^cluster\//u, + /^deploy\/charts\/t4-cluster\//u, + /^e2e\/cluster-operator\.spec\.ts$/u, + /^packages\/cluster-(?:operator|server)\//u, + /^packages\/host-(?:service|wire)\/(?:src\/|package\.json$)/u, + /^scripts\/cluster-ci\//u, + ], tooling: [ /^\.github\//u, /^compat\//u, diff --git a/scripts/ci-paths.test.mjs b/scripts/ci-paths.test.mjs index 1a326c83..87c04c52 100644 --- a/scripts/ci-paths.test.mjs +++ b/scripts/ci-paths.test.mjs @@ -4,6 +4,7 @@ import { classifyCiPaths, formatGitHubOutputs } from "./ci-paths.mjs"; const none = { continuity: false, + cluster: false, tooling: false, android_debug: false, flutter: false, @@ -15,6 +16,7 @@ test("host runtime source runs host gates without unrelated platform builds", () assert.deepEqual(classifyCiPaths(["packages/host-service/src/rpc-child.ts"]), { ...none, continuity: true, + cluster: true, tooling: true, }); }); @@ -30,6 +32,13 @@ test("lifecycle harness and architecture docs run tooling only", () => { ); }); +test("cluster implementation changes run the cluster gate", () => { + assert.deepEqual(classifyCiPaths(["packages/cluster-operator/controllers/session_controller.go"]), { + ...none, + cluster: true, + }); +}); + test("Flutter changes run all Flutter legs", () => { assert.deepEqual(classifyCiPaths(["apps/flutter/lib/src/client/t4_client_controller.dart"]), { ...none, @@ -42,6 +51,7 @@ test("Flutter changes run all Flutter legs", () => { test("host wire changes run every dependent client and continuity gate", () => { assert.deepEqual(classifyCiPaths(["packages/host-wire/src/command.ts"]), { continuity: true, + cluster: true, tooling: true, android_debug: true, flutter: true, @@ -69,6 +79,7 @@ test("dependency graph changes conservatively run every leg", () => { for (const path of ["package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml"]) { assert.deepEqual(classifyCiPaths([path]), { continuity: true, + cluster: true, tooling: true, android_debug: true, flutter: true, @@ -81,6 +92,7 @@ test("dependency graph changes conservatively run every leg", () => { test("workflow changes run tooling on the PR and the full matrix after merge", () => { assert.deepEqual(classifyCiPaths([".github/workflows/ci.yml"]), { ...none, + cluster: true, tooling: true, }); }); @@ -89,6 +101,6 @@ test("paths are normalized and GitHub outputs are stable", () => { const result = classifyCiPaths(["./apps\\flutter\\pubspec.yaml", "./apps/flutter/pubspec.yaml"]); assert.equal( formatGitHubOutputs(result), - "continuity=false\ntooling=false\nandroid_debug=false\nflutter=true\nflutter_android=true\nflutter_apple=true\n", + "continuity=false\ncluster=false\ntooling=false\nandroid_debug=false\nflutter=true\nflutter_android=true\nflutter_apple=true\n", ); }); diff --git a/scripts/cluster-ci/assemble-image-manifest.mjs b/scripts/cluster-ci/assemble-image-manifest.mjs new file mode 100644 index 00000000..dafca81d --- /dev/null +++ b/scripts/cluster-ci/assemble-image-manifest.mjs @@ -0,0 +1,332 @@ +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + AUTHORIZED_CI_MIRROR, + CANONICAL_BUILD_SOURCE_REPOSITORY, + IMAGE_COMPONENTS, + createFileEvidence, + validateImagePublicationManifest, +} from "./proof-contract.mjs"; + +const repoRoot = resolve(import.meta.dirname, "../.."); +const artifactDirectory = resolve(repoRoot, "artifacts/cluster-proof/images"); +const outputPath = resolve(repoRoot, "artifacts/cluster-proof/image-publication.json"); +const CANONICAL_BUILD_SOURCE_URL = `https://github.com/${CANONICAL_BUILD_SOURCE_REPOSITORY}`; +const HARBOR_REGISTRY = "harbor.tailb18de3.ts.net"; +const QUARANTINE_PREFIX = "quarantine"; +const suffixes = { + controller: "t4-cluster-operator", + "cluster-server": "t4-cluster-server", + "session-runtime": "t4-session-runtime", +}; + +function requiredEnvironment(name, environment = process.env) { + const value = environment[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +function woodpeckerIdentity(environment = process.env) { + const url = requiredEnvironment("CI_PIPELINE_URL", environment); + const parsedUrl = new URL(url); + const match = parsedUrl.pathname.match(/\/repos\/([1-9][0-9]*)\/pipeline\/([1-9][0-9]*)\/?$/u); + const pipelineNumber = Number(requiredEnvironment("CI_PIPELINE_NUMBER", environment)); + if ( + parsedUrl.origin !== "https://woodpecker-ci-dev.tailb18de3.ts.net" || + parsedUrl.username || + parsedUrl.password || + parsedUrl.search || + parsedUrl.hash || + !match || + !Number.isSafeInteger(pipelineNumber) || + pipelineNumber <= 0 + ) { + throw new Error("Woodpecker pipeline URL/number identity is invalid"); + } + return { + repositoryId: Number(match[1]), + pipelineId: Number(match[2]), + pipelineNumber, + url, + }; +} + +async function json(path, label) { + let value; + try { + value = JSON.parse(await readFile(path, "utf8")); + } catch (error) { + throw new Error(`${label} is not valid JSON`, { cause: error }); + } + return value; +} + +function exactImagePurl(locator, repository, digest) { + if (typeof locator !== "string" || locator.length > 2048) return false; + let decoded; + try { + decoded = decodeURIComponent(locator); + } catch { + return false; + } + const queryIndex = decoded.indexOf("?"); + const identity = queryIndex === -1 ? decoded : decoded.slice(0, queryIndex); + if (!identity.startsWith("pkg:oci/") || !identity.endsWith(`@${digest}`)) return false; + const parameters = new URLSearchParams(queryIndex === -1 ? "" : decoded.slice(queryIndex + 1)); + return parameters.get("repository_url") === repository; +} + +export function verifySpdx(sbom, { repository, digest, reference }) { + if ( + !sbom || + typeof sbom !== "object" || + sbom.spdxVersion !== "SPDX-2.3" || + sbom.dataLicense !== "CC0-1.0" || + sbom.SPDXID !== "SPDXRef-DOCUMENT" || + sbom.name !== reference || + typeof sbom.documentNamespace !== "string" || + !sbom.documentNamespace.includes(digest.slice("sha256:".length)) || + !Array.isArray(sbom.documentDescribes) || + sbom.documentDescribes.length !== 1 || + !Array.isArray(sbom.packages) || + sbom.packages.length < 1 || + sbom.packages.length > 100_000 + ) { + throw new Error("SPDX document identity is not bound to the scanned image"); + } + const imagePackage = sbom.packages.find(({ SPDXID }) => SPDXID === sbom.documentDescribes[0]); + if ( + !imagePackage || + typeof imagePackage.name !== "string" || + imagePackage.name !== repository.slice(repository.lastIndexOf("/") + 1) || + !Array.isArray(imagePackage.externalRefs) || + !imagePackage.externalRefs.some( + (externalRef) => + externalRef?.referenceCategory === "PACKAGE-MANAGER" && + externalRef?.referenceType === "purl" && + exactImagePurl(externalRef.referenceLocator, repository, digest), + ) + ) { + throw new Error("SPDX described package/external reference does not bind the image repository and digest"); + } +} + +export function vulnerabilityCounts(report, { repository, digest, reference }) { + if ( + !report || + typeof report !== "object" || + report.ArtifactName !== reference || + report.ArtifactType !== "container_image" || + !report.Metadata || + !Array.isArray(report.Metadata.RepoDigests) || + !report.Metadata.RepoDigests.includes(`${repository}@${digest}`) || + typeof report.Metadata.ImageID !== "string" || + !/^sha256:[0-9a-f]{64}$/u.test(report.Metadata.ImageID) || + !Array.isArray(report.Results) || + report.Results.length < 1 || + report.Results.length > 4096 + ) { + throw new Error("Trivy report artifact/results identity is malformed or unbound"); + } + const counts = { critical: 0, high: 0 }; + for (const result of report.Results) { + if ( + typeof result?.Target !== "string" || + result.Target.length < 1 || + result.Target.length > 2048 || + typeof result.Class !== "string" || + typeof result.Type !== "string" || + (result.Vulnerabilities !== undefined && !Array.isArray(result.Vulnerabilities)) + ) { + throw new Error("Trivy result entry is malformed"); + } + for (const vulnerability of result.Vulnerabilities ?? []) { + if (vulnerability?.Severity === "CRITICAL") counts.critical += 1; + else if (vulnerability?.Severity === "HIGH") counts.high += 1; + } + } + if (counts.critical !== 0 || counts.high !== 0) { + throw new Error(`Trivy found ${counts.critical} critical and ${counts.high} high vulnerabilities`); + } + return counts; +} + +function boundedStrings(value, depth = 0, output = []) { + if (depth > 12 || output.length > 4096) throw new Error("provenance exceeded its structural bound"); + if (typeof value === "string") { + if (value.length > 4096) throw new Error("provenance string exceeded its bound"); + output.push(value); + } else if (Array.isArray(value)) { + value.forEach((item) => boundedStrings(item, depth + 1, output)); + } else if (value && typeof value === "object") { + Object.values(value).forEach((item) => boundedStrings(item, depth + 1, output)); + } + return output; +} + +function trustedSourceMaterial(material, commit) { + if (!material || typeof material !== "object" || typeof material.uri !== "string") return false; + let source; + try { + source = new URL(material.uri.replace(/^git\+/u, "")); + } catch { + return false; + } + return ( + source.protocol === "https:" && + source.hostname === "github.com" && + source.pathname === `/${CANONICAL_BUILD_SOURCE_REPOSITORY}.git` && + source.hash === `#${commit}` && + material.digest?.sha1 === commit + ); +} + +export function verifyProvenance(jsonLines, { repository, digest, commit }) { + const expectedDigest = digest.slice("sha256:".length); + const lines = jsonLines.split("\n").filter(Boolean); + if (lines.length < 1 || lines.length > 32) throw new Error("provenance attestation count is invalid"); + const statements = []; + for (const line of lines) { + let envelope; + let statement; + try { + envelope = JSON.parse(line); + statement = JSON.parse(Buffer.from(envelope.payload, "base64").toString("utf8")); + } catch (error) { + throw new Error("provenance attestation is not valid DSSE JSON", { cause: error }); + } + if (envelope?.payloadType !== "application/vnd.in-toto+json" || typeof envelope.payload !== "string") { + throw new Error("provenance attestation is not an in-toto envelope"); + } + statements.push(statement); + } + const provenance = statements.find((statement) => { + const predicate = statement?.predicate; + const sourceMaterial = predicate?.materials?.find((material) => trustedSourceMaterial(material, commit)); + const baseMaterial = predicate?.materials?.some( + (material) => + typeof material?.uri === "string" && + material.uri.startsWith("pkg:docker/") && + /^[0-9a-f]{64}$/u.test(material.digest?.sha256 ?? ""), + ); + const invocationStrings = boundedStrings(predicate?.invocation?.parameters ?? {}); + return ( + statement?._type === "https://in-toto.io/Statement/v0.1" && + typeof statement.predicateType === "string" && + predicate?.builder?.id === "https://mobyproject.org/buildkit@v1" && + statement.predicateType.startsWith("https://slsa.dev/provenance/") && + predicate?.buildType === "https://mobyproject.org/buildkit@v1" && + statement.subject?.some( + (subject) => subject?.name === repository && subject?.digest?.sha256 === expectedDigest, + ) && + sourceMaterial && + baseMaterial && + invocationStrings.includes(commit) && + invocationStrings.includes(CANONICAL_BUILD_SOURCE_URL) + ); + }); + if (!provenance) { + throw new Error("BuildKit provenance does not bind subject, trusted source repository, CI commit, and materials"); + } +} + +export function provenanceVerificationMode(environment = process.env) { + const identity = environment.T4_COSIGN_CERTIFICATE_IDENTITY?.trim() ?? ""; + const issuer = environment.T4_COSIGN_CERTIFICATE_OIDC_ISSUER?.trim() ?? ""; + if (Boolean(identity) !== Boolean(issuer)) { + throw new Error("cosign certificate identity and OIDC issuer must be configured together"); + } + return identity ? { mode: "cosign-keyless", signatureVerified: true } : { mode: "buildkit-content", signatureVerified: false }; +} + +function validateProvenanceVerification(value, expected) { + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + Object.keys(value).sort().join(",") !== "mode,signatureVerified" || + value.mode !== expected.mode || + value.signatureVerified !== expected.signatureVerified + ) { + throw new Error("provenance signer verification record is missing or not truthful"); + } + return value; +} + +async function imageEntry(component, commit, registry, project, expectedVerification) { + const digest = (await readFile(resolve(artifactDirectory, `${component}.digest`), "utf8")).trim(); + const repository = `${registry}/${project}/${suffixes[component]}`; + const evidenceRepository = `${registry}/${project}/${QUARANTINE_PREFIX}/${suffixes[component]}`; + const evidenceReference = `${evidenceRepository}@${digest}`; + const sbomPath = resolve(artifactDirectory, `${component}.spdx.json`); + const provenancePath = resolve(artifactDirectory, `${component}.provenance.jsonl`); + const provenanceVerificationPath = resolve(artifactDirectory, `${component}.provenance-verification.json`); + const vulnerabilityPath = resolve(artifactDirectory, `${component}.trivy.json`); + verifySpdx(await json(sbomPath, `${component} SBOM`), { + repository: evidenceRepository, + digest, + reference: evidenceReference, + }); + verifyProvenance(await readFile(provenancePath, "utf8"), { + repository: evidenceRepository, + digest, + commit, + }); + const verification = validateProvenanceVerification( + await json(provenanceVerificationPath, `${component} provenance verification`), + expectedVerification, + ); + const counts = vulnerabilityCounts(await json(vulnerabilityPath, `${component} vulnerability report`), { + repository: evidenceRepository, + digest, + reference: evidenceReference, + }); + return { + component, + repository, + tag: commit, + digest, + reference: `${repository}@${digest}`, + sbom: await createFileEvidence(sbomPath, { artifactRoot: repoRoot }), + provenance: { ...(await createFileEvidence(provenancePath, { artifactRoot: repoRoot })), ...verification }, + vulnerability: { + ...(await createFileEvidence(vulnerabilityPath, { artifactRoot: repoRoot })), + scanner: "trivy", + ...counts, + }, + }; +} + +export async function assembleImagePublicationManifest(environment = process.env) { + const commit = requiredEnvironment("CI_COMMIT_SHA", environment); + const ciRepository = requiredEnvironment("CI_REPO", environment); + const registry = requiredEnvironment("HARBOR_REGISTRY", environment).replace(/\/$/u, ""); + const project = requiredEnvironment("HARBOR_PROJECT", environment).replace(/^\/+|\/+$/gu, ""); + if (ciRepository !== AUTHORIZED_CI_MIRROR) throw new Error("CI_REPO is not the authorized CI mirror"); + if (registry !== HARBOR_REGISTRY) throw new Error("HARBOR_REGISTRY must be the exact HTTPS tailnet Harbor host"); + const provenanceVerification = provenanceVerificationMode(environment); + const manifest = { + schemaVersion: "t4-cluster-images/1", + source: { + repository: CANONICAL_BUILD_SOURCE_REPOSITORY, + commit, + woodpecker: { repository: ciRepository, ...woodpeckerIdentity(environment) }, + }, + images: await Promise.all( + IMAGE_COMPONENTS.map((component) => imageEntry(component, commit, registry, project, provenanceVerification)), + ), + }; + return validateImagePublicationManifest(manifest); +} + +const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)); +if (isMain) { + const manifest = await assembleImagePublicationManifest(); + await mkdir(resolve(repoRoot, "artifacts/cluster-proof"), { recursive: true }); + const temporaryPath = `${outputPath}.tmp`; + await writeFile(temporaryPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 }); + await rename(temporaryPath, outputPath); + console.log(`Wrote ${outputPath}`); +} diff --git a/scripts/cluster-ci/assemble-proof.mjs b/scripts/cluster-ci/assemble-proof.mjs new file mode 100644 index 00000000..3d48c1a5 --- /dev/null +++ b/scripts/cluster-ci/assemble-proof.mjs @@ -0,0 +1,318 @@ +import { mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + AUTHORIZED_CI_MIRROR, + CANONICAL_BUILD_SOURCE_REPOSITORY, + OBSERVATION_SYSTEMS, + PROOF_SCENARIOS, + createFileEvidence, + redactFrame, + validateImagePublicationManifest, + validateProofManifest, +} from "./proof-contract.mjs"; + +const repoRoot = resolve(import.meta.dirname, "../.."); +const proofRoot = resolve(repoRoot, "artifacts/cluster-proof"); +const MAX_LOCAL_ARTIFACTS = 32; +const SCENARIO_ASSERTION = /^[a-z0-9][a-z0-9._-]{0,127}$/u; +const CONTRACT_SCENARIOS = new Set([ + "wire-reconnect-idempotency", + "gui-auth-isolation", + "desktop-viewport", + "mobile-viewport", +]); +const WOODPECKER_ORIGIN = "https://woodpecker-ci-dev.tailb18de3.ts.net"; +const OBSERVATION_ORIGINS = Object.freeze({ + prometheus: "https://interview-responder-prometheus.tailb18de3.ts.net", + loki: "https://interview-responder-loki.tailb18de3.ts.net", + grafana: "https://grafana.tailb18de3.ts.net", +}); + + +function requiredEnvironment(name) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +function woodpeckerIdentity() { + const value = requiredEnvironment("CI_PIPELINE_URL"); + const url = new URL(value); + const match = url.pathname.match(/\/repos\/([1-9][0-9]*)\/pipeline\/([1-9][0-9]*)\/?$/u); + const pipelineNumber = Number(requiredEnvironment("CI_PIPELINE_NUMBER")); + if ( + url.origin !== WOODPECKER_ORIGIN || + url.username || + url.password || + url.search || + url.hash || + !match || + !Number.isSafeInteger(pipelineNumber) || + pipelineNumber <= 0 + ) { + throw new Error("Woodpecker pipeline URL/number identity is invalid"); + } + return { + repositoryId: Number(match[1]), + pipelineId: Number(match[2]), + pipelineNumber, + url: value, + }; +} + + +function exactKeys(value, keys, label) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`); + const expected = new Set(keys); + for (const key of Object.keys(value)) { + if (!expected.has(key)) throw new Error(`${label} has unexpected field ${key}`); + } + for (const key of keys) { + if (!(key in value)) throw new Error(`${label} is missing ${key}`); + } +} + +function utcTimestamp(value, label) { + if ( + typeof value !== "string" || + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/u.test(value) || + !Number.isFinite(Date.parse(value)) + ) { + throw new Error(`${label} must be a UTC RFC 3339 timestamp`); + } + return value; +} + +async function scenarioEntries() { + const directory = resolve(proofRoot, "scenarios"); + const entries = []; + for (const id of PROOF_SCENARIOS) { + const path = resolve(directory, `${id}.json`); + const record = JSON.parse(await readFile(path, "utf8")); + exactKeys(record, ["schemaVersion", "id", "status", "observedAt", "assertions"], `scenario ${id}`); + if ( + record.schemaVersion !== "t4-cluster-scenario/1" || + record.id !== id || + record.status !== "passed" || + !Array.isArray(record.assertions) || + record.assertions.length < 1 || + record.assertions.length > 64 || + new Set(record.assertions).size !== record.assertions.length || + record.assertions.some((assertion) => !SCENARIO_ASSERTION.test(assertion)) + ) { + throw new Error(`scenario ${id} did not contain an exact passing contract result`); + } + entries.push({ + id, + status: "passed", + evidenceType: CONTRACT_SCENARIOS.has(id) ? "contract" : "live", + observedAt: utcTimestamp(record.observedAt, `scenario ${id}.observedAt`), + assertions: record.assertions, + evidence: [await createFileEvidence(path, { artifactRoot: repoRoot })], + }); + } + return entries; +} + +function safeSummary(value, label, depth = 0) { + if (depth > 8) throw new Error(`${label} exceeded its depth bound`); + if (value === null || typeof value === "boolean" || typeof value === "number") return; + if (typeof value === "string") { + if (value.length > 2048) throw new Error(`${label} contained an oversized string`); + return; + } + if (Array.isArray(value)) { + if (value.length > 128) throw new Error(`${label} exceeded its item bound`); + value.forEach((item, index) => safeSummary(item, `${label}[${index}]`, depth + 1)); + return; + } + if (!value || typeof value !== "object") throw new Error(`${label} contained an unsupported value`); + const entries = Object.entries(value); + if (entries.length > 128) throw new Error(`${label} exceeded its field bound`); + for (const [key, item] of entries) { + if (/api.?key|private.?key|authorization|auth|bearer|cookie|credential|password|prompt|secret|token|transcript/iu.test(key)) { + throw new Error(`${label} contained sensitive field ${key}`); + } + safeSummary(item, `${label}.${key}`, depth + 1); + } +} + +async function observationEntries() { + const directory = resolve(proofRoot, "observations"); + await mkdir(directory, { recursive: true }); + const commit = requiredEnvironment("CI_COMMIT_SHA"); + const entries = []; + + const woodpecker = woodpeckerIdentity(); + const woodpeckerObservedAt = new Date().toISOString(); + const woodpeckerPath = resolve(directory, "woodpecker.json"); + await writeFile( + woodpeckerPath, + `${JSON.stringify({ schemaVersion: "t4-woodpecker-observation/1", observedAt: woodpeckerObservedAt, ...woodpecker }, null, 2)}\n`, + { mode: 0o600 }, + ); + entries.push({ + system: "woodpecker", + observedAt: woodpeckerObservedAt, + url: WOODPECKER_ORIGIN, + ids: [String(woodpecker.repositoryId), String(woodpecker.pipelineId), String(woodpecker.pipelineNumber)], + evidence: await createFileEvidence(woodpeckerPath, { artifactRoot: repoRoot }), + }); + + const kubernetesPath = resolve(directory, "kubernetes.json"); + const kubernetes = JSON.parse(await readFile(kubernetesPath, "utf8")); + if ( + kubernetes?.schemaVersion !== "t4-cluster-readonly-snapshot/1" || + !Array.isArray(kubernetes.deployments) || + kubernetes.deployments.length < 2 + ) { + throw new Error("Kubernetes observation does not contain exact read-only cluster evidence"); + } + entries.push({ + system: "kubernetes", + observedAt: utcTimestamp(kubernetes.observedAt, "kubernetes observedAt"), + url: null, + ids: kubernetes.deployments.map(({ name }) => name), + evidence: await createFileEvidence(kubernetesPath, { artifactRoot: repoRoot }), + }); + + for (const system of ["prometheus", "loki", "grafana"]) { + const path = resolve(directory, `${system}.json`); + const payload = JSON.parse(await readFile(path, "utf8")); + exactKeys(payload, ["schemaVersion", "sourceCommit", "system", "observedAt", "redacted", "ids", "summary"], `${system} evidence`); + if ( + payload.schemaVersion !== "t4-cluster-observation/1" || + payload.sourceCommit !== commit || + payload.system !== system || + payload.redacted !== true || + !Array.isArray(payload.ids) || + payload.ids.length < 1 || + payload.ids.length > 16 + ) { + throw new Error(`${system} evidence is not an exact source-bound redacted observation`); + } + utcTimestamp(payload.observedAt, `${system} evidence observedAt`); + safeSummary(payload.summary, `${system} evidence summary`); + entries.push({ + system, + observedAt: payload.observedAt, + url: OBSERVATION_ORIGINS[system], + ids: payload.ids, + evidence: await createFileEvidence(path, { artifactRoot: repoRoot }), + }); + } + if (entries.length !== OBSERVATION_SYSTEMS.length) { + throw new Error("observation coverage does not match the truthfully available systems"); + } + return entries; +} + +async function localArtifacts(kind, extensions) { + const directory = resolve(proofRoot, kind); + const names = (await readdir(directory)).filter((name) => extensions.some((extension) => name.endsWith(extension))).sort(); + if (names.length < 1 || names.length > MAX_LOCAL_ARTIFACTS) { + throw new Error(`${kind} artifact count is outside its bound`); + } + const results = []; + for (const name of names) { + const path = resolve(directory, name); + if (kind === "frames") { + const parsed = JSON.parse(await readFile(path, "utf8")); + const frames = Array.isArray(parsed) ? parsed : [parsed]; + if (frames.length < 1 || frames.length > 256) throw new Error(`${name} frame count is outside its bound`); + for (const frame of frames) { + if (JSON.stringify(redactFrame(frame)) !== JSON.stringify(frame)) { + throw new Error(`${name} contains unredacted authority-sensitive frame state`); + } + } + } + const viewport = /mobile/iu.test(name) ? "mobile" : /desktop/iu.test(name) ? "desktop" : undefined; + results.push({ + ...(await createFileEvidence(path, { artifactRoot: repoRoot })), + redacted: true, + evidenceType: kind === "frames" ? "live" : "contract", + ...(viewport ? { viewport } : {}), + }); + } + return results; +} + +async function verifyLiveImageDigests(images) { + const snapshot = JSON.parse( + await readFile(resolve(proofRoot, "observations/kubernetes.json"), "utf8"), + ); + if ( + snapshot?.schemaVersion !== "t4-cluster-readonly-snapshot/1" || + !Array.isArray(snapshot.images) || + snapshot.images.length > 64 + ) { + throw new Error("Kubernetes observation has no bounded live image identity"); + } + const identities = { + controller: { component: "controller", container: "controller" }, + "cluster-server": { component: "server", container: "server" }, + "session-runtime": { name: "t4-session-runtime", container: "session-runtime" }, + }; + for (const image of images) { + const identity = identities[image.component]; + const matches = snapshot.images.filter((observed) => + observed?.labels?.partOf === "t4-cluster" && + (identity.component ? observed.labels.component === identity.component : observed.labels.name === identity.name) && + observed.container === identity.container && + observed.phase === "Running" && + observed.ready === true && + observed.image === image.reference && + typeof observed.imageID === "string" && + observed.imageID.endsWith(`@${image.digest}`), + ); + if (matches.length < 1) { + throw new Error(`live Kubernetes pods do not run the exact ready ${image.component} reference ${image.reference}`); + } + } +} + +export async function assembleProofManifest() { + const imageManifest = validateImagePublicationManifest( + JSON.parse(await readFile(resolve(proofRoot, "image-publication.json"), "utf8")), + ); + if (imageManifest.source.commit !== requiredEnvironment("CI_COMMIT_SHA")) { + throw new Error("image publication manifest does not match this source commit"); + } + await verifyLiveImageDigests(imageManifest.images); + const ciRepository = requiredEnvironment("CI_REPO"); + if (ciRepository !== AUTHORIZED_CI_MIRROR) throw new Error("CI_REPO is not the authorized CI mirror"); + const source = { + repository: CANONICAL_BUILD_SOURCE_REPOSITORY, + commit: requiredEnvironment("CI_COMMIT_SHA"), + woodpecker: { repository: ciRepository, ...woodpeckerIdentity() }, + }; + const manifest = { + schemaVersion: "t4-cluster-proof/1", + source, + images: imageManifest.images, + scenarios: await scenarioEntries(), + observations: await observationEntries(), + artifacts: { + frames: await localArtifacts("frames", [".json"]), + screenshots: await localArtifacts("screenshots", [".png", ".webp"]), + }, + }; + return validateProofManifest(manifest); +} + +const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)); +if (isMain) { + try { + const manifest = await assembleProofManifest(); + const outputPath = resolve(proofRoot, "manifest.json"); + const temporaryPath = `${outputPath}.tmp`; + await writeFile(temporaryPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 }); + await rename(temporaryPath, outputPath); + console.log(`Wrote ${outputPath}`); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/cluster-ci/build-image.sh b/scripts/cluster-ci/build-image.sh new file mode 100755 index 00000000..c143ef0c --- /dev/null +++ b/scripts/cluster-ci/build-image.sh @@ -0,0 +1,83 @@ +#!/bin/sh +set -eu + +umask 077 +canonical_build_source_repository=usr-bin-roygbiv/t4-code +authorized_ci_mirror=z-peterson/t4-code + +component=${1:-} +repository_suffix=${2:-} +dockerfile=${3:-} + +case "$component:$repository_suffix:$dockerfile" in + controller:t4-cluster-operator:cluster/images/controller/Dockerfile | \ + cluster-server:t4-cluster-server:cluster/images/cluster-server/Dockerfile | \ + session-runtime:t4-session-runtime:cluster/images/session-runtime/Dockerfile) + ;; + *) + echo "component, repository suffix, and Dockerfile do not match the fixed T4 image contract" >&2 + exit 64 + ;; +esac + +case "${CI_COMMIT_SHA:-}" in + [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) + ;; + *) + echo "CI_COMMIT_SHA must be an exact lowercase 40-character SHA" >&2 + exit 64 + ;; +esac + +: "${BUILDKIT_ADDR:?BUILDKIT_ADDR is required}" +: "${HARBOR_REGISTRY:?HARBOR_REGISTRY is required}" +: "${HARBOR_PROJECT:?HARBOR_PROJECT is required}" +if [ "$HARBOR_REGISTRY" != "harbor.tailb18de3.ts.net" ]; then + echo "HARBOR_REGISTRY must be the exact HTTPS tailnet Harbor host" >&2 + exit 64 +fi +: "${CI_REPO:?CI_REPO is required}" +if [ "$CI_REPO" != "$authorized_ci_mirror" ]; then + echo "CI_REPO must identify the authorized Woodpecker mirror" >&2 + exit 64 +fi +auth_dir=${T4_REGISTRY_AUTH_DIR:-${CI_WORKSPACE:-$PWD}/.cluster-ci/registry-auth} +test -r "$auth_dir/config.json" +export DOCKER_CONFIG="$auth_dir" + +test -f "$dockerfile" +artifact_dir="artifacts/cluster-proof/images" +mkdir -p "$artifact_dir" +chmod 1777 "$artifact_dir" +metadata="$artifact_dir/$component.buildkit.json" +digest_file="$artifact_dir/$component.digest" + +repository="$HARBOR_REGISTRY/$HARBOR_PROJECT/quarantine/$repository_suffix" +reference="$repository:$CI_COMMIT_SHA" +source_context="https://github.com/$canonical_build_source_repository.git#$CI_COMMIT_SHA" + +buildctl --addr "$BUILDKIT_ADDR" build \ + --frontend dockerfile.v0 \ + --opt "context=$source_context" \ + --opt "filename=$dockerfile" \ + --opt platform=linux/amd64,linux/arm64 \ + --opt "build-arg:SOURCE_COMMIT=$CI_COMMIT_SHA" \ + --opt "build-arg:SOURCE_REPOSITORY=https://github.com/$canonical_build_source_repository" \ + --opt "label:org.opencontainers.image.source=https://github.com/$canonical_build_source_repository" \ + --opt "label:org.opencontainers.image.revision=$CI_COMMIT_SHA" \ + --output "type=image,name=$reference,push=true,compression=zstd,force-compression=true,oci-mediatypes=true" \ + --attest type=sbom \ + --attest type=provenance,mode=max \ + --metadata-file "$metadata" + +digest=$(sed -n 's/.*"containerimage\.digest"[[:space:]]*:[[:space:]]*"\(sha256:[0-9a-f]\{64\}\)".*/\1/p' "$metadata") +case "$digest" in + sha256:????????????????????????????????????????????????????????????????) ;; + *) + echo "BuildKit did not return an immutable image digest" >&2 + exit 65 + ;; +esac +printf '%s\n' "$digest" > "$digest_file" +printf '%s@%s\n' "$repository" "$digest" > "$artifact_dir/$component.reference" +chmod 0444 "$metadata" "$digest_file" "$artifact_dir/$component.reference" diff --git a/scripts/cluster-ci/capture-image-evidence.sh b/scripts/cluster-ci/capture-image-evidence.sh new file mode 100755 index 00000000..1f6c2031 --- /dev/null +++ b/scripts/cluster-ci/capture-image-evidence.sh @@ -0,0 +1,82 @@ +#!/bin/sh +set -eu + +umask 077 + +mode=${1:-} +component=${2:-} +repository_suffix=${3:-} +case "$component:$repository_suffix" in + controller:t4-cluster-operator | cluster-server:t4-cluster-server | session-runtime:t4-session-runtime) ;; + *) + echo "component and repository suffix do not match the fixed T4 image contract" >&2 + exit 64 + ;; +esac +case "$mode" in + sbom | vulnerability | provenance) ;; + *) + echo "evidence mode must be sbom, vulnerability, or provenance" >&2 + exit 64 + ;; +esac + +: "${CI_COMMIT_SHA:?CI_COMMIT_SHA is required}" +: "${HARBOR_REGISTRY:?HARBOR_REGISTRY is required}" +: "${HARBOR_PROJECT:?HARBOR_PROJECT is required}" +if [ "$HARBOR_REGISTRY" != "harbor.tailb18de3.ts.net" ]; then + echo "HARBOR_REGISTRY must be the exact HTTPS tailnet Harbor host" >&2 + exit 64 +fi +auth_dir=${T4_REGISTRY_AUTH_DIR:-${CI_WORKSPACE:-$PWD}/.cluster-ci/registry-auth} +test -r "$auth_dir/config.json" +export DOCKER_CONFIG="$auth_dir" + +artifact_dir="artifacts/cluster-proof/images" +digest=$(cat "$artifact_dir/$component.digest") +case "$digest" in + sha256:????????????????????????????????????????????????????????????????) ;; + *) + echo "image digest artifact is malformed" >&2 + exit 65 + ;; +esac +reference="$HARBOR_REGISTRY/$HARBOR_PROJECT/quarantine/$repository_suffix@$digest" + +case "$mode" in + sbom) + syft "registry:$reference" -o "spdx-json=$artifact_dir/$component.spdx.json" + test -s "$artifact_dir/$component.spdx.json" + ;; + vulnerability) + trivy image \ + --format json \ + --output "$artifact_dir/$component.trivy.json" \ + --scanners vuln \ + --severity HIGH,CRITICAL \ + --exit-code 1 \ + "$reference" + test -s "$artifact_dir/$component.trivy.json" + ;; + provenance) + identity=${T4_COSIGN_CERTIFICATE_IDENTITY:-} + issuer=${T4_COSIGN_CERTIFICATE_OIDC_ISSUER:-} + if [ -n "$identity" ] && [ -n "$issuer" ]; then + cosign verify-attestation \ + --type slsaprovenance \ + --certificate-identity "$identity" \ + --certificate-oidc-issuer "$issuer" \ + "$reference" > "$artifact_dir/$component.provenance.jsonl" + printf '%s\n' '{"mode":"cosign-keyless","signatureVerified":true}' > "$artifact_dir/$component.provenance-verification.json" + elif [ -z "$identity" ] && [ -z "$issuer" ]; then + cosign download attestation "$reference" > "$artifact_dir/$component.provenance.jsonl" + printf '%s\n' '{"mode":"buildkit-content","signatureVerified":false}' > "$artifact_dir/$component.provenance-verification.json" + else + echo "cosign certificate identity and OIDC issuer must be configured together" >&2 + exit 64 + fi + unset identity issuer + test -s "$artifact_dir/$component.provenance.jsonl" + test -s "$artifact_dir/$component.provenance-verification.json" + ;; +esac diff --git a/scripts/cluster-ci/capture-redacted-frames.mjs b/scripts/cluster-ci/capture-redacted-frames.mjs new file mode 100644 index 00000000..bce7b872 --- /dev/null +++ b/scripts/cluster-ci/capture-redacted-frames.mjs @@ -0,0 +1,270 @@ +import { mkdir, rename, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import WebSocket from "ws"; + +import { redactFrame } from "./proof-contract.mjs"; + +const repoRoot = resolve(import.meta.dirname, "../.."); +const outputPath = resolve(repoRoot, "artifacts/cluster-proof/frames/live-operator-state.json"); +const MAX_INBOUND_BYTES = 1024 * 1024; +const MAX_FRAMES = 32; +const SAFE_AGENT_ID_KEYS = ["agentId", "rootAgentId", "activeAgentId", "parentAgentId"]; +const CLUSTER_HOST = "t4-dev.tailb18de3.ts.net"; +const EXPECTED_OMP_VERSION = "17.0.5"; +const EXPECTED_OMP_BUILD = "8476f4451ed95c5d5401785d279a93d3c659fac4"; +const REQUIRED_CAPABILITIES = Object.freeze([ + "sessions.read", + "ci.trigger", + "preview.read", + "preview.control", + "preview.input", +]); + +export function proofHelloFrame() { + return { + v: "omp-app/1", + type: "hello", + protocol: { min: "omp-app/1", max: "omp-app/1" }, + client: { name: "t4-cluster-proof", version: "1", build: "1", platform: "woodpecker" }, + requestedFeatures: ["cluster.operator", "resume", "preview.control"], + capabilities: { client: [...REQUIRED_CAPABILITIES] }, + savedCursors: [], + }; +} + +function requiredEnvironment(name) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +export function clusterWebSocketUrl(baseUrl) { + const url = new URL(baseUrl); + if ( + url.protocol !== "https:" || + url.hostname !== CLUSTER_HOST || + url.port || + url.pathname !== "/" || + url.username || + url.password || + url.search || + url.hash + ) { + throw new Error(`T4_CLUSTER_BASE_URL must be the credential-free HTTPS origin https://${CLUSTER_HOST}/`); + } + url.protocol = "wss:"; + url.pathname = "/v1/ws"; + return url; +} + +function boundedId(value, label) { + if (typeof value !== "string" || value.length < 1 || value.length > 253 || /\p{Cc}/u.test(value)) { + throw new Error(`${label} is not a bounded identifier`); + } + return value; +} + +function cursor(value, label) { + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + typeof value.epoch !== "string" || + value.epoch.length < 1 || + value.epoch.length > 253 || + !Number.isSafeInteger(value.seq) || + value.seq < 0 + ) { + throw new Error(`${label} is not a bounded cursor`); + } + return { epoch: value.epoch, seq: value.seq }; +} + +function agentIds(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) return {}; + const ids = {}; + const stack = [value]; + let visited = 0; + while (stack.length > 0) { + const current = stack.pop(); + if (!current || typeof current !== "object" || Array.isArray(current)) continue; + visited += 1; + if (visited > 256) throw new Error("session state exceeded its object bound"); + for (const [key, item] of Object.entries(current)) { + if (SAFE_AGENT_ID_KEYS.includes(key) && typeof item === "string" && !(key in ids)) { + ids[key] = boundedId(item, key); + } else if (item && typeof item === "object") { + stack.push(item); + } + } + } + return ids; +} + +function projectItems(items, kind) { + if (!Array.isArray(items) || items.length > 256) throw new Error(`${kind} list exceeded its bound`); + if (kind === "sessions") { + return items.map((item, index) => ({ + hostId: boundedId(item?.hostId, `sessions[${index}].hostId`), + sessionId: boundedId(item?.sessionId, `sessions[${index}].sessionId`), + revision: boundedId(item?.revision, `sessions[${index}].revision`), + ...(typeof item?.liveState?.cluster?.workspaceId === "string" + ? { workspaceId: boundedId(item.liveState.cluster.workspaceId, `sessions[${index}].workspaceId`) } + : {}), + ...agentIds(item?.liveState), + })); + } + return items.map((item, index) => ({ + hostId: boundedId(item?.hostId, `workspaces[${index}].hostId`), + workspaceId: boundedId(item?.workspaceId ?? item?.id, `workspaces[${index}].workspaceId`), + revision: boundedId(item?.revision, `workspaces[${index}].revision`), + })); +} + +function projectListResult(kind, hostId, value) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${kind} result is malformed`); + const items = value[kind]; + return { + v: "omp-app/1", + type: `${kind}.proof`, + hostId, + cursor: cursor(value.cursor, `${kind}.cursor`), + items: projectItems(items, kind), + }; +} + +async function capture() { + const baseUrl = requiredEnvironment("T4_CLUSTER_BASE_URL"); + const origin = new URL(baseUrl).origin; + const pipelineNumber = requiredEnvironment("CI_PIPELINE_NUMBER"); + const sessionRequest = `proof-session-list-${pipelineNumber}`; + const workspaceRequest = `proof-workspace-list-${pipelineNumber}`; + const socket = new WebSocket(clusterWebSocketUrl(baseUrl), { + headers: { Origin: origin }, + handshakeTimeout: 15_000, + maxPayload: MAX_INBOUND_BYTES, + perMessageDeflate: false, + }); + const frames = []; + let hostId; + let sessionResult; + let workspaceResult; + + const complete = new Promise((resolveComplete, rejectComplete) => { + const timeout = setTimeout(() => rejectComplete(new Error("timed out waiting for bounded cluster frames")), 20_000); + const finish = (callback, value) => { + clearTimeout(timeout); + callback(value); + }; + socket.once("error", (error) => finish(rejectComplete, error)); + socket.once("close", (code) => { + if (!sessionResult || !workspaceResult) finish(rejectComplete, new Error(`cluster socket closed early (${code})`)); + }); + socket.once("open", () => { + socket.send(JSON.stringify(proofHelloFrame())); + }); + socket.on("message", (data, isBinary) => { + if (isBinary || data.length > MAX_INBOUND_BYTES) return finish(rejectComplete, new Error("cluster frame was binary or oversized")); + let frame; + try { + frame = JSON.parse(data.toString("utf8")); + } catch { + return finish(rejectComplete, new Error("cluster frame was not JSON")); + } + if (!frame || typeof frame !== "object" || Array.isArray(frame) || frame.v !== "omp-app/1") { + return finish(rejectComplete, new Error("cluster frame was malformed")); + } + if (frame.type === "welcome") { + hostId = boundedId(frame.hostId, "welcome.hostId"); + if ( + frame.selectedProtocol !== "omp-app/1" || + frame.ompVersion !== EXPECTED_OMP_VERSION || + frame.ompBuild !== EXPECTED_OMP_BUILD || + !Array.isArray(frame.grantedFeatures) || + !frame.grantedFeatures.includes("cluster.operator") || + !frame.grantedFeatures.includes("preview.control") || + !Array.isArray(frame.grantedCapabilities) || + REQUIRED_CAPABILITIES.some((capability) => !frame.grantedCapabilities.includes(capability)) + ) { + return finish(rejectComplete, new Error("cluster operator sessions.read/CI/preview capability contract was not negotiated")); + } + frames.push({ + v: "omp-app/1", + type: "welcome.proof", + hostId, + epoch: boundedId(frame.epoch, "welcome.epoch"), + selectedProtocol: frame.selectedProtocol, + ompVersion: frame.ompVersion, + ompBuild: frame.ompBuild, + clusterOperator: true, + grantedCapabilities: REQUIRED_CAPABILITIES, + }); + for (const [requestId, command] of [ + [sessionRequest, "session.list"], + [workspaceRequest, "workspace.list"], + ]) { + socket.send( + JSON.stringify({ + v: "omp-app/1", + type: "command", + requestId, + commandId: requestId, + hostId, + command, + args: {}, + }), + ); + } + } else if (frame.type === "response" && frame.requestId === sessionRequest) { + if (frame.ok !== true) return finish(rejectComplete, new Error("session.list proof command failed")); + sessionResult = projectListResult("sessions", hostId, frame.result); + frames.push(sessionResult); + } else if (frame.type === "response" && frame.requestId === workspaceRequest) { + if (frame.ok !== true) return finish(rejectComplete, new Error("workspace.list proof command failed")); + workspaceResult = projectListResult("workspaces", hostId, frame.result); + frames.push(workspaceResult); + } + if (frames.length > MAX_FRAMES) return finish(rejectComplete, new Error("cluster proof frame count exceeded its bound")); + if (sessionResult && workspaceResult) finish(resolveComplete); + }); + }); + + try { + await complete; + } finally { + socket.close(1000, "proof complete"); + } + if (sessionResult.items.length < 1 || workspaceResult.items.length < 1) { + throw new Error("cluster proof requires at least one workspace and session revision"); + } + const artifact = redactFrame({ + schemaVersion: "t4-cluster-redacted-frames/1", + redacted: true, + capturedAt: new Date().toISOString(), + frames, + }); + await mkdir(resolve(proofRoot(), "frames"), { recursive: true }); + const temporaryPath = `${outputPath}.tmp`; + await writeFile(temporaryPath, `${JSON.stringify(artifact, null, 2)}\n`, { mode: 0o600 }); + await rename(temporaryPath, outputPath); + return artifact; +} + +function proofRoot() { + return resolve(repoRoot, "artifacts/cluster-proof"); +} + +const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)); +if (isMain) { + try { + const artifact = await capture(); + console.log(`Captured ${artifact.frames.length} bounded redacted cluster frames`); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} + +export { capture as captureRedactedFrameEvidence }; diff --git a/scripts/cluster-ci/cluster-ci-contract.test.mjs b/scripts/cluster-ci/cluster-ci-contract.test.mjs new file mode 100644 index 00000000..b53264d4 --- /dev/null +++ b/scripts/cluster-ci/cluster-ci-contract.test.mjs @@ -0,0 +1,800 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; + +import yaml from "js-yaml"; + +import { + IMAGE_COMPONENTS, + OBSERVATION_SYSTEMS, + PROOF_SCENARIOS, + createFileEvidence, + redactFrame, + validateProofManifest, +} from "./proof-contract.mjs"; +import { + collectReadOnlyClusterSnapshot, + validateClusterSnapshot, + validateDefaultOffRender, +} from "./readonly-cluster-proof.mjs"; +import { grafanaHealthSummary, lokiLogSummary, prometheusSample } from "./collect-readonly-cluster.mjs"; +import { HARBOR_REGISTRY_ALIASES, normalizeRegistryAuth } from "./normalize-registry-auth.mjs"; +import { provenanceVerificationMode, verifyProvenance, verifySpdx, vulnerabilityCounts } from "./assemble-image-manifest.mjs"; +import { clusterWebSocketUrl, proofHelloFrame } from "./capture-redacted-frames.mjs"; + +const COMMIT = "0123456789abcdef0123456789abcdef01234567"; +const DIGEST = `sha256:${"a".repeat(64)}`; +const FILE_SHA = "b".repeat(64); +const CANONICAL_BUILD_SOURCE_REPOSITORY = "usr-bin-roygbiv/t4-code"; +const AUTHORIZED_CI_MIRROR = "z-peterson/t4-code"; +const OBSERVED_AT = "2026-07-20T12:34:56.000Z"; +const repoRoot = resolve(import.meta.dirname, "../.."); +const OBSERVATION_HOSTS = { + woodpecker: "woodpecker-ci-dev.tailb18de3.ts.net", + prometheus: "interview-responder-prometheus.tailb18de3.ts.net", + loki: "interview-responder-loki.tailb18de3.ts.net", + grafana: "grafana.tailb18de3.ts.net", +}; +const CONTRACT_SCENARIOS = new Set(["wire-reconnect-idempotency", "gui-auth-isolation", "desktop-viewport", "mobile-viewport"]); +const CLUSTER_VALIDATION = { + now: Date.parse(OBSERVED_AT), + ciMapping: { repositoryId: "71", ref: "refs/heads/agent/t4-cluster-operator", commit: COMMIT }, +}; + +function fileEvidence(path) { + return { path: `artifacts/cluster-proof/${path}`, sha256: FILE_SHA }; +} + +function validProof() { + const suffixes = { + controller: "t4-cluster-operator", + "cluster-server": "t4-cluster-server", + "session-runtime": "t4-session-runtime", + }; + return { + schemaVersion: "t4-cluster-proof/1", + source: { + repository: "usr-bin-roygbiv/t4-code", + commit: COMMIT, + woodpecker: { + repository: "z-peterson/t4-code", + repositoryId: 71, + pipelineId: 401, + pipelineNumber: 99, + url: "https://woodpecker-ci-dev.tailb18de3.ts.net/repos/71/pipeline/99", + }, + }, + images: IMAGE_COMPONENTS.map((component) => { + const repository = `harbor.example.test/t4/${suffixes[component]}`; + return { + component, + repository, + tag: COMMIT, + digest: DIGEST, + reference: `${repository}@${DIGEST}`, + sbom: fileEvidence(`images/${component}.spdx.json`), + provenance: { ...fileEvidence(`images/${component}.provenance.json`), mode: "buildkit-content", signatureVerified: false }, + vulnerability: { + ...fileEvidence(`images/${component}.trivy.json`), + scanner: "trivy", + critical: 0, + high: 0, + }, + }; + }), + scenarios: PROOF_SCENARIOS.map((id) => ({ + id, + status: "passed", + evidenceType: CONTRACT_SCENARIOS.has(id) ? "contract" : "live", + observedAt: OBSERVED_AT, + assertions: [`${id}.observable-contract`], + evidence: [fileEvidence(`scenarios/${id}.json`)], + })), + observations: OBSERVATION_SYSTEMS.map((system, index) => ({ + system, + observedAt: OBSERVED_AT, + url: system === "kubernetes" ? null : `https://${OBSERVATION_HOSTS[system]}/`, + ids: [`${system}-${index + 1}`], + evidence: fileEvidence(`observations/${system}.json`), + })), + artifacts: { + frames: [ + { ...fileEvidence("frames/omp-app.json"), redacted: true, evidenceType: "live" }, + ], + screenshots: [ + { ...fileEvidence("screenshots/desktop.png"), redacted: true, evidenceType: "contract", viewport: "desktop" }, + { ...fileEvidence("screenshots/mobile.png"), redacted: true, evidenceType: "contract", viewport: "mobile" }, + ], + }, + }; +} + +function liveClusterResponses() { + const workloadImage = `harbor.tailb18de3.ts.net/linkedin-bot/workload@${DIGEST}`; + const labels = (component) => ({ + "app.kubernetes.io/name": "t4-cluster", + "app.kubernetes.io/part-of": "t4-cluster", + "app.kubernetes.io/component": component, + }); + const workloadPod = (name, component, container) => ({ + metadata: { name, labels: labels(component) }, + spec: { nodeName: "k3s-worker-01" }, + status: { + phase: "Running", + conditions: [{ type: "Ready", status: "True" }], + containerStatuses: [{ name: container, image: workloadImage, imageID: `containerd://${workloadImage}`, ready: true }], + }, + }); + return { + deployments: { + items: [ + { + metadata: { name: "t4-cluster-controller", generation: 4, labels: labels("controller") }, + spec: { + replicas: 2, + strategy: { type: "RollingUpdate", rollingUpdate: { maxUnavailable: 0 } }, + }, + status: { observedGeneration: 4, availableReplicas: 2 }, + }, + { + metadata: { name: "t4-cluster-server", generation: 6, labels: labels("server") }, + spec: { + replicas: 3, + strategy: { type: "RollingUpdate", rollingUpdate: { maxUnavailable: 0 } }, + }, + status: { observedGeneration: 6, availableReplicas: 3 }, + }, + ], + }, + lease: { + metadata: { name: "t4-cluster-operator.cluster.t4.dev" }, + spec: { holderIdentity: "t4-cluster-controller-7cbbc8-x7v2k", renewTime: OBSERVED_AT }, + }, + customresourcedefinitions: { + items: ["t4clusterhosts", "t4workspaces", "t4sessions"].map((plural) => ({ + metadata: { name: `${plural}.cluster.t4.dev` }, + spec: { + group: "cluster.t4.dev", + scope: "Namespaced", + names: { plural }, + versions: [{ name: "v1alpha1", served: true, storage: true }], + }, + })), + }, + t4clusterhosts: { + items: [{ metadata: { name: "development", generation: 2 }, status: { observedGeneration: 2 } }], + }, + t4workspaces: { + items: [ + { + metadata: { name: "proof-workspace", generation: 3 }, + spec: { retentionPolicy: "Retain" }, + status: { observedGeneration: 3, pvcName: "proof-workspace-data", phase: "Ready" }, + }, + ], + }, + t4sessions: { + items: [ + { + metadata: { name: "proof-session", generation: 5 }, + spec: { + workspaceRef: "proof-workspace", + ci: { repositoryId: "71", ref: "refs/heads/agent/t4-cluster-operator", commit: COMMIT }, + }, + status: { observedGeneration: 5, phase: "Running", podName: "t4-session-proof-session" }, + }, + ], + }, + persistentvolumeclaims: { + items: [ + { + metadata: { + name: "proof-workspace-data", + labels: { + "app.kubernetes.io/part-of": "t4-cluster", + "cluster.t4.dev/workspace": "proof-workspace", + }, + }, + spec: { accessModes: ["ReadWriteMany"], storageClassName: "t4-workspaces-rwx" }, + status: { phase: "Bound", capacity: { storage: "20Gi" } }, + }, + ], + }, + pods: { + items: [ + workloadPod("t4-cluster-controller-a", "controller", "controller"), + workloadPod("t4-cluster-server-a", "server", "server"), + workloadPod("t4-cluster-server-b", "server", "server"), + { + metadata: { + name: "t4-session-proof-session", + labels: { + "app.kubernetes.io/name": "t4-session-runtime", + "app.kubernetes.io/part-of": "t4-cluster", + "cluster.t4.dev/session": "t4-session-proof-session", + }, + }, + spec: { nodeName: "k3s-worker-01" }, + status: { + phase: "Running", + conditions: [{ type: "Ready", status: "True" }], + containerStatuses: [{ name: "session-runtime", image: workloadImage, imageID: `containerd://${workloadImage}`, ready: true }], + }, + }, + ], + }, + services: { + items: [ + { + metadata: { name: "t4-cluster-server", labels: labels("server") }, + spec: { ports: [{ name: "websocket", port: 8080 }] }, + }, + { + metadata: { name: "t4-cluster-metrics", labels: labels("server") }, + spec: { ports: [{ name: "metrics", port: 9090 }] }, + }, + ], + }, + }; +} + +test("Woodpecker keeps upstream gates and serializes bounded cluster publication", async () => { + const pipeline = yaml.load(await readFile(resolve(repoRoot, ".woodpecker.yml"), "utf8")); + assert.equal(typeof pipeline, "object"); + const steps = pipeline.steps; + const coreCommands = steps["upstream-core"].commands; + assert.deepEqual(coreCommands, [ + 'export PATH="$PWD/.ci:$PATH"', + "corepack enable", + "pnpm check:release && pnpm check:provenance && pnpm lint && pnpm --filter '!@t4-code/flutter' -r typecheck", + "VP_RUN_CONCURRENCY_LIMIT=1 pnpm --filter '!@t4-code/flutter' -r test", + "pnpm --filter '!@t4-code/flutter' -r build", + "pnpm exec playwright install --with-deps chromium", + "pnpm test:e2e", + "pnpm test:packaging", + ]); + const unfilteredSdkCommand = /(?:^|\s)pnpm(?:\s+-r)?\s+(?:check|typecheck|test|build)(?:\s|$)/u; + for (const command of coreCommands) { + assert.doesNotMatch( + command, + unfilteredSdkCommand, + `pipeline 38:64 reproduced unfiltered core workspace traversal as "Failed to find executable flutter": ${command}`, + ); + } + assert.ok(steps["legacy-bridge-continuity"].commands.includes("pnpm test:legacy-bridge-continuity")); + assert.equal(steps["legacy-bridge-continuity"].environment.T4_OMP_SOURCE_DIR, ".continuity/omp"); + assert.ok( + steps["android-debug"].commands.includes("pnpm --filter @t4-code/mobile check:android:debug"), + ); + assert.ok(steps["cluster-ci-contracts"].commands.includes("pnpm test:cluster:ci")); + assert.deepEqual(steps["cluster-operator-tests"].commands, [ + "GOMAXPROCS=1 GOFLAGS=-p=1 go test ./api/... ./controllers/... ./cmd/...", + "mkdir -p ../../artifacts/cluster-proof", + "CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS=-p=1 go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test", + ]); + assert.equal( + steps["cluster-operator-tests"].backend_options.kubernetes.resources.limits.memory, + "2Gi", + ); + assert.ok( + steps["cluster-chart-tests"].commands.includes("helm lint ../../../deploy/charts/t4-cluster"), + ); + assert.ok( + steps["cluster-chart-tests"].commands.some((command) => command.endsWith("chart-contract.test")), + ); + assert.match(steps["cluster-server-tests"].image, /oven\/bun:[^@]+@sha256:[0-9a-f]{64}$/u); + assert.ok( + steps["cluster-server-tests"].commands.includes("(cd packages/cluster-server && bun --bun run test)"), + ); + assert.match(steps["cluster-wire-tests"].image, /library\/node:[^@]+@sha256:[0-9a-f]{64}$/u); + assert.deepEqual(steps["cluster-wire-tests"].depends_on, ["cluster-server-tests", "bun-runtime"]); + assert.ok(steps["cluster-wire-tests"].commands.includes('export PATH="$PWD/.ci:$PATH"')); + assert.ok( + steps["cluster-wire-tests"].commands.includes( + "(cd packages/host-wire && bun test test/cluster-operator.test.ts)", + ), + ); + assert.ok( + steps["cluster-wire-tests"].commands.includes( + "(cd packages/host-service && bun test test/cluster-default-off.test.ts)", + ), + ); + assert.equal( + steps["cluster-wire-tests"].commands.includes("(cd packages/host-service && bun --bun run test)"), + false, + ); + assert.equal(JSON.stringify(pipeline).includes("from_secret"), false); + assert.deepEqual(steps["harbor-auth"].depends_on, ["cluster-chart-tests", "android-debug"]); + assert.equal( + steps["harbor-auth"].backend_options.kubernetes.serviceAccountName, + "woodpecker-dev-verifier", + ); + assert.deepEqual(steps["build-controller"].depends_on, ["harbor-auth"]); + assert.deepEqual(steps["live-cluster-observations"].depends_on, ["cleanup-image-registry-auth"]); + assert.deepEqual(steps["publish-live-proof"].depends_on, ["harbor-auth-live-proof"]); + assert.deepEqual(steps["image-publication-manifest"].depends_on, ["provenance-session-runtime"]); + assert.deepEqual(steps["promote-images"].depends_on, ["image-publication-manifest"]); + assert.deepEqual(steps["publish-image-evidence"].depends_on, ["promote-images"]); + assert.equal(steps["image-publication-manifest"].environment.HARBOR_REGISTRY, "harbor.tailb18de3.ts.net"); + assert.match(pipeline.clone.git.image, /@sha256:[0-9a-f]{64}$/u); + + const orderedBuilds = ["build-controller", "build-cluster-server", "build-session-runtime"]; + for (const [index, name] of orderedBuilds.entries()) { + const step = steps[name]; + assert.equal(step.backend_options.kubernetes.serviceAccountName, "woodpecker-ci-untrusted"); + assert.ok(step.backend_options.kubernetes.resources.limits.memory); + if (index > 0) assert.deepEqual(step.depends_on, [orderedBuilds[index - 1]]); + } + assert.match(steps["build-controller"].commands[0], /t4-cluster-operator/u); + assert.match(steps["build-cluster-server"].commands[0], /t4-cluster-server/u); + assert.match(steps["build-session-runtime"].commands[0], /t4-session-runtime/u); + + const busyboxEntrypoint = [ + "/busybox/sh", + "-c", + 'echo "$CI_SCRIPT" | /busybox/base64 -d | /busybox/sh -e', + ]; + for (const component of ["controller", "cluster-server", "session-runtime"]) { + const sbomStep = steps[`sbom-${component}`]; + assert.match(sbomStep.image, /anchore\/syft:v1\.33\.0-debug@sha256:[0-9a-f]{64}$/u); + assert.equal(sbomStep.environment.PATH, "/busybox:/"); + assert.deepEqual(sbomStep.entrypoint, busyboxEntrypoint); + assert.match(sbomStep.commands[0], /^sh scripts\/cluster-ci\/capture-image-evidence\.sh sbom /u); + + const provenanceStep = steps[`provenance-${component}`]; + assert.match(provenanceStep.image, /projectsigstore\/cosign:v2\.6\.0-dev@sha256:[0-9a-f]{64}$/u); + assert.deepEqual(provenanceStep.entrypoint, busyboxEntrypoint); + assert.match( + provenanceStep.commands[0], + /^sh scripts\/cluster-ci\/capture-image-evidence\.sh provenance /u, + ); + } + + for (const [name, step] of Object.entries(steps)) { + assert.match(step.image, /@sha256:[0-9a-f]{64}$/u, `${name} image must be immutable`); + if (step.environment?.HARBOR_REGISTRY) { + assert.equal(step.environment.HARBOR_REGISTRY, "harbor.tailb18de3.ts.net"); + } + for (const condition of step.when ?? []) { + if (condition.event === "manual") { + assert.deepEqual(condition.branch, ["main", "agent/t4-cluster-operator"]); + } + } + } + assert.deepEqual(steps["cleanup-image-registry-auth"].when[0].status, ["success", "failure"]); + assert.deepEqual(steps["cleanup-live-registry-auth"].when[0].status, ["success", "failure"]); + assert.deepEqual(steps["live-frame-proof"].commands, ["node scripts/cluster-ci/capture-redacted-frames.mjs"]); + assert.equal(steps["live-frame-proof"].environment.T4_CLUSTER_BASE_URL, "https://t4-dev.tailb18de3.ts.net/"); + assert.deepEqual(steps["live-proof-assembly"].depends_on, ["live-frame-proof"]); + const frameCaptureSource = await readFile(resolve(repoRoot, "scripts/cluster-ci/capture-redacted-frames.mjs"), "utf8"); + assert.doesNotMatch(frameCaptureSource, /client.?secret|deviceId|access.?token/iu); + for (const capability of ["sessions.read", "ci.trigger", "preview.read", "preview.control", "preview.input"]) { + assert.match(frameCaptureSource, new RegExp(capability.replace(".", "\\."), "u")); + } + assert.match(frameCaptureSource, /\/v1\/ws/u); + + const buildSource = await readFile(resolve(repoRoot, "scripts/cluster-ci/build-image.sh"), "utf8"); + assert.match(buildSource, /platform=linux\/amd64,linux\/arm64/u); + assert.match(buildSource, /source_context="https:\/\/github\.com\/\$canonical_build_source_repository\.git#\$CI_COMMIT_SHA"/u); + assert.match(buildSource, /SOURCE_REPOSITORY=https:\/\/github\.com\/\$canonical_build_source_repository/u); + assert.doesNotMatch(buildSource, /https:\/\/github\.com\/z-peterson\/t4-code/u); + assert.match(buildSource, /^canonical_build_source_repository=usr-bin-roygbiv\/t4-code$/mu); + assert.match(buildSource, /^authorized_ci_mirror=z-peterson\/t4-code$/mu); + assert.equal(buildSource.match(/usr-bin-roygbiv\/t4-code/gu)?.length, 1); + assert.equal(buildSource.match(/z-peterson\/t4-code/gu)?.length, 1); + assert.match(buildSource, /quarantine/u); + assert.match(buildSource, /chmod 1777 "\$artifact_dir"/u); + assert.match(buildSource, /chmod 0444 "\$metadata" "\$digest_file"/u); + + const sourceArgument = "ARG SOURCE_REPOSITORY=https://github.com/LycaonLLC/t4-code"; + const sourceLabel = 'org.opencontainers.image.source="${SOURCE_REPOSITORY}"'; + for (const component of IMAGE_COMPONENTS) { + const dockerfileSource = await readFile(resolve(repoRoot, "cluster/images", component, "Dockerfile"), "utf8"); + const finalStageOffset = dockerfileSource.lastIndexOf("\nFROM "); + const sourceArgumentOffset = dockerfileSource.indexOf(sourceArgument, finalStageOffset); + const sourceLabelOffset = dockerfileSource.indexOf(sourceLabel, sourceArgumentOffset); + assert.ok(finalStageOffset >= 0, `${component} must have a final image stage`); + assert.ok(sourceArgumentOffset > finalStageOffset, `${component} must declare SOURCE_REPOSITORY in its final stage`); + assert.ok(sourceLabelOffset > sourceArgumentOffset, `${component} must use SOURCE_REPOSITORY for its OCI source label`); + assert.equal(dockerfileSource.match(/ARG SOURCE_REPOSITORY=/gu)?.length, 1); + assert.equal(dockerfileSource.match(/org\.opencontainers\.image\.source=/gu)?.length, 1); + assert.doesNotMatch(dockerfileSource, /org\.opencontainers\.image\.source="https:\/\//u); + } + + const promotionSource = await readFile(resolve(repoRoot, "scripts/cluster-ci/promote-images.sh"), "utf8"); + const preflightResolveOffset = promotionSource.indexOf('if resolved=$(oras resolve "$destination" 2>&1); then'); + const differentDigestGuardOffset = promotionSource.indexOf('if [ "$resolved" != "$digest" ]; then', preflightResolveOffset); + const recursiveCopyOffset = promotionSource.indexOf('oras copy --recursive "$source" "$destination"', differentDigestGuardOffset); + const verificationResolveOffset = promotionSource.indexOf('resolved=$(oras resolve "$destination")', recursiveCopyOffset); + assert.ok( + preflightResolveOffset >= 0 && + preflightResolveOffset < differentDigestGuardOffset && + differentDigestGuardOffset < recursiveCopyOffset && + recursiveCopyOffset < verificationResolveOffset, + "promotion must resolve and reject an occupied tag before copying, then verify the result", + ); + const promotionGuard = promotionSource.slice(preflightResolveOffset, recursiveCopyOffset); + assert.match(promotionGuard, /if \[ "\$resolved" != "\$digest" \]; then[\s\S]*?exit 65\n fi\n else/u); + assert.match(promotionGuard, /"failed to resolve digest: \$CI_COMMIT_SHA: not found"/u); + assert.doesNotMatch(promotionSource.slice(0, preflightResolveOffset), /oras copy/u); + const authSource = await readFile(resolve(repoRoot, "scripts/cluster-ci/load-registry-auth.sh"), "utf8"); + assert.match(authSource, /chmod 0711 "\$auth_parent" "\$auth_dir"/u); + const provenanceSource = await readFile(resolve(repoRoot, "scripts/cluster-ci/capture-image-evidence.sh"), "utf8"); + assert.match(provenanceSource, /cosign verify-attestation/u); + assert.match(provenanceSource, /cosign download attestation/u); + assert.match(provenanceSource, /must be configured together/u); + assert.doesNotMatch(provenanceSource, /INSECURE|plain-http/iu); + const dockerignore = await readFile(resolve(repoRoot, ".dockerignore"), "utf8"); + assert.match(dockerignore, /^\.cluster-ci\/registry-auth$/mu); + assert.deepEqual(provenanceVerificationMode({}), { mode: "buildkit-content", signatureVerified: false }); + assert.deepEqual( + provenanceVerificationMode({ T4_COSIGN_CERTIFICATE_IDENTITY: "builder", T4_COSIGN_CERTIFICATE_OIDC_ISSUER: "https://issuer.example" }), + { mode: "cosign-keyless", signatureVerified: true }, + ); + assert.throws( + () => provenanceVerificationMode({ T4_COSIGN_CERTIFICATE_IDENTITY: "builder" }), + /configured together/u, + ); + + for (const script of ["build-image.sh", "capture-image-evidence.sh", "promote-images.sh", "publish-artifact.sh"]) { + const source = await readFile(resolve(repoRoot, "scripts/cluster-ci", script), "utf8"); + assert.doesNotMatch(source, /HARBOR_(?:USERNAME|PASSWORD)/u); + assert.match(source, /DOCKER_CONFIG/u); + } +}); + +test("registry auth is restricted to the exact HTTPS tailnet Harbor aliases", () => { + const auth = "dXNlcjpwYXNz"; + const normalized = normalizeRegistryAuth({ + auths: { + "harbor.tailb18de3.ts.net": { auth }, + "unrelated.example": { auth: "dW53YW50ZWQ6Y3JlZGVudGlhbA==" }, + }, + }); + assert.deepEqual(Object.keys(normalized.auths), HARBOR_REGISTRY_ALIASES); + assert.ok(Object.values(normalized.auths).every((entry) => entry.auth === auth)); + assert.equal("unrelated.example" in normalized.auths, false); + assert.throws(() => normalizeRegistryAuth({ auths: {} }), /no bounded registry authentication entry/u); +}); + +test("proof schema is strict and enumerates every bounded evidence domain", async () => { + const schema = JSON.parse( + await readFile(resolve(repoRoot, "scripts/cluster-ci/cluster-proof.schema.json"), "utf8"), + ); + assert.equal(schema.additionalProperties, false); + assert.deepEqual(schema.$defs.scenario.properties.id.enum, PROOF_SCENARIOS); + assert.deepEqual(schema.$defs.observation.properties.system.enum, OBSERVATION_SYSTEMS); + assert.deepEqual(schema.$defs.image.properties.component.enum, IMAGE_COMPONENTS); + assert.equal(schema.$defs.source.properties.repository.const, CANONICAL_BUILD_SOURCE_REPOSITORY); + assert.equal( + schema.$defs.source.properties.woodpecker.properties.repository.const, + AUTHORIZED_CI_MIRROR, + ); + assert.ok(schema.$defs.source.properties.woodpecker.required.includes("repository")); + assert.equal(schema.$defs.artifacts.properties.frames.maxItems, 32); + assert.equal("videos" in schema.$defs.artifacts.properties, false); +}); + +test("proof validation accepts exact run/image/scenario identity and rejects fabricated gaps", () => { + const proof = validProof(); + assert.equal(validateProofManifest(proof), proof); + + const missingScenario = structuredClone(proof); + missingScenario.scenarios.pop(); + assert.throws(() => validateProofManifest(missingScenario), /scenario coverage/u); + + const failedScenario = structuredClone(proof); + failedScenario.scenarios[0].status = "failed"; + assert.throws(() => validateProofManifest(failedScenario), /must be passed/u); + + const mutableImage = structuredClone(proof); + mutableImage.images[0].reference = `${mutableImage.images[0].repository}:latest`; + assert.throws(() => validateProofManifest(mutableImage), /immutable digest/u); + + const unredacted = structuredClone(proof); + unredacted.artifacts.frames[0].redacted = false; + assert.throws(() => validateProofManifest(unredacted), /redacted/u); + + const fabricatedLiveScreenshot = structuredClone(proof); + fabricatedLiveScreenshot.artifacts.screenshots[0].evidenceType = "live"; + assert.throws(() => validateProofManifest(fabricatedLiveScreenshot), /contract evidence/u); + + const fabricatedLiveScenario = structuredClone(proof); + fabricatedLiveScenario.scenarios.find(({ id }) => id === "desktop-viewport").evidenceType = "live"; + assert.throws(() => validateProofManifest(fabricatedLiveScenario), /must be contract/u); + + const unauthenticatedSignerClaim = structuredClone(proof); + unauthenticatedSignerClaim.images[0].provenance.mode = "cosign-keyless"; + assert.throws(() => validateProofManifest(unauthenticatedSignerClaim), /signer verification claim/u); + + const extra = structuredClone(proof); + extra.source.token = "must-not-survive"; + assert.throws(() => validateProofManifest(extra), /unexpected field/u); +}); + +test("proof source distinguishes the canonical build repository from the authorized CI mirror", () => { + const proof = validProof(); + assert.equal(proof.source.repository, CANONICAL_BUILD_SOURCE_REPOSITORY); + assert.equal(proof.source.woodpecker.repository, AUTHORIZED_CI_MIRROR); + + const mirrorAsBuildSource = structuredClone(proof); + mirrorAsBuildSource.source.repository = AUTHORIZED_CI_MIRROR; + assert.throws(() => validateProofManifest(mirrorAsBuildSource), /canonical build source/u); + + const buildSourceAsMirror = structuredClone(proof); + buildSourceAsMirror.source.woodpecker.repository = CANONICAL_BUILD_SOURCE_REPOSITORY; + assert.throws(() => validateProofManifest(buildSourceAsMirror), /authorized CI mirror/u); +}); + +test("file evidence is content-addressed instead of trusting a claimed result", async () => { + const directory = await mkdtemp(join(tmpdir(), "t4-cluster-proof-")); + const absolutePath = join(directory, "observation.json"); + await writeFile(absolutePath, '{"observed":true}\n', "utf8"); + const evidence = await createFileEvidence(absolutePath, { + artifactRoot: directory, + artifactPrefix: "artifacts/cluster-proof/observations", + }); + assert.equal(evidence.path, "artifacts/cluster-proof/observations/observation.json"); + assert.match(evidence.sha256, /^[0-9a-f]{64}$/u); + assert.notEqual(evidence.sha256, FILE_SHA); +}); + +test("frame redaction strips authority-sensitive content and bounds retained state", () => { + const redacted = redactFrame({ + type: "session.state", + sessionId: "proof-session", + cursor: 11, + revision: 7, + authorization: "Bearer secret", + auth: "private-auth", + bearer: "private-bearer", + apiKey: "private-api-key", + privateKey: "private-key", + token: "secret-token", + prompt: "private prompt", + transcript: [{ text: "private output" }], + payload: { cookie: "private-cookie", status: "running" }, + }); + assert.deepEqual(redacted, { + type: "session.state", + sessionId: "proof-session", + cursor: 11, + revision: 7, + authorization: "[REDACTED]", + auth: "[REDACTED]", + bearer: "[REDACTED]", + apiKey: "[REDACTED]", + privateKey: "[REDACTED]", + token: "[REDACTED]", + prompt: "[REDACTED]", + transcript: "[REDACTED]", + payload: { cookie: "[REDACTED]", status: "running" }, + }); + assert.throws(() => redactFrame({ payload: "x".repeat(70_000) }), /bound/u); +}); + +test("read-only collection executes only bounded Kubernetes GETs", async () => { + const responses = liveClusterResponses(); + const calls = []; + const snapshot = await collectReadOnlyClusterSnapshot({ + namespace: "t4-development", + run: async (command, args) => { + calls.push([command, args]); + assert.equal(command, "kubectl"); + assert.equal(args[0], "get"); + assert.ok(!args.some((arg) => ["apply", "create", "delete", "patch", "replace"].includes(arg))); + return JSON.stringify(responses[args[1]]); + }, + }); + assert.equal(calls.length, 9); + assert.equal(validateClusterSnapshot(snapshot, CLUSTER_VALIDATION), snapshot); +}); + +test("live snapshot validation rejects stale or loosely matched cluster truth", () => { + const responses = liveClusterResponses(); + assert.equal(validateClusterSnapshot(responses, CLUSTER_VALIDATION), responses); + + const noLeader = structuredClone(responses); + noLeader.lease.spec.holderIdentity = ""; + assert.throws(() => validateClusterSnapshot(noLeader, CLUSTER_VALIDATION), /leader Lease/u); + + const staleLeader = structuredClone(responses); + staleLeader.lease.spec.renewTime = "2026-07-20T12:33:00.000Z"; + assert.throws(() => validateClusterSnapshot(staleLeader, CLUSTER_VALIDATION), /freshly held/u); + + const wrongLeaseName = structuredClone(responses); + wrongLeaseName.lease.metadata.name = "t4-cluster-controller"; + assert.throws(() => validateClusterSnapshot(wrongLeaseName, CLUSTER_VALIDATION), /t4-cluster-operator\.cluster\.t4\.dev/u); + + const staleGeneration = structuredClone(responses); + staleGeneration.t4sessions.items[0].status.observedGeneration = 4; + assert.throws(() => validateClusterSnapshot(staleGeneration, CLUSTER_VALIDATION), /metadata\.generation exactly/u); + + const wrongControllerRollout = structuredClone(responses); + wrongControllerRollout.deployments.items[0].spec.strategy.rollingUpdate.maxUnavailable = 1; + assert.throws(() => validateClusterSnapshot(wrongControllerRollout, CLUSTER_VALIDATION), /exact HA rollout/u); + + const legacyPvcField = structuredClone(responses); + legacyPvcField.t4workspaces.items[0].status.pvcRef = { name: "proof-workspace-data" }; + delete legacyPvcField.t4workspaces.items[0].status.pvcName; + assert.throws(() => validateClusterSnapshot(legacyPvcField, CLUSTER_VALIDATION), /status\.pvcName/u); + + const wrongStorage = structuredClone(responses); + wrongStorage.persistentvolumeclaims.items[0].spec.accessModes = ["ReadWriteOnce"]; + assert.throws(() => validateClusterSnapshot(wrongStorage, CLUSTER_VALIDATION), /ReadWriteMany/u); + + const notReady = structuredClone(responses); + notReady.pods.items[3].status.conditions[0].status = "False"; + assert.throws(() => validateClusterSnapshot(notReady, CLUSTER_VALIDATION), /Running and Ready/u); + + const mutableContainer = structuredClone(responses); + mutableContainer.pods.items[3].status.containerStatuses[0].imageID = "containerd://mutable:latest"; + assert.throws(() => validateClusterSnapshot(mutableContainer, CLUSTER_VALIDATION), /current digest/u); + + const mismatchedContainerDigest = structuredClone(responses); + mismatchedContainerDigest.pods.items[3].status.containerStatuses[0].imageID = `containerd://harbor.tailb18de3.ts.net/linkedin-bot/workload@sha256:${"c".repeat(64)}`; + assert.throws(() => validateClusterSnapshot(mismatchedContainerDigest, CLUSTER_VALIDATION), /declared current digest/u); + + const wrongCiCommit = structuredClone(responses); + wrongCiCommit.t4sessions.items[0].spec.ci.commit = "f".repeat(40); + assert.throws(() => validateClusterSnapshot(wrongCiCommit, CLUSTER_VALIDATION), /CI mapping/u); + + const forbiddenPlacement = structuredClone(responses); + forbiddenPlacement.pods.items[3].spec.nodeName = "k3s-worker-03"; + assert.throws(() => validateClusterSnapshot(forbiddenPlacement, CLUSTER_VALIDATION), /durable session placement/u); + + const wrongSessionLabel = structuredClone(responses); + wrongSessionLabel.pods.items[3].metadata.labels["cluster.t4.dev/session"] = "proof-session"; + assert.throws(() => validateClusterSnapshot(wrongSessionLabel, CLUSTER_VALIDATION), /label cluster\.t4\.dev\/session/u); + + const wrongPortName = structuredClone(responses); + wrongPortName.services.items[0].spec.ports[0].name = "omp-app"; + assert.throws(() => validateClusterSnapshot(wrongPortName, CLUSTER_VALIDATION), /exact websocket port/u); + + const wrongMetricsPort = structuredClone(responses); + wrongMetricsPort.services.items[1].spec.ports[0].port = 8080; + assert.throws(() => validateClusterSnapshot(wrongMetricsPort, CLUSTER_VALIDATION), /admin metrics port/u); +}); + +test("SPDX, Trivy, and BuildKit content bind every image identity field", () => { + const repository = "harbor.tailb18de3.ts.net/linkedin-bot/quarantine/t4-cluster-operator"; + const reference = `${repository}@${DIGEST}`; + const spdx = { + spdxVersion: "SPDX-2.3", + dataLicense: "CC0-1.0", + SPDXID: "SPDXRef-DOCUMENT", + name: reference, + documentNamespace: `https://anchore.com/syft/image/${DIGEST.slice(7)}`, + documentDescribes: ["SPDXRef-Image"], + packages: [{ + SPDXID: "SPDXRef-Image", + name: "t4-cluster-operator", + externalRefs: [{ + referenceCategory: "PACKAGE-MANAGER", + referenceType: "purl", + referenceLocator: `pkg:oci/t4-cluster-operator@${DIGEST}?repository_url=${encodeURIComponent(repository)}`, + }], + }], + }; + assert.doesNotThrow(() => verifySpdx(spdx, { repository, digest: DIGEST, reference })); + assert.throws(() => verifySpdx({ spdxVersion: "SPDX-2.3" }, { repository, digest: DIGEST, reference }), /SPDX document identity/u); + const wrongSpdxDigest = structuredClone(spdx); + wrongSpdxDigest.packages[0].externalRefs[0].referenceLocator = "pkg:oci/t4-cluster-operator@sha256:deadbeef"; + assert.throws(() => verifySpdx(wrongSpdxDigest, { repository, digest: DIGEST, reference }), /external reference/u); + + const trivy = { + ArtifactName: reference, + ArtifactType: "container_image", + Metadata: { ImageID: `sha256:${"b".repeat(64)}`, RepoDigests: [reference] }, + Results: [{ Target: "debian", Class: "os-pkgs", Type: "debian", Vulnerabilities: [] }], + }; + assert.deepEqual(vulnerabilityCounts(trivy, { repository, digest: DIGEST, reference }), { critical: 0, high: 0 }); + assert.throws( + () => vulnerabilityCounts({ ...trivy, Results: [] }, { repository, digest: DIGEST, reference }), + /results identity/u, + ); + assert.throws( + () => vulnerabilityCounts({ ...trivy, ArtifactName: "wrong" }, { repository, digest: DIGEST, reference }), + /artifact\/results identity/u, + ); + const wrongTrivyIndexDigest = structuredClone(trivy); + wrongTrivyIndexDigest.Metadata.RepoDigests = [`${repository}@sha256:${"c".repeat(64)}`]; + assert.throws( + () => vulnerabilityCounts(wrongTrivyIndexDigest, { repository, digest: DIGEST, reference }), + /artifact\/results identity/u, + ); + const malformedTrivyChildId = structuredClone(trivy); + malformedTrivyChildId.Metadata.ImageID = "mutable-child"; + assert.throws( + () => vulnerabilityCounts(malformedTrivyChildId, { repository, digest: DIGEST, reference }), + /artifact\/results identity/u, + ); + + const statement = { + _type: "https://in-toto.io/Statement/v0.1", + predicateType: "https://slsa.dev/provenance/v0.2", + subject: [{ name: repository, digest: { sha256: DIGEST.slice(7) } }], + predicate: { + builder: { id: "https://mobyproject.org/buildkit@v1" }, + buildType: "https://mobyproject.org/buildkit@v1", + invocation: { parameters: { source: "https://github.com/usr-bin-roygbiv/t4-code", commit: COMMIT } }, + materials: [ + { uri: `https://github.com/usr-bin-roygbiv/t4-code.git#${COMMIT}`, digest: { sha1: COMMIT } }, + { uri: `pkg:docker/node@${DIGEST}`, digest: { sha256: DIGEST.slice(7) } }, + ], + }, + }; + const envelope = (payload) => `${JSON.stringify({ + payloadType: "application/vnd.in-toto+json", + payload: Buffer.from(JSON.stringify(payload)).toString("base64"), + })}\n`; + assert.doesNotThrow(() => verifyProvenance(envelope(statement), { repository, digest: DIGEST, commit: COMMIT })); + const wrongSource = structuredClone(statement); + wrongSource.predicate.materials[0].uri = `https://github.com/attacker/t4-code.git#${COMMIT}`; + assert.throws(() => verifyProvenance(envelope(wrongSource), { repository, digest: DIGEST, commit: COMMIT }), /trusted source/u); + assert.throws(() => verifyProvenance(envelope(statement), { repository, digest: DIGEST, commit: "f".repeat(40) }), /CI commit/u); +}); + +test("live observability parsers retain only bounded source-safe summaries", () => { + assert.deepEqual( + prometheusSample( + { status: "success", data: { resultType: "vector", result: [{ value: [1_721_480_000, "3"] }] } }, + "t4-cluster-up", + ), + { name: "t4-cluster-up", value: 3, sampledAt: "2024-07-20T12:53:20.000Z" }, + ); + assert.throws( + () => prometheusSample({ status: "success", data: { resultType: "vector", result: [] } }, "missing"), + /one bounded nonnegative sample/u, + ); + assert.deepEqual( + lokiLogSummary({ + status: "success", + data: { + resultType: "streams", + result: [{ stream: { namespace: "t4-development" }, values: [["1721480000000000000", '{"level":"info","event":"reconcile"}']] }], + }, + }), + { streamCount: 1, entryCount: 1, errorCount: 0 }, + ); + assert.deepEqual( + grafanaHealthSummary({ database: "ok", version: "12.4.2", commit: "e".repeat(40) }), + { database: "ok", version: "12.4.2", commit: "e".repeat(40) }, + ); + assert.throws( + () => grafanaHealthSummary({ database: "failed", version: "12.4.2", commit: "e".repeat(40) }), + /unhealthy/u, + ); +}); + +test("frame proof uses the exact cluster origin and typed Hello capabilities", () => { + assert.equal(clusterWebSocketUrl("https://t4-dev.tailb18de3.ts.net/").href, "wss://t4-dev.tailb18de3.ts.net/v1/ws"); + assert.throws(() => clusterWebSocketUrl("https://other.tailb18de3.ts.net/"), /credential-free HTTPS origin/u); + const hello = proofHelloFrame(); + assert.deepEqual(Object.keys(hello.capabilities), ["client"]); + assert.deepEqual(hello.capabilities.client, [ + "sessions.read", + "ci.trigger", + "preview.read", + "preview.control", + "preview.input", + ]); +}); + +test("default-off proof evaluates rendered resources rather than chart source text", () => { + assert.deepEqual( + validateDefaultOffRender([ + { apiVersion: "apiextensions.k8s.io/v1", kind: "CustomResourceDefinition", metadata: { name: "t4sessions.cluster.t4.dev" } }, + ]), + { clusterOperatorEnabled: false, workloadCount: 0 }, + ); + assert.throws( + () => + validateDefaultOffRender([ + { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: "t4-cluster-server" } }, + ]), + /default-off render created workload/u, + ); +}); diff --git a/scripts/cluster-ci/cluster-proof.schema.json b/scripts/cluster-ci/cluster-proof.schema.json new file mode 100644 index 00000000..2dbc4ca3 --- /dev/null +++ b/scripts/cluster-ci/cluster-proof.schema.json @@ -0,0 +1,211 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://t4.dev/schemas/cluster-proof-v1.json", + "title": "T4 cluster operator proof artifact", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "source", "images", "scenarios", "observations", "artifacts"], + "properties": { + "schemaVersion": { "const": "t4-cluster-proof/1" }, + "source": { "$ref": "#/$defs/source" }, + "images": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": { "$ref": "#/$defs/image" } + }, + "scenarios": { + "type": "array", + "minItems": 9, + "maxItems": 9, + "items": { "$ref": "#/$defs/scenario" } + }, + "observations": { + "type": "array", + "minItems": 5, + "maxItems": 5, + "items": { "$ref": "#/$defs/observation" } + }, + "artifacts": { "$ref": "#/$defs/artifacts" } + }, + "$defs": { + "safeString": { "type": "string", "minLength": 1, "maxLength": 512 }, + "timestamp": { "type": "string", "format": "date-time", "maxLength": 40 }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "digest": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }, + "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "httpsUrl": { "type": "string", "format": "uri", "pattern": "^https://", "maxLength": 2048 }, + "artifactPath": { + "type": "string", + "pattern": "^artifacts/cluster-proof/(?!.*(?:/\\.\\.?/|/\\.\\.?$))[A-Za-z0-9][A-Za-z0-9._/-]{0,255}$", + "maxLength": 288 + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["repository", "commit", "woodpecker"], + "properties": { + "repository": { "const": "usr-bin-roygbiv/t4-code" }, + "commit": { "$ref": "#/$defs/commit" }, + "woodpecker": { + "type": "object", + "additionalProperties": false, + "required": ["repository", "repositoryId", "pipelineId", "pipelineNumber", "url"], + "properties": { + "repository": { "const": "z-peterson/t4-code" }, + "repositoryId": { "type": "integer", "minimum": 1 }, + "pipelineId": { "type": "integer", "minimum": 1 }, + "pipelineNumber": { "type": "integer", "minimum": 1 }, + "url": { "$ref": "#/$defs/httpsUrl" } + } + } + } + }, + "image": { + "type": "object", + "additionalProperties": false, + "required": ["component", "repository", "tag", "digest", "reference", "sbom", "provenance", "vulnerability"], + "properties": { + "component": { "enum": ["controller", "cluster-server", "session-runtime"] }, + "repository": { + "type": "string", + "pattern": "^[a-z0-9.-]+(?::[0-9]+)?/[a-z0-9._/-]+$", + "maxLength": 255 + }, + "tag": { "$ref": "#/$defs/commit" }, + "digest": { "$ref": "#/$defs/digest" }, + "reference": { + "type": "string", + "pattern": "^[a-z0-9.-]+(?::[0-9]+)?/[a-z0-9._/-]+@sha256:[0-9a-f]{64}$", + "maxLength": 340 + }, + "sbom": { "$ref": "#/$defs/fileEvidence" }, + "provenance": { "$ref": "#/$defs/provenanceEvidence" }, + "vulnerability": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256", "scanner", "critical", "high"], + "properties": { + "path": { "$ref": "#/$defs/artifactPath" }, + "sha256": { "$ref": "#/$defs/sha256" }, + "scanner": { "const": "trivy" }, + "critical": { "type": "integer", "const": 0 }, + "high": { "type": "integer", "const": 0 } + } + } + } + }, + "fileEvidence": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256"], + "properties": { + "path": { "$ref": "#/$defs/artifactPath" }, + "sha256": { "$ref": "#/$defs/sha256" } + } + }, + "provenanceEvidence": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256", "mode", "signatureVerified"], + "properties": { + "path": { "$ref": "#/$defs/artifactPath" }, + "sha256": { "$ref": "#/$defs/sha256" }, + "mode": { "enum": ["buildkit-content", "cosign-keyless"] }, + "signatureVerified": { "type": "boolean" } + }, + "allOf": [ + { + "if": { "properties": { "mode": { "const": "cosign-keyless" } } }, + "then": { "properties": { "signatureVerified": { "const": true } } }, + "else": { "properties": { "signatureVerified": { "const": false } } } + } + ] + }, + "scenario": { + "type": "object", + "additionalProperties": false, + "required": ["id", "status", "evidenceType", "observedAt", "assertions", "evidence"], + "properties": { + "id": { + "enum": [ + "ha-manifest", + "leader-election", + "crd-reconcile-storage", + "feature-off", + "wire-reconnect-idempotency", + "gui-auth-isolation", + "ci-mapping", + "desktop-viewport", + "mobile-viewport" + ] + }, + "status": { "const": "passed" }, + "evidenceType": { "enum": ["contract", "live"] }, + "observedAt": { "$ref": "#/$defs/timestamp" }, + "assertions": { + "type": "array", + "minItems": 1, + "maxItems": 64, + "uniqueItems": true, + "items": { "type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{0,127}$" } + }, + "evidence": { + "type": "array", + "minItems": 1, + "maxItems": 32, + "items": { "$ref": "#/$defs/fileEvidence" } + } + } + }, + "observation": { + "type": "object", + "additionalProperties": false, + "required": ["system", "observedAt", "url", "ids", "evidence"], + "properties": { + "system": { "enum": ["woodpecker", "kubernetes", "prometheus", "loki", "grafana"] }, + "observedAt": { "$ref": "#/$defs/timestamp" }, + "url": { "oneOf": [{ "$ref": "#/$defs/httpsUrl" }, { "type": "null" }] }, + "ids": { + "type": "array", + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "items": { "$ref": "#/$defs/safeString" } + }, + "evidence": { "$ref": "#/$defs/fileEvidence" } + } + }, + "visualEvidence": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256", "redacted", "evidenceType"], + "properties": { + "path": { "$ref": "#/$defs/artifactPath" }, + "sha256": { "$ref": "#/$defs/sha256" }, + "redacted": { "const": true }, + "evidenceType": { "enum": ["contract", "live"] }, + "viewport": { "enum": ["desktop", "mobile"] } + } + }, + "artifacts": { + "type": "object", + "additionalProperties": false, + "required": ["frames", "screenshots"], + "properties": { + "frames": { + "type": "array", + "minItems": 1, + "maxItems": 32, + "items": { "$ref": "#/$defs/visualEvidence" } + }, + "screenshots": { + "type": "array", + "minItems": 2, + "maxItems": 32, + "items": { "$ref": "#/$defs/visualEvidence" } + } + } + } + } +} diff --git a/scripts/cluster-ci/collect-readonly-cluster.mjs b/scripts/cluster-ci/collect-readonly-cluster.mjs new file mode 100644 index 00000000..17316136 --- /dev/null +++ b/scripts/cluster-ci/collect-readonly-cluster.mjs @@ -0,0 +1,273 @@ +import { execFile } from "node:child_process"; +import { mkdir, rename, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import yaml from "js-yaml"; + +import { AUTHORIZED_CI_MIRROR } from "./proof-contract.mjs"; +import { + collectReadOnlyClusterSnapshot, + summarizeClusterSnapshot, + validateDefaultOffRender, +} from "./readonly-cluster-proof.mjs"; + +const execFileAsync = promisify(execFile); +const repoRoot = resolve(import.meta.dirname, "../.."); +const proofRoot = resolve(repoRoot, "artifacts/cluster-proof"); +const scenarioRoot = resolve(proofRoot, "scenarios"); +const observationRoot = resolve(proofRoot, "observations"); +const MAX_OBSERVATION_BYTES = 2 * 1024 * 1024; +const OBSERVABILITY_SERVICES = Object.freeze({ + prometheus: "http://prometheus-prometheus.linkedin-monitoring.svc.cluster.local:9090", + loki: "http://loki.linkedin-monitoring.svc.cluster.local:3100", + grafana: "http://prometheus-grafana.linkedin-monitoring.svc.cluster.local", +}); +const PROMETHEUS_QUERIES = Object.freeze([ + ["t4-cluster-up", 'sum(up{job="t4-cluster"})'], + ["t4-cluster-reconcile-success", 'sum(t4_cluster_reconcile_total{result="success"})'], + ["t4-cluster-conditions", "sum(t4_cluster_condition)"], +]); + +function requiredEnvironment(name) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +function currentCiMapping() { + const pipelineUrl = new URL(requiredEnvironment("CI_PIPELINE_URL")); + const match = pipelineUrl.pathname.match(/^\/repos\/([1-9][0-9]*)\/pipeline\/[1-9][0-9]*\/?$/u); + if ( + pipelineUrl.origin !== "https://woodpecker-ci-dev.tailb18de3.ts.net" || + pipelineUrl.username || + pipelineUrl.password || + pipelineUrl.search || + pipelineUrl.hash || + !match + ) { + throw new Error("CI_PIPELINE_URL does not identify the exact credential-free Woodpecker repository"); + } + const ciRepository = requiredEnvironment("CI_REPO"); + if (ciRepository !== AUTHORIZED_CI_MIRROR) throw new Error("CI_REPO is not the authorized CI mirror"); + return { + repositoryId: match[1], + ref: requiredEnvironment("CI_COMMIT_REF"), + commit: requiredEnvironment("CI_COMMIT_SHA"), + }; +} + +async function atomicJson(path, value) { + const temporaryPath = `${path}.tmp`; + await writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); + await rename(temporaryPath, path); +} + +async function scenario(id, observedAt, assertions) { + await atomicJson(resolve(scenarioRoot, `${id}.json`), { + schemaVersion: "t4-cluster-scenario/1", + id, + status: "passed", + observedAt, + assertions, + }); +} + +async function defaultOffEvidence(namespace) { + const { stdout } = await execFileAsync( + "helm", + ["template", "t4-default-off", "deploy/charts/t4-cluster", "--namespace", namespace], + { cwd: repoRoot, encoding: "utf8", maxBuffer: 8 * 1024 * 1024, timeout: 30_000 }, + ); + const documents = []; + yaml.loadAll(stdout, (document) => { + if (document) documents.push(document); + }); + const result = validateDefaultOffRender(documents); + await atomicJson(resolve(observationRoot, "feature-off-render.json"), { + schemaVersion: "t4-cluster-feature-off/1", + observedAt: new Date().toISOString(), + ...result, + renderedKinds: [...new Set(documents.map(({ kind }) => kind).filter(Boolean))].sort(), + }); + return result; +} + +async function boundedJson(url, label, fetchImpl) { + const response = await fetchImpl(url, { + headers: { Accept: "application/json", "User-Agent": "t4-cluster-proof/1" }, + redirect: "error", + signal: AbortSignal.timeout(20_000), + }); + if (!response.ok) throw new Error(`${label} returned HTTP ${response.status}`); + const declaredLength = Number(response.headers.get("content-length") ?? 0); + if (declaredLength > MAX_OBSERVATION_BYTES) throw new Error(`${label} exceeded its byte bound`); + const bytes = Buffer.from(await response.arrayBuffer()); + if (bytes.length === 0 || bytes.length > MAX_OBSERVATION_BYTES) { + throw new Error(`${label} was empty or exceeded its byte bound`); + } + try { + return JSON.parse(bytes.toString("utf8")); + } catch (error) { + throw new Error(`${label} was not valid JSON`, { cause: error }); + } +} + +export function prometheusSample(payload, name) { + const result = payload?.data?.result; + const sample = Array.isArray(result) && result.length === 1 ? result[0]?.value : undefined; + const timestamp = Array.isArray(sample) ? Number(sample[0]) : Number.NaN; + const value = Array.isArray(sample) ? Number(sample[1]) : Number.NaN; + if ( + payload?.status !== "success" || + payload?.data?.resultType !== "vector" || + !Array.isArray(sample) || + sample.length !== 2 || + !Number.isFinite(timestamp) || + timestamp <= 0 || + !Number.isFinite(value) || + value < 0 + ) { + throw new Error(`Prometheus ${name} did not return one bounded nonnegative sample`); + } + return { name, value, sampledAt: new Date(timestamp * 1000).toISOString() }; +} + +export function lokiLogSummary(payload) { + const streams = payload?.data?.result; + if ( + payload?.status !== "success" || + payload?.data?.resultType !== "streams" || + !Array.isArray(streams) || + streams.length < 1 || + streams.length > 128 + ) { + throw new Error("Loki did not return a bounded T4 log stream result"); + } + let entryCount = 0; + let errorCount = 0; + for (const stream of streams) { + if (!stream?.stream || typeof stream.stream !== "object" || !Array.isArray(stream.values) || stream.values.length > 1000) { + throw new Error("Loki returned a malformed T4 log stream"); + } + for (const entry of stream.values) { + if (!Array.isArray(entry) || entry.length !== 2 || !/^[0-9]{1,20}$/u.test(entry[0]) || typeof entry[1] !== "string" || entry[1].length > 65_536) { + throw new Error("Loki returned a malformed T4 log entry"); + } + entryCount += 1; + try { + if (JSON.parse(entry[1])?.level === "error") errorCount += 1; + } catch { + if (/"level"\s*:\s*"error"/u.test(entry[1])) errorCount += 1; + } + } + } + if (entryCount < 1) throw new Error("Loki returned no current T4 log entries"); + return { streamCount: streams.length, entryCount, errorCount }; +} + +export function grafanaHealthSummary(payload) { + if ( + payload?.database !== "ok" || + typeof payload.version !== "string" || + !/^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][A-Za-z0-9.-]+)?$/u.test(payload.version) || + typeof payload.commit !== "string" || + !/^[0-9a-f]{40}$/u.test(payload.commit) + ) { + throw new Error("Grafana health identity is malformed or unhealthy"); + } + return { database: "ok", version: payload.version, commit: payload.commit }; +} + +export async function collectObservabilityEvidence({ sourceCommit, namespace, fetchImpl = fetch, now = Date.now() }) { + if (!/^[0-9a-f]{40}$/u.test(sourceCommit)) throw new Error("observability source commit is invalid"); + if (!/^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u.test(namespace)) throw new Error("observability namespace is invalid"); + const observedAt = new Date(now).toISOString(); + const metrics = []; + for (const [name, query] of PROMETHEUS_QUERIES) { + const url = new URL("/api/v1/query", OBSERVABILITY_SERVICES.prometheus); + url.searchParams.set("query", query); + metrics.push(prometheusSample(await boundedJson(url, `Prometheus ${name}`, fetchImpl), name)); + } + const lokiUrl = new URL("/loki/api/v1/query_range", OBSERVABILITY_SERVICES.loki); + lokiUrl.searchParams.set("query", `{namespace="${namespace}"}`); + lokiUrl.searchParams.set("start", String((now - 15 * 60_000) * 1_000_000)); + lokiUrl.searchParams.set("end", String(now * 1_000_000)); + lokiUrl.searchParams.set("limit", "256"); + lokiUrl.searchParams.set("direction", "backward"); + const logs = lokiLogSummary(await boundedJson(lokiUrl, "Loki T4 logs", fetchImpl)); + if (logs.errorCount !== 0) throw new Error(`Loki found ${logs.errorCount} current T4 error event(s)`); + const grafana = grafanaHealthSummary( + await boundedJson(new URL("/api/health", OBSERVABILITY_SERVICES.grafana), "Grafana health", fetchImpl), + ); + const records = [ + { system: "prometheus", ids: metrics.map(({ name }) => name), summary: { metrics } }, + { system: "loki", ids: [namespace], summary: logs }, + { system: "grafana", ids: [grafana.version, grafana.commit], summary: grafana }, + ]; + for (const record of records) { + await atomicJson(resolve(observationRoot, `${record.system}.json`), { + schemaVersion: "t4-cluster-observation/1", + sourceCommit, + system: record.system, + observedAt, + redacted: true, + ids: record.ids, + summary: record.summary, + }); + } + return records; +} + +export async function collectClusterEvidence({ namespace, ciMapping = currentCiMapping() }) { + await mkdir(scenarioRoot, { recursive: true }); + await mkdir(observationRoot, { recursive: true }); + const snapshot = await collectReadOnlyClusterSnapshot({ namespace }); + const summary = summarizeClusterSnapshot(snapshot, { ciMapping }); + await atomicJson(resolve(observationRoot, "kubernetes.json"), summary); + await scenario("ha-manifest", summary.observedAt, [ + "controller.replicas-2", + "controller.rolling-update.max-unavailable-0", + "server.replicas-3", + "server.rolling-update.max-unavailable-0", + ]); + await scenario("leader-election", summary.observedAt, [ + "lease.active-holder", + "lease.renew-time-observed", + "reconcile.single-active-leader", + ]); + await scenario("crd-reconcile-storage", summary.observedAt, [ + "crd.namespaced-v1alpha1", + "reconcile.observed-generation", + "storage.bound-read-write-many", + "placement.session-worker-exclusions", + ]); + await scenario("ci-mapping", summary.observedAt, [ + "ci.repository-id-exact", + "ci.ref-exact", + "ci.commit-exact", + "ci.session-running-ready", + ]); + const off = await defaultOffEvidence(namespace); + await scenario("feature-off", new Date().toISOString(), [ + off.clusterOperatorEnabled ? "feature-off.invalid" : "feature-off.no-workloads", + ]); + await collectObservabilityEvidence({ sourceCommit: ciMapping.commit, namespace }); + return summary; +} + +const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)); +if (isMain) { + try { + const namespace = process.env.T4_CLUSTER_NAMESPACE?.trim(); + if (!namespace) throw new Error("T4_CLUSTER_NAMESPACE is required"); + const summary = await collectClusterEvidence({ namespace }); + console.log( + `Captured read-only T4 cluster evidence for ${summary.workspaces.length} workspace(s) and ${summary.sessions.length} session(s)`, + ); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/cluster-ci/load-registry-auth.sh b/scripts/cluster-ci/load-registry-auth.sh new file mode 100755 index 00000000..be480d3e --- /dev/null +++ b/scripts/cluster-ci/load-registry-auth.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu + +umask 077 +workspace=${CI_WORKSPACE:-$PWD} +auth_dir=${T4_REGISTRY_AUTH_DIR:-$workspace/.cluster-ci/registry-auth} +case "$auth_dir" in + "$workspace"/.cluster-ci/*) ;; + *) + echo "registry auth directory must remain inside the CI workspace" >&2 + exit 64 + ;; +esac + +rm -rf "$auth_dir" +mkdir -p "$auth_dir" +auth_parent=${auth_dir%/*} +chmod 0711 "$auth_parent" "$auth_dir" +unset auth_parent +encoded=$(kubectl -n linkedin-ci get secret harbor-registry-credentials -o 'jsonpath={.data.\.dockerconfigjson}') +if [ -z "$encoded" ]; then + echo "Harbor credential resource is missing its Docker config key" >&2 + exit 65 +fi +printf '%s' "$encoded" | base64 -d > "$auth_dir/config.json" +unset encoded +node scripts/cluster-ci/normalize-registry-auth.mjs "$auth_dir/config.json" +test -r "$auth_dir/config.json" diff --git a/scripts/cluster-ci/normalize-registry-auth.mjs b/scripts/cluster-ci/normalize-registry-auth.mjs new file mode 100644 index 00000000..3dce5db8 --- /dev/null +++ b/scripts/cluster-ci/normalize-registry-auth.mjs @@ -0,0 +1,41 @@ +import { chmod, readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const HARBOR_REGISTRY_ALIASES = Object.freeze([ + "harbor.tailb18de3.ts.net", + "https://harbor.tailb18de3.ts.net", +]); + +function isObject(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +export function normalizeRegistryAuth(config) { + if (!isObject(config) || !isObject(config.auths)) { + throw new Error("Harbor Docker config must contain an auths object"); + } + + const source = HARBOR_REGISTRY_ALIASES.map((host) => config.auths[host]).find( + (entry) => isObject(entry) && typeof entry.auth === "string" && /^[A-Za-z0-9+/]+={0,2}$/u.test(entry.auth), + ); + if (!source) { + throw new Error("Harbor Docker config has no bounded registry authentication entry"); + } + + return { + auths: Object.fromEntries(HARBOR_REGISTRY_ALIASES.map((host) => [host, { auth: source.auth }])), + }; +} + +export async function normalizeRegistryAuthFile(path) { + const config = JSON.parse(await readFile(path, "utf8")); + await writeFile(path, `${JSON.stringify(normalizeRegistryAuth(config))}\n`, { mode: 0o444 }); + await chmod(path, 0o444); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const path = process.argv[2]; + if (!path) throw new Error("Docker config path is required"); + await normalizeRegistryAuthFile(path); +} diff --git a/scripts/cluster-ci/promote-images.sh b/scripts/cluster-ci/promote-images.sh new file mode 100755 index 00000000..fef7a978 --- /dev/null +++ b/scripts/cluster-ci/promote-images.sh @@ -0,0 +1,58 @@ +#!/bin/sh +set -eu + +umask 077 + +: "${CI_COMMIT_SHA:?CI_COMMIT_SHA is required}" +: "${HARBOR_REGISTRY:?HARBOR_REGISTRY is required}" +: "${HARBOR_PROJECT:?HARBOR_PROJECT is required}" +if [ "$HARBOR_REGISTRY" != "harbor.tailb18de3.ts.net" ]; then + echo "HARBOR_REGISTRY must be the exact HTTPS tailnet Harbor host" >&2 + exit 64 +fi +case "$CI_COMMIT_SHA" in + [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) ;; + *) echo "CI_COMMIT_SHA must be an exact lowercase 40-character SHA" >&2; exit 64 ;; +esac + +auth_dir=${T4_REGISTRY_AUTH_DIR:-${CI_WORKSPACE:-$PWD}/.cluster-ci/registry-auth} +test -r "$auth_dir/config.json" +export DOCKER_CONFIG="$auth_dir" +artifact_dir=artifacts/cluster-proof/images + +for entry in \ + controller:t4-cluster-operator \ + cluster-server:t4-cluster-server \ + session-runtime:t4-session-runtime +do + component=${entry%%:*} + repository_suffix=${entry#*:} + digest=$(cat "$artifact_dir/$component.digest") + case "$digest" in + sha256:????????????????????????????????????????????????????????????????) ;; + *) echo "$component image digest artifact is malformed" >&2; exit 65 ;; + esac + source="$HARBOR_REGISTRY/$HARBOR_PROJECT/quarantine/$repository_suffix@$digest" + destination="$HARBOR_REGISTRY/$HARBOR_PROJECT/$repository_suffix:$CI_COMMIT_SHA" + if resolved=$(oras resolve "$destination" 2>&1); then + if [ "$resolved" != "$digest" ]; then + echo "$component destination commit tag already resolves to a different digest" >&2 + exit 65 + fi + else + case "$resolved" in + *"failed to resolve digest: $CI_COMMIT_SHA: not found") ;; + *) + printf '%s\n' "$resolved" >&2 + echo "$component destination commit tag could not be resolved safely" >&2 + exit 65 + ;; + esac + oras copy --recursive "$source" "$destination" + fi + resolved=$(oras resolve "$destination") + if [ "$resolved" != "$digest" ]; then + echo "$component promoted reference did not resolve to the gated digest" >&2 + exit 65 + fi +done diff --git a/scripts/cluster-ci/proof-contract.mjs b/scripts/cluster-ci/proof-contract.mjs new file mode 100644 index 00000000..1d89a07b --- /dev/null +++ b/scripts/cluster-ci/proof-contract.mjs @@ -0,0 +1,362 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { basename, relative, resolve, sep } from "node:path"; + +export const CANONICAL_BUILD_SOURCE_REPOSITORY = "usr-bin-roygbiv/t4-code"; +export const AUTHORIZED_CI_MIRROR = "z-peterson/t4-code"; +export const IMAGE_COMPONENTS = Object.freeze(["controller", "cluster-server", "session-runtime"]); +export const PROOF_SCENARIOS = Object.freeze([ + "ha-manifest", + "leader-election", + "crd-reconcile-storage", + "feature-off", + "wire-reconnect-idempotency", + "gui-auth-isolation", + "ci-mapping", + "desktop-viewport", + "mobile-viewport", +]); +export const OBSERVATION_SYSTEMS = Object.freeze([ + "woodpecker", + "kubernetes", + "prometheus", + "loki", + "grafana", +]); +const CONTRACT_SCENARIOS = new Set([ + "wire-reconnect-idempotency", + "gui-auth-isolation", + "desktop-viewport", + "mobile-viewport", +]); + +const COMMIT_PATTERN = /^[0-9a-f]{40}$/u; +const SHA_PATTERN = /^[0-9a-f]{64}$/u; +const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; +const REPOSITORY_PATTERN = /^[a-z0-9.-]+(?::[0-9]+)?\/[a-z0-9._/-]+$/u; +const ARTIFACT_PATH_PATTERN = /^artifacts\/cluster-proof\/[A-Za-z0-9][A-Za-z0-9._/-]{0,255}$/u; +const ASSERTION_PATTERN = /^[a-z0-9][a-z0-9._-]{0,127}$/u; +const SENSITIVE_KEY_PATTERN = /api.?key|private.?key|authorization|auth|bearer|cookie|credential|password|prompt|secret|token|transcript/iu; +const MAX_FRAME_BYTES = 64 * 1024; +const MAX_DEPTH = 12; +const MAX_OBJECT_FIELDS = 128; +const MAX_ARRAY_ITEMS = 256; +const OBSERVATION_HOSTS = Object.freeze({ + woodpecker: new Set(["woodpecker-ci-dev.tailb18de3.ts.net"]), + prometheus: new Set(["interview-responder-prometheus.tailb18de3.ts.net"]), + loki: new Set(["interview-responder-loki.tailb18de3.ts.net"]), + grafana: new Set(["grafana.tailb18de3.ts.net"]), +}); + +function fail(message) { + throw new Error(`T4 cluster proof ${message}`); +} + +function object(value, label) { + if (!value || typeof value !== "object" || Array.isArray(value)) fail(`${label} must be an object`); + return value; +} + +function exactFields(value, required, label) { + object(value, label); + const expected = new Set(required); + for (const key of Object.keys(value)) { + if (!expected.has(key)) fail(`${label} has unexpected field ${key}`); + } + for (const key of required) { + if (!(key in value)) fail(`${label} is missing ${key}`); + } + return value; +} + +function positiveInteger(value, label) { + if (!Number.isSafeInteger(value) || value <= 0) fail(`${label} must be a positive integer`); + return value; +} + +function boundedString(value, label, maxLength = 512) { + if (typeof value !== "string" || value.length === 0 || value.length > maxLength) { + fail(`${label} must be a non-empty string no longer than ${maxLength}`); + } + return value; +} + +function exactSet(values, expected, label) { + if (!Array.isArray(values) || values.length !== expected.length) fail(`${label} coverage is incomplete`); + const actual = values.map((item) => item.id ?? item.system ?? item.component); + if (new Set(actual).size !== actual.length) fail(`${label} contains duplicates`); + for (const required of expected) { + if (!actual.includes(required)) fail(`${label} coverage is missing ${required}`); + } +} + +function timestamp(value, label) { + boundedString(value, label, 40); + if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/u.test(value) || !Number.isFinite(Date.parse(value))) { + fail(`${label} must be a UTC RFC 3339 timestamp`); + } + return value; +} + +function httpsUrl(value, label) { + boundedString(value, label, 2048); + let url; + try { + url = new URL(value); + } catch { + fail(`${label} must be a valid HTTPS URL`); + } + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash + ) { + fail(`${label} must be a credential-free HTTPS URL without query or fragment`); + } + return value; +} + +function observationUrl(value, system, label) { + if (system === "kubernetes") { + if (value !== null) fail(`${label} must be null because Kubernetes was captured through bounded kubectl GETs`); + return value; + } + httpsUrl(value, label); + const url = new URL(value); + if (url.port || !OBSERVATION_HOSTS[system]?.has(url.hostname)) { + fail(`${label} is outside the exact ${system} tailnet allowlist`); + } + return value; +} + +function fileEvidence(value, label, extraFields = []) { + const fields = ["path", "sha256", ...extraFields]; + exactFields(value, fields, label); + if (!ARTIFACT_PATH_PATTERN.test(value.path) || value.path.includes("..")) { + fail(`${label}.path must stay inside artifacts/cluster-proof`); + } + if (!SHA_PATTERN.test(value.sha256)) fail(`${label}.sha256 must be a lowercase SHA-256`); + return value; +} + +function visualEvidence(value, label) { + const fields = ["path", "sha256", "redacted", "evidenceType", ...(value.viewport === undefined ? [] : ["viewport"] )]; + fileEvidence(value, label, fields.slice(2)); + if (value.redacted !== true) fail(`${label} must be redacted`); + if (!["contract", "live"].includes(value.evidenceType)) fail(`${label}.evidenceType is invalid`); + if (value.viewport !== undefined && !["desktop", "mobile"].includes(value.viewport)) { + fail(`${label}.viewport is invalid`); + } + return value; +} + +function validateSource(source) { + exactFields(source, ["repository", "commit", "woodpecker"], "source"); + if (source.repository !== CANONICAL_BUILD_SOURCE_REPOSITORY) { + fail("source.repository is not the canonical build source"); + } + if (!COMMIT_PATTERN.test(source.commit)) fail("source.commit must be an exact lowercase SHA"); + exactFields( + source.woodpecker, + ["repository", "repositoryId", "pipelineId", "pipelineNumber", "url"], + "source.woodpecker", + ); + if (source.woodpecker.repository !== AUTHORIZED_CI_MIRROR) { + fail("source.woodpecker.repository is not the authorized CI mirror"); + } + positiveInteger(source.woodpecker.repositoryId, "source.woodpecker.repositoryId"); + positiveInteger(source.woodpecker.pipelineId, "source.woodpecker.pipelineId"); + positiveInteger(source.woodpecker.pipelineNumber, "source.woodpecker.pipelineNumber"); + httpsUrl(source.woodpecker.url, "source.woodpecker.url"); + return source; +} + +export function validateImageEntries(images, sourceCommit) { + exactSet(images, IMAGE_COMPONENTS, "image"); + for (const [index, image] of images.entries()) { + const label = `images[${index}]`; + exactFields( + image, + ["component", "repository", "tag", "digest", "reference", "sbom", "provenance", "vulnerability"], + label, + ); + if (!IMAGE_COMPONENTS.includes(image.component)) fail(`${label}.component is invalid`); + if (!REPOSITORY_PATTERN.test(image.repository) || image.repository.length > 255) { + fail(`${label}.repository is invalid`); + } + if (image.tag !== sourceCommit || !COMMIT_PATTERN.test(image.tag)) { + fail(`${label}.tag must equal the exact source commit`); + } + if (!DIGEST_PATTERN.test(image.digest)) fail(`${label}.digest is invalid`); + if (image.reference !== `${image.repository}@${image.digest}`) { + fail(`${label}.reference must use the immutable digest`); + } + fileEvidence(image.sbom, `${label}.sbom`); + fileEvidence(image.provenance, `${label}.provenance`, ["mode", "signatureVerified"]); + if ( + !["buildkit-content", "cosign-keyless"].includes(image.provenance.mode) || + image.provenance.signatureVerified !== (image.provenance.mode === "cosign-keyless") + ) { + fail(`${label}.provenance signer verification claim is not truthful`); + } + fileEvidence(image.vulnerability, `${label}.vulnerability`, ["scanner", "critical", "high"]); + if ( + image.vulnerability.scanner !== "trivy" || + image.vulnerability.critical !== 0 || + image.vulnerability.high !== 0 + ) { + fail(`${label}.vulnerability must contain a clean Trivy result`); + } + } + return images; +} + +function validateScenarioEntries(scenarios) { + exactSet(scenarios, PROOF_SCENARIOS, "scenario"); + for (const [index, scenario] of scenarios.entries()) { + const label = `scenarios[${index}]`; + exactFields(scenario, ["id", "status", "evidenceType", "observedAt", "assertions", "evidence"], label); + if (!PROOF_SCENARIOS.includes(scenario.id)) fail(`${label}.id is invalid`); + if (scenario.status !== "passed") fail(`${label} must be passed`); + const expectedType = CONTRACT_SCENARIOS.has(scenario.id) ? "contract" : "live"; + if (scenario.evidenceType !== expectedType) fail(`${label}.evidenceType must be ${expectedType}`); + timestamp(scenario.observedAt, `${label}.observedAt`); + if ( + !Array.isArray(scenario.assertions) || + scenario.assertions.length < 1 || + scenario.assertions.length > 64 || + new Set(scenario.assertions).size !== scenario.assertions.length || + scenario.assertions.some((assertion) => !ASSERTION_PATTERN.test(assertion)) + ) { + fail(`${label}.assertions are invalid or outside their bound`); + } + if (!Array.isArray(scenario.evidence) || scenario.evidence.length < 1 || scenario.evidence.length > 32) { + fail(`${label}.evidence is outside its bound`); + } + scenario.evidence.forEach((evidence, evidenceIndex) => + fileEvidence(evidence, `${label}.evidence[${evidenceIndex}]`), + ); + } + return scenarios; +} + +function validateObservationEntries(observations) { + exactSet(observations, OBSERVATION_SYSTEMS, "observation"); + for (const [index, observation] of observations.entries()) { + const label = `observations[${index}]`; + exactFields(observation, ["system", "observedAt", "url", "ids", "evidence"], label); + if (!OBSERVATION_SYSTEMS.includes(observation.system)) fail(`${label}.system is invalid`); + timestamp(observation.observedAt, `${label}.observedAt`); + observationUrl(observation.url, observation.system, `${label}.url`); + if ( + !Array.isArray(observation.ids) || + observation.ids.length < 1 || + observation.ids.length > 16 || + new Set(observation.ids).size !== observation.ids.length + ) { + fail(`${label}.ids are invalid or outside their bound`); + } + observation.ids.forEach((id, idIndex) => boundedString(id, `${label}.ids[${idIndex}]`)); + fileEvidence(observation.evidence, `${label}.evidence`); + } + return observations; +} + +function validateArtifacts(artifacts) { + exactFields(artifacts, ["frames", "screenshots"], "artifacts"); + const bounds = { + frames: [1, 32], + screenshots: [2, 32], + }; + for (const [kind, [minimum, maximum]] of Object.entries(bounds)) { + const entries = artifacts[kind]; + if (!Array.isArray(entries) || entries.length < minimum || entries.length > maximum) { + fail(`artifacts.${kind} is outside its bound`); + } + const expectedType = kind === "frames" ? "live" : "contract"; + entries.forEach((entry, index) => { + visualEvidence(entry, `artifacts.${kind}[${index}]`); + if (entry.evidenceType !== expectedType) fail(`artifacts.${kind} must be ${expectedType} evidence`); + }); + } + const viewports = new Set(artifacts.screenshots.map(({ viewport }) => viewport)); + if (!viewports.has("desktop") || !viewports.has("mobile")) { + fail("artifacts.screenshots must include desktop and mobile contract evidence"); + } + return artifacts; +} + +export function validateImagePublicationManifest(manifest) { + exactFields(manifest, ["schemaVersion", "source", "images"], "image publication manifest"); + if (manifest.schemaVersion !== "t4-cluster-images/1") fail("image publication schemaVersion is invalid"); + validateSource(manifest.source); + validateImageEntries(manifest.images, manifest.source.commit); + return manifest; +} + +export function validateProofManifest(manifest) { + exactFields( + manifest, + ["schemaVersion", "source", "images", "scenarios", "observations", "artifacts"], + "manifest", + ); + if (manifest.schemaVersion !== "t4-cluster-proof/1") fail("schemaVersion is invalid"); + validateSource(manifest.source); + validateImageEntries(manifest.images, manifest.source.commit); + validateScenarioEntries(manifest.scenarios); + validateObservationEntries(manifest.observations); + validateArtifacts(manifest.artifacts); + return manifest; +} + +function redactValue(value, key, depth) { + if (depth > MAX_DEPTH) fail("frame exceeded its depth bound"); + if (SENSITIVE_KEY_PATTERN.test(key)) return "[REDACTED]"; + if (value === null || typeof value === "boolean" || typeof value === "number") return value; + if (typeof value === "string") return value.length <= 16_384 ? value : fail("frame string exceeded its bound"); + if (Array.isArray(value)) { + if (value.length > MAX_ARRAY_ITEMS) fail("frame array exceeded its bound"); + return value.map((item) => redactValue(item, "", depth + 1)); + } + if (value && typeof value === "object") { + const entries = Object.entries(value); + if (entries.length > MAX_OBJECT_FIELDS) fail("frame object exceeded its bound"); + return Object.fromEntries(entries.map(([field, item]) => [field, redactValue(item, field, depth + 1)])); + } + fail("frame contained an unsupported value"); +} + +export function redactFrame(frame) { + object(frame, "frame"); + const redacted = redactValue(frame, "", 0); + if (Buffer.byteLength(JSON.stringify(redacted)) > MAX_FRAME_BYTES) fail("frame exceeded its byte bound"); + return redacted; +} + +export async function createFileEvidence( + absolutePath, + { artifactRoot = process.cwd(), artifactPrefix } = {}, +) { + const root = resolve(artifactRoot); + const target = resolve(absolutePath); + const relativePath = relative(root, target); + if (relativePath === "" || relativePath === ".." || relativePath.startsWith(`..${sep}`)) { + fail("evidence file must be inside its artifact root"); + } + const bytes = await readFile(target); + if (bytes.length === 0 || bytes.length > 64 * 1024 * 1024) { + fail("evidence file is empty or exceeded its size bound"); + } + const path = artifactPrefix + ? `${artifactPrefix.replace(/\/$/u, "")}/${basename(relativePath)}` + : relativePath.split(sep).join("/"); + if (!ARTIFACT_PATH_PATTERN.test(path) || path.includes("..")) { + fail("evidence path must stay inside artifacts/cluster-proof"); + } + return { + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + }; +} diff --git a/scripts/cluster-ci/publish-artifact.sh b/scripts/cluster-ci/publish-artifact.sh new file mode 100755 index 00000000..2deeb99f --- /dev/null +++ b/scripts/cluster-ci/publish-artifact.sh @@ -0,0 +1,55 @@ +#!/bin/sh +set -eu + +umask 077 + +mode=${1:-} +case "$mode" in + images) + repository_suffix=t4-cluster-image-evidence + tag=${CI_COMMIT_SHA:-} + artifact_type=application/vnd.t4.cluster.images.v1 + files="artifacts/cluster-proof/image-publication.json artifacts/cluster-proof/images/*" + ;; + proof) + repository_suffix=t4-cluster-proof + tag="${CI_COMMIT_SHA:-}-${CI_PIPELINE_NUMBER:-}" + artifact_type=application/vnd.t4.cluster.proof.v1 + files="artifacts/cluster-proof/manifest.json artifacts/cluster-proof/scenarios/* artifacts/cluster-proof/observations/* artifacts/cluster-proof/frames/* artifacts/cluster-proof/screenshots/*" + ;; + *) + echo "artifact mode must be images or proof" >&2 + exit 64 + ;; +esac + +case "${CI_COMMIT_SHA:-}" in + [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) ;; + *) + echo "CI_COMMIT_SHA must be an exact lowercase 40-character SHA" >&2 + exit 64 + ;; +esac +if [ "$mode" = proof ]; then + case "${CI_PIPELINE_NUMBER:-}" in + '' | *[!0-9]*) echo "CI_PIPELINE_NUMBER must be numeric" >&2; exit 64 ;; + esac +fi + +: "${HARBOR_REGISTRY:?HARBOR_REGISTRY is required}" +: "${HARBOR_PROJECT:?HARBOR_PROJECT is required}" +if [ "$HARBOR_REGISTRY" != "harbor.tailb18de3.ts.net" ]; then + echo "HARBOR_REGISTRY must be the exact HTTPS tailnet Harbor host" >&2 + exit 64 +fi +auth_dir=${T4_REGISTRY_AUTH_DIR:-${CI_WORKSPACE:-$PWD}/.cluster-ci/registry-auth} +test -r "$auth_dir/config.json" +export DOCKER_CONFIG="$auth_dir" +reference="$HARBOR_REGISTRY/$HARBOR_PROJECT/$repository_suffix:$tag" +# shellcheck disable=SC2086 +oras push \ + --artifact-type "$artifact_type" \ + --format json \ + "$reference" \ + $files > "artifacts/cluster-proof/$mode-oci-publication.json" +test -s "artifacts/cluster-proof/$mode-oci-publication.json" diff --git a/scripts/cluster-ci/readonly-cluster-proof.mjs b/scripts/cluster-ci/readonly-cluster-proof.mjs new file mode 100644 index 00000000..a2025e9e --- /dev/null +++ b/scripts/cluster-ci/readonly-cluster-proof.mjs @@ -0,0 +1,360 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +const MAX_LEASE_AGE_MS = 30_000; +const MAX_CLOCK_SKEW_MS = 5_000; +const NAMESPACE_PATTERN = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; +const DIGEST_PATTERN = /@sha256:[0-9a-f]{64}$/u; +const FORBIDDEN_SESSION_NODES = new Set(["k3s-worker-02", "k3s-worker-03"]); +const WORKLOAD_KINDS = new Set(["CronJob", "DaemonSet", "Deployment", "Job", "Pod", "StatefulSet"]); +const LEASE_NAME = "t4-cluster-operator.cluster.t4.dev"; +const DEPLOYMENTS = Object.freeze([ + { name: "t4-cluster-controller", component: "controller", container: "controller", replicas: 2, maxUnavailable: 0, minimumAvailable: 1 }, + { name: "t4-cluster-server", component: "server", container: "server", replicas: 3, maxUnavailable: 0, minimumAvailable: 2 }, +]); + +const REQUESTS = Object.freeze([ + ["deployments", ["get", "deployments", "-n", "$NAMESPACE", "-l", "app.kubernetes.io/part-of=t4-cluster", "-o", "json"]], + ["lease", ["get", "lease", LEASE_NAME, "-n", "$NAMESPACE", "-o", "json"]], + [ + "customresourcedefinitions", + [ + "get", + "customresourcedefinitions", + "t4clusterhosts.cluster.t4.dev", + "t4workspaces.cluster.t4.dev", + "t4sessions.cluster.t4.dev", + "-o", + "json", + ], + ], + ["t4clusterhosts", ["get", "t4clusterhosts", "-n", "$NAMESPACE", "-o", "json"]], + ["t4workspaces", ["get", "t4workspaces", "-n", "$NAMESPACE", "-o", "json"]], + ["t4sessions", ["get", "t4sessions", "-n", "$NAMESPACE", "-o", "json"]], + ["persistentvolumeclaims", ["get", "persistentvolumeclaims", "-n", "$NAMESPACE", "-l", "app.kubernetes.io/part-of=t4-cluster", "-o", "json"]], + ["pods", ["get", "pods", "-n", "$NAMESPACE", "-l", "app.kubernetes.io/part-of=t4-cluster", "-o", "json"]], + ["services", ["get", "services", "-n", "$NAMESPACE", "-l", "app.kubernetes.io/part-of=t4-cluster", "-o", "json"]], +]); + +function fail(message) { + throw new Error(`T4 read-only cluster proof ${message}`); +} + +function list(snapshot, key) { + const value = snapshot?.[key]; + if (!value || typeof value !== "object" || !Array.isArray(value.items) || value.items.length > 256) { + fail(`${key} response was malformed or exceeded its bound`); + } + return value.items; +} + +function exactNamed(items, name, label) { + const matches = items.filter(({ metadata }) => metadata?.name === name); + if (matches.length !== 1) fail(`expected exactly one ${label} named ${name}`); + return matches[0]; +} + +function positive(value) { + return Number.isSafeInteger(value) && value > 0; +} + +function observed(resource, label) { + if (!positive(resource.metadata?.generation) || resource.status?.observedGeneration !== resource.metadata.generation) { + fail(`${label} status does not observe metadata.generation exactly`); + } +} + +function label(resource, key, expected, resourceLabel) { + if (resource.metadata?.labels?.[key] !== expected) { + fail(`${resourceLabel} label ${key} is not exactly ${expected}`); + } +} + +function readyPod(pod, labelText) { + if ( + pod.status?.phase !== "Running" || + !Array.isArray(pod.status?.conditions) || + !pod.status.conditions.some(({ type, status }) => type === "Ready" && status === "True") + ) { + fail(`${labelText} is not Running and Ready`); + } +} + +function currentContainer(pod, name, labelText) { + const statuses = pod.status?.containerStatuses; + const matches = Array.isArray(statuses) ? statuses.filter((status) => status?.name === name) : []; + const imageDigest = matches[0]?.image?.match(/@(sha256:[0-9a-f]{64})$/u)?.[1]; + if ( + matches.length !== 1 || + matches[0].ready !== true || + !imageDigest || + !DIGEST_PATTERN.test(matches[0].imageID ?? "") || + !matches[0].imageID.endsWith(`@${imageDigest}`) + ) { + fail(`${labelText} has no exact ready ${name} container at its declared current digest`); + } + return matches[0]; +} + +function deploymentContract(deployment, contract) { + label(deployment, "app.kubernetes.io/part-of", "t4-cluster", `${contract.name} Deployment`); + label(deployment, "app.kubernetes.io/component", contract.component, `${contract.name} Deployment`); + observed(deployment, `${contract.name} Deployment`); + if ( + deployment.spec?.replicas !== contract.replicas || + deployment.spec?.strategy?.type !== "RollingUpdate" || + ![contract.maxUnavailable, String(contract.maxUnavailable)].includes(deployment.spec?.strategy?.rollingUpdate?.maxUnavailable) || + (deployment.status?.availableReplicas ?? 0) < contract.minimumAvailable + ) { + fail(`${contract.name} Deployment did not satisfy its exact HA rollout contract`); + } +} + +function defaultRunner(command, args) { + return execFileAsync(command, args, { + encoding: "utf8", + maxBuffer: MAX_RESPONSE_BYTES, + timeout: 30_000, + windowsHide: true, + }).then(({ stdout }) => stdout); +} + +export async function collectReadOnlyClusterSnapshot({ namespace, run = defaultRunner }) { + if (!NAMESPACE_PATTERN.test(namespace ?? "")) fail("namespace is invalid"); + const snapshot = {}; + for (const [key, template] of REQUESTS) { + const args = template.map((argument) => (argument === "$NAMESPACE" ? namespace : argument)); + const stdout = await run("kubectl", args); + if (typeof stdout !== "string" || Buffer.byteLength(stdout) > MAX_RESPONSE_BYTES) { + fail(`${key} response exceeded its byte bound`); + } + try { + snapshot[key] = JSON.parse(stdout); + } catch { + fail(`${key} response was not JSON`); + } + } + return snapshot; +} + +function validateCiMapping(sessions, expected) { + const mappings = sessions.filter(({ spec }) => { + const ci = spec?.ci; + return ( + ci && + typeof ci.repositoryId === "string" && ci.repositoryId.length > 0 && ci.repositoryId.length <= 128 && + typeof ci.ref === "string" && ci.ref.length > 0 && ci.ref.length <= 256 && + typeof ci.commit === "string" && /^[0-9a-f]{40}$/u.test(ci.commit) + ); + }); + if (mappings.length < 1) fail("no live T4Session carries an exact CI mapping"); + if (expected) { + const matches = mappings.filter(({ spec }) => + spec.ci.repositoryId === expected.repositoryId && + spec.ci.ref === expected.ref && + spec.ci.commit === expected.commit, + ); + if (matches.length !== 1) fail("live T4Session CI mapping does not exactly match this repository, ref, and commit"); + return matches; + } + return mappings; +} + +export function validateClusterSnapshot(snapshot, { now = Date.now(), ciMapping } = {}) { + const deployments = list(snapshot, "deployments"); + for (const contract of DEPLOYMENTS) { + deploymentContract(exactNamed(deployments, contract.name, `${contract.component} Deployment`), contract); + } + + const lease = snapshot?.lease; + const renewTime = Date.parse(lease?.spec?.renewTime ?? ""); + if ( + !lease || + lease.metadata?.name !== LEASE_NAME || + typeof lease.spec?.holderIdentity !== "string" || + lease.spec.holderIdentity.length < 1 || + lease.spec.holderIdentity.length > 253 || + !Number.isFinite(renewTime) || + renewTime > now + MAX_CLOCK_SKEW_MS || + now - renewTime > MAX_LEASE_AGE_MS + ) { + fail(`controller leader Lease ${LEASE_NAME} is not freshly held`); + } + + const crds = list(snapshot, "customresourcedefinitions"); + for (const plural of ["t4clusterhosts", "t4workspaces", "t4sessions"]) { + const crd = exactNamed(crds, `${plural}.cluster.t4.dev`, `${plural} CRD`); + const versions = Array.isArray(crd.spec?.versions) ? crd.spec.versions : []; + const version = versions.find(({ name }) => name === "v1alpha1"); + if ( + crd.spec?.group !== "cluster.t4.dev" || + crd.spec?.scope !== "Namespaced" || + crd.spec?.names?.plural !== plural || + version?.served !== true || + version?.storage !== true + ) { + fail(`${plural} CRD does not satisfy the exact v1alpha1 contract`); + } + } + + const hosts = list(snapshot, "t4clusterhosts"); + if (hosts.length < 1) fail("T4ClusterHost proof resource is absent"); + hosts.forEach((host) => observed(host, `T4ClusterHost ${host.metadata?.name ?? "unknown"}`)); + + const workspaces = list(snapshot, "t4workspaces"); + const sessions = list(snapshot, "t4sessions"); + const pvcs = list(snapshot, "persistentvolumeclaims"); + const pods = list(snapshot, "pods"); + if (workspaces.length < 1 || sessions.length < 1 || pvcs.length < 1) { + fail("workspace/session/storage proof resources are absent"); + } + for (const workspace of workspaces) { + const name = workspace.metadata?.name ?? "unknown"; + observed(workspace, `workspace ${name}`); + if (workspace.status?.phase !== "Ready" || !["Retain", "Delete"].includes(workspace.spec?.retentionPolicy)) { + fail(`workspace ${name} is not reconciled Ready`); + } + const pvcName = workspace.status?.pvcName; + if (typeof pvcName !== "string" || pvcName.length < 1 || pvcName.length > 63) { + fail(`workspace ${name} status.pvcName is invalid`); + } + const pvc = exactNamed(pvcs, pvcName, `workspace ${name} PVC`); + label(pvc, "app.kubernetes.io/part-of", "t4-cluster", `workspace ${name} PVC`); + label(pvc, "cluster.t4.dev/workspace", name, `workspace ${name} PVC`); + if (pvc.status?.phase !== "Bound") fail(`workspace ${name} PVC is not Bound`); + if ( + !Array.isArray(pvc.spec?.accessModes) || + pvc.spec.accessModes.length !== 1 || + pvc.spec.accessModes[0] !== "ReadWriteMany" + ) { + fail(`workspace ${name} PVC is not ReadWriteMany`); + } + if (typeof pvc.spec?.storageClassName !== "string" || pvc.spec.storageClassName.length === 0) { + fail(`workspace ${name} PVC has no StorageClass`); + } + } + + for (const session of sessions) { + const name = session.metadata?.name ?? "unknown"; + observed(session, `session ${name}`); + if (session.status?.phase !== "Running") fail(`session ${name} is not reconciled Running`); + if (!workspaces.some(({ metadata }) => metadata?.name === session.spec?.workspaceRef)) { + fail(`session ${name} references an unknown workspace`); + } + const podName = session.status?.podName; + if (typeof podName !== "string" || podName.length < 1 || podName.length > 63) { + fail(`session ${name} status.podName is invalid`); + } + const pod = exactNamed(pods, podName, `session ${name} Pod`); + label(pod, "app.kubernetes.io/name", "t4-session-runtime", `session ${name} Pod`); + label(pod, "app.kubernetes.io/part-of", "t4-cluster", `session ${name} Pod`); + label(pod, "cluster.t4.dev/session", podName, `session ${name} Pod`); + readyPod(pod, `session ${name} Pod`); + currentContainer(pod, "session-runtime", `session ${name} Pod`); + if (FORBIDDEN_SESSION_NODES.has(pod.spec?.nodeName)) { + fail(`durable session placement is invalid for ${podName}`); + } + } + validateCiMapping(sessions, ciMapping); + + for (const contract of DEPLOYMENTS) { + const matchingPods = pods.filter((pod) => + pod.metadata?.labels?.["app.kubernetes.io/part-of"] === "t4-cluster" && + pod.metadata?.labels?.["app.kubernetes.io/component"] === contract.component, + ); + if (matchingPods.length < contract.minimumAvailable) { + fail(`${contract.name} has too few exactly labelled pods`); + } + for (const pod of matchingPods) { + readyPod(pod, `${contract.name} Pod ${pod.metadata?.name ?? "unknown"}`); + currentContainer(pod, contract.container, `${contract.name} Pod ${pod.metadata?.name ?? "unknown"}`); + } + } + + const services = list(snapshot, "services"); + const serverService = exactNamed(services, "t4-cluster-server", "cluster-server Service"); + label(serverService, "app.kubernetes.io/part-of", "t4-cluster", "cluster-server Service"); + label(serverService, "app.kubernetes.io/component", "server", "cluster-server Service"); + const ports = new Map((serverService.spec?.ports ?? []).map(({ name, port }) => [name, port])); + if (ports.get("websocket") !== 8080 || ports.size !== 1) { + fail("cluster-server Service does not expose the exact websocket port"); + } + const metricsService = exactNamed(services, "t4-cluster-metrics", "cluster metrics Service"); + label(metricsService, "app.kubernetes.io/part-of", "t4-cluster", "cluster metrics Service"); + label(metricsService, "app.kubernetes.io/component", "server", "cluster metrics Service"); + const metricsPorts = new Map((metricsService.spec?.ports ?? []).map(({ name, port }) => [name, port])); + if (metricsPorts.get("metrics") !== 9090 || metricsPorts.size !== 1) { + fail("cluster metrics Service does not expose the exact admin metrics port"); + } + return snapshot; +} + +export function validateDefaultOffRender(documents) { + if (!Array.isArray(documents) || documents.length > 512) fail("default-off render was malformed or exceeded its bound"); + const workloads = documents.filter((document) => document && WORKLOAD_KINDS.has(document.kind)); + if (workloads.length > 0) { + fail(`default-off render created workload ${workloads[0].kind}/${workloads[0].metadata?.name ?? "unknown"}`); + } + return { clusterOperatorEnabled: false, workloadCount: 0 }; +} + +export function summarizeClusterSnapshot(snapshot, options = {}) { + validateClusterSnapshot(snapshot, options); + const deployments = list(snapshot, "deployments"); + const workspaces = list(snapshot, "t4workspaces"); + const sessions = list(snapshot, "t4sessions"); + const pvcs = list(snapshot, "persistentvolumeclaims"); + const allPods = list(snapshot, "pods"); + const lease = snapshot.lease; + const capturedAt = new Date(options.now ?? Date.now()).toISOString(); + return { + schemaVersion: "t4-cluster-readonly-snapshot/1", + observedAt: capturedAt, + deployments: deployments.map(({ metadata, spec, status }) => ({ + name: metadata.name, + component: metadata.labels["app.kubernetes.io/component"], + replicas: spec.replicas, + maxUnavailable: spec.strategy.rollingUpdate.maxUnavailable, + availableReplicas: status.availableReplicas, + generation: metadata.generation, + observedGeneration: status.observedGeneration, + })), + leader: { lease: lease.metadata.name, holderIdentity: lease.spec.holderIdentity, renewTime: lease.spec.renewTime }, + crds: list(snapshot, "customresourcedefinitions").map(({ metadata }) => metadata.name), + workspaces: workspaces.map(({ metadata, status }) => ({ name: metadata.name, phase: status.phase, pvc: status.pvcName })), + sessions: sessions.map(({ metadata, spec, status }) => ({ + name: metadata.name, + phase: status.phase, + pod: status.podName, + ...(spec.ci ? { ci: { repositoryId: spec.ci.repositoryId, ref: spec.ci.ref, commit: spec.ci.commit } } : {}), + })), + storage: pvcs.map(({ metadata, spec, status }) => ({ + name: metadata.name, + storageClassName: spec.storageClassName, + accessModes: spec.accessModes, + phase: status.phase, + capacity: status.capacity?.storage ?? "unknown", + })), + placements: allPods + .filter(({ metadata }) => metadata?.labels?.["cluster.t4.dev/session"]) + .map(({ metadata, spec }) => ({ name: metadata.name, node: spec.nodeName })), + images: allPods.flatMap(({ metadata, status }) => + (status?.containerStatuses ?? []).map(({ name, image, imageID, ready }) => ({ + pod: metadata.name, + labels: { + name: metadata.labels?.["app.kubernetes.io/name"], + component: metadata.labels?.["app.kubernetes.io/component"], + session: metadata.labels?.["cluster.t4.dev/session"], + partOf: metadata.labels?.["app.kubernetes.io/part-of"], + }, + phase: status.phase, + ready: ready === true, + container: name, + image, + imageID, + })), + ), + }; +} diff --git a/scripts/cluster-ci/validate-proof.mjs b/scripts/cluster-ci/validate-proof.mjs new file mode 100644 index 00000000..e52b0d41 --- /dev/null +++ b/scripts/cluster-ci/validate-proof.mjs @@ -0,0 +1,43 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + validateImagePublicationManifest, + validateProofManifest, +} from "./proof-contract.mjs"; + +export async function validateProofFile(path, { imagesOnly = false } = {}) { + let manifest; + try { + manifest = JSON.parse(await readFile(path, "utf8")); + } catch (error) { + throw new Error(`proof artifact ${path} is not valid JSON`, { cause: error }); + } + return imagesOnly + ? validateImagePublicationManifest(manifest) + : validateProofManifest(manifest); +} + +function parseArguments(args) { + const options = { imagesOnly: false }; + for (const argument of args) { + if (argument === "--images-only") options.imagesOnly = true; + else if (!options.path) options.path = argument; + else throw new Error(`unexpected argument ${argument}`); + } + if (!options.path) throw new Error("usage: validate-proof.mjs PATH [--images-only]"); + return options; +} + +const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)); +if (isMain) { + try { + const { path, imagesOnly } = parseArguments(process.argv.slice(2)); + await validateProofFile(path, { imagesOnly }); + console.log(`Validated ${imagesOnly ? "image publication" : "cluster proof"} artifact ${path}`); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/legacy-bridge-continuity.mjs b/scripts/legacy-bridge-continuity.mjs index 602b7f7a..af809704 100755 --- a/scripts/legacy-bridge-continuity.mjs +++ b/scripts/legacy-bridge-continuity.mjs @@ -16,7 +16,11 @@ import { tmpdir } from "node:os"; import { dirname, join, relative, resolve } from "node:path"; import { spawn, spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; -import { createOmpClient } from "../packages/client/src/index.ts"; +import { + clusterOperatorRequestedCapabilities, + clusterOperatorRequestedFeatures, + createOmpClient, +} from "../packages/client/src/index.ts"; import { UnixWebSocketTransport } from "../apps/desktop/src/transport.ts"; import { ADDITIVE_FEATURES, @@ -24,6 +28,11 @@ import { DEVICE_CAPABILITIES, } from "../packages/protocol/src/index.ts"; +const LEGACY_DEVICE_CAPABILITIES = + clusterOperatorRequestedCapabilities(DEVICE_CAPABILITIES); +const LEGACY_REQUESTED_FEATURES = + clusterOperatorRequestedFeatures(ADDITIVE_FEATURES); + const PROCESS_TIMEOUT_MS = 30_000; const RECONNECT_TIMEOUT_MS = 45_000; const POLL_MS = 100; @@ -462,9 +471,9 @@ class T4Probe { this.transport = auditedTransportFactory(socketPath, label, wireJournal); this.client = createOmpClient({ transport: this.transport.factory, - capabilities: DEVICE_CAPABILITIES, - requestedFeatures: ADDITIVE_FEATURES, - compatibilityRequestedFeatures: ADDITIVE_FEATURES.filter( + capabilities: LEGACY_DEVICE_CAPABILITIES, + requestedFeatures: LEGACY_REQUESTED_FEATURES, + compatibilityRequestedFeatures: LEGACY_REQUESTED_FEATURES.filter( (feature) => feature !== "prompt.images" && feature !== "transcript.images", ), client: { @@ -952,6 +961,7 @@ export async function runLegacyBridgeContinuity(argv = []) { "reconnected T4 client recovered no transcript rows", ); + const restartMark = t4Primary.mark(); const beforeRestartEpoch = t4Primary.client.snapshot().epoch; await stopProcess(primaryProcess, "primary appserver before restart"); running.delete("continuity-a"); @@ -977,6 +987,16 @@ export async function runLegacyBridgeContinuity(argv = []) { "second T4 reconnect after appserver restart", RECONNECT_TIMEOUT_MS, ); + const postRestartSnapshotEvent = await t4Primary.event( + "snapshot", + (payload) => payload.sessionId === target.sessionId, + restartMark, + RECONNECT_TIMEOUT_MS, + ); + const postRestartSnapshot = { + label: "post-restart transcript snapshot", + ...postRestartSnapshotEvent.payload, + }; const persistedStatus = await adminRequest(restarted, "POST", "/admin/test/status", { runId: profiles[0].runId, }); @@ -1060,11 +1080,6 @@ export async function runLegacyBridgeContinuity(argv = []) { }); assert.equal(secondRename.ok, true, "fresh revision did not recover control"); - const postRestartSnapshot = await attachSnapshot( - t4Primary, - target.sessionId, - "post-restart transcript snapshot", - ); const searchResponse = await t4Primary.client.command({ hostId: t4Primary.hostId(), command: "transcript.search", diff --git a/scripts/t4-maintainer-integration.test.mjs b/scripts/t4-maintainer-integration.test.mjs index 0da55734..23e74cdc 100644 --- a/scripts/t4-maintainer-integration.test.mjs +++ b/scripts/t4-maintainer-integration.test.mjs @@ -25,10 +25,10 @@ const repoRoot = resolve(import.meta.dirname, ".."); const deployScript = resolve(repoRoot, "ops/t4-maintainer/deploy-local.sh"); const runnerScript = resolve(repoRoot, "ops/t4-maintainer/run.sh"); const bashPath = "/bin/bash"; -// Full-suite contention can push a successful convergence run past 30 seconds -// on macOS. Keep this above the production process boundary so the fixture +// Full-suite contention on shared CI and macOS can push a successful convergence +// run past one minute. Keep this above the production process boundary so the fixture // reports the child result instead of a test-harness timeout. -const integrationProcessTimeoutMs = 60_000; +const integrationProcessTimeoutMs = 180_000; const upstreamCommit = "a".repeat(40); const integrationCommit = "b".repeat(40); const t4Commit = "c".repeat(40); @@ -706,7 +706,7 @@ SH --arg allowedOrigin "$origin" \ --argjson port "$port" \ --arg appSocket "$app_socket" \ - --arg label "$label" \ + --arg hostLabel "$label" \ --arg deploymentIdentity "$deployment_identity" \ --arg webRoot "$web_root" \ --argjson profileRoutes "$profile_routes" \ @@ -719,7 +719,7 @@ SH allowedOrigin:$allowedOrigin, port:$port, appSocket:$appSocket, - label:$label, + "label":$hostLabel, deploymentIdentity:$deploymentIdentity } + (if $profileRoutes == null then {} else { profileRoutes:$profileRoutes, @@ -972,7 +972,7 @@ jq -n \ --arg allowedOrigin "$gateway_origin" \ --argjson port "$gateway_port" \ --arg appSocket "$gateway_socket" \ - --arg label "$gateway_label" \ + --arg hostLabel "$gateway_label" \ --arg nodeExecutable "$node_executable" \ --arg gatewayScript "$gateway_script" \ --arg webRoot "$web_root" \ @@ -982,7 +982,7 @@ jq -n \ allowedOrigin: $allowedOrigin, port: $port, appSocket: $appSocket, - label: $label, + "label": $hostLabel, nodeExecutable: $nodeExecutable, gatewayScript: $gatewayScript, webRoot: $webRoot, @@ -1060,7 +1060,7 @@ jq -n \ allowedOrigin: $gateway_origin, port: $gateway_port, appSocket: $gateway_socket, - label: $gateway_label, + "label": $gateway_label, nodeExecutable: $node_executable, deploymentIdentity: $deployment_identity, artifacts: { diff --git a/scripts/tailnet-gateway.mjs b/scripts/tailnet-gateway.mjs index 59512a9f..1f93da5c 100644 --- a/scripts/tailnet-gateway.mjs +++ b/scripts/tailnet-gateway.mjs @@ -159,6 +159,28 @@ export function websocketUrlForOrigin(origin) { url.protocol = "wss:"; return url.toString(); } +export function normalizeClusterWebSocketUrl(value) { + const text = requiredText(value, "T4_CLUSTER_WS_URL"); + let url; + try { + url = new URL(text); + } catch { + throw new Error("T4_CLUSTER_WS_URL must be one secure WSS cluster target"); + } + if ( + url.protocol !== "wss:" || + url.username !== "" || + url.password !== "" || + url.port !== "" || + url.pathname !== "/v1/ws" || + url.search !== "" || + url.hash !== "" || + !url.hostname.endsWith(".ts.net") + ) { + throw new Error("T4_CLUSTER_WS_URL must be one credential-free secure WSS cluster target"); + } + return url.toString(); +} function safeJson(value) { return JSON.stringify(value) @@ -172,9 +194,13 @@ export function injectBackendConfig(indexHtml, options) { const marker = ""; const offset = indexHtml.indexOf(marker); if (offset === -1) throw new Error("web index is missing "); + const clusterOperatorEnabled = options.clusterOperatorEnabled === true; const payload = safeJson({ - wsUrl: websocketUrlForOrigin(options.allowedOrigin), + wsUrl: clusterOperatorEnabled + ? normalizeClusterWebSocketUrl(options.clusterWsUrl) + : websocketUrlForOrigin(options.allowedOrigin), label: requiredText(options.label, "gateway label", 128), + ...(clusterOperatorEnabled ? { clusterOperatorEnabled: true } : {}), }); const script = ` \n`; return `${indexHtml.slice(0, offset)}${script}${indexHtml.slice(offset)}`; @@ -202,8 +228,8 @@ export function safeStaticPath(webRoot, pathname) { return candidate === root || candidate.startsWith(`${root}${sep}`) ? candidate : undefined; } -function gatewayCsp(allowedOrigin) { - const websocketOrigin = new URL(websocketUrlForOrigin(allowedOrigin)).origin; +function gatewayCsp(allowedOrigin, clusterWsUrl) { + const websocketOrigin = new URL(clusterWsUrl ?? websocketUrlForOrigin(allowedOrigin)).origin; return [ "default-src 'self'", "script-src 'self'", @@ -219,8 +245,8 @@ function gatewayCsp(allowedOrigin) { ].join("; "); } -function applySecurityHeaders(response, allowedOrigin) { - response.setHeader("Content-Security-Policy", gatewayCsp(allowedOrigin)); +function applySecurityHeaders(response, allowedOrigin, clusterWsUrl) { + response.setHeader("Content-Security-Policy", gatewayCsp(allowedOrigin, clusterWsUrl)); response.setHeader("Permissions-Policy", "camera=(), geolocation=(), microphone=(), payment=(), usb=()"); response.setHeader("Referrer-Policy", "no-referrer"); response.setHeader("X-Content-Type-Options", "nosniff"); @@ -491,6 +517,11 @@ export async function startTailnetGateway(input) { nativeAllowedOrigins: normalizeNativeAllowedOrigins(input.nativeAllowedOrigins), label: input.label ?? "OMP on this Tailnet host", deploymentIdentity: normalizeDeploymentIdentity(input.deploymentIdentity), + clusterOperatorEnabled: input.clusterOperatorEnabled === true, + clusterWsUrl: + input.clusterOperatorEnabled === true + ? normalizeClusterWebSocketUrl(input.clusterWsUrl) + : undefined, heartbeatIntervalMs: input.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS, profiles: normalizeProfileRoutes(input.profileRoutes ?? input.profiles ?? []), startProfiles: input.startProfiles === true || input.enableProfileStarts === true, @@ -527,7 +558,7 @@ export async function startTailnetGateway(input) { }); const server = createServer((request, response) => { void (async () => { - applySecurityHeaders(response, options.allowedOrigin); + applySecurityHeaders(response, options.allowedOrigin, options.clusterWsUrl); const pathname = requestPath(request.url); if (pathname === "/healthz") { const [web, resolvedAppSocket] = await Promise.all([ @@ -657,6 +688,17 @@ export function optionsFromEnvironment(environment = process.env) { throw new Error("T4_PROFILE_ROUTES must be valid JSON"); } } + const clusterOperatorEnabled = environment.T4_CLUSTER_OPERATOR_ENABLED === "true"; + if ( + environment.T4_CLUSTER_OPERATOR_ENABLED !== undefined && + environment.T4_CLUSTER_OPERATOR_ENABLED !== "false" && + !clusterOperatorEnabled + ) { + throw new Error("T4_CLUSTER_OPERATOR_ENABLED must be true or false"); + } + const clusterWsUrl = clusterOperatorEnabled + ? normalizeClusterWebSocketUrl(environment.T4_CLUSTER_WS_URL) + : undefined; return { webRoot: environment.T4_WEB_ROOT ?? resolve(scriptDirectory, "..", "apps", "web", "dist"), appSocket: environment.T4_APP_SERVER_SOCKET ?? defaultSocketPath(environment), @@ -669,6 +711,8 @@ export function optionsFromEnvironment(environment = process.env) { : environment.T4_NATIVE_ALLOWED_ORIGINS.split(","), label: environment.T4_HOST_LABEL ?? "OMP on this Tailnet host", deploymentIdentity: environment.T4_DEPLOYMENT_IDENTITY, + clusterOperatorEnabled, + clusterWsUrl, profileRoutes, startProfiles: environment.T4_ENABLE_PROFILE_STARTS === "1", }; diff --git a/scripts/tailnet-gateway.test.mjs b/scripts/tailnet-gateway.test.mjs index 7373e33f..d36810e2 100644 --- a/scripts/tailnet-gateway.test.mjs +++ b/scripts/tailnet-gateway.test.mjs @@ -12,6 +12,7 @@ import { CAPACITOR_NATIVE_ORIGINS, cacheControlForStaticPath, injectBackendConfig, + normalizeClusterWebSocketUrl, normalizeAllowedOrigin, normalizeDeploymentIdentity, normalizeNativeAllowedOrigins, @@ -164,6 +165,60 @@ test("backend injection is explicit, credential-free, and script-safe", () => { assert.doesNotMatch(injected, /token|password|credential/iu); }); +test("cluster gateway configuration is default-off and adds no implicit target", () => { + const options = optionsFromEnvironment({ + T4_ALLOWED_ORIGIN: ALLOWED_ORIGIN, + T4_DEPLOYMENT_IDENTITY: DEPLOYMENT_IDENTITY, + XDG_RUNTIME_DIR: "/run/user/1000", + }); + assert.equal(options.clusterOperatorEnabled, false); + assert.equal(options.clusterWsUrl, undefined); + + const injected = injectBackendConfig("", { + allowedOrigin: ALLOWED_ORIGIN, + label: "Ordinary host", + }); + const payload = JSON.parse(/