From db580441d07916fda7149cee47bd610ba295a8cb Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Mon, 29 Jun 2026 15:21:48 -0400 Subject: [PATCH 01/15] ci(longhaul): add image build, deploy, and monitor workflows Introduces the GitHub Actions plumbing for the long-haul test driver that landed in PR #405. Three workflows + two deploy manifests: * .github/workflows/longhaul-image-build.yml Build/push test/longhaul image to GHCR on main push and on demand. Tags every run as :sha- (immutable) plus :main. * .github/workflows/longhaul-deploy.yml Roll an image onto the long-haul AKS cluster. Auto-triggered after a successful image build (pins to :sha-) and via workflow_dispatch for rollbacks. Uses a namespace-scoped kubeconfig in the LONGHAUL_KUBECONFIG secret. * .github/workflows/longhaul-monitor.yaml Hourly health poll: Deployment ready, report ConfigMap fresh (<=2h), test result != FAIL. Auto-upgrade and DocumentDB version publishing are intentionally left out and will land in a separate upgrade PR. * test/longhaul/deploy/deployment.yaml Single-replica Deployment + ConfigMap. Image fields templated (__OWNER__/__IMAGE_TAG__) for the deploy workflow to substitute. Aligned with the post-PR-405 env-var surface (LONGHAUL_DOCUMENTDB_URI, no NUM_VERIFIERS) and the credential secret name documented in test/longhaul/README.md (longhaul-documentdb-credentials with a uri key). * test/longhaul/deploy/rbac.yaml Namespace-scoped ServiceAccount/Role/RoleBinding (pods, dbs, configmaps) plus a ClusterRole for metrics.k8s.io. Splits PR #348 part 3 of 5. Operator/DocumentDB auto-upgrade plus post-upgrade verification follow in PR-4. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/longhaul-deploy.yml | 155 ++++++++++++++ .github/workflows/longhaul-image-build.yml | 105 +++++++++ .github/workflows/longhaul-monitor.yaml | 236 +++++++++++++++++++++ test/longhaul/deploy/deployment.yaml | 94 ++++++++ test/longhaul/deploy/rbac.yaml | 81 +++++++ 5 files changed, 671 insertions(+) create mode 100644 .github/workflows/longhaul-deploy.yml create mode 100644 .github/workflows/longhaul-image-build.yml create mode 100644 .github/workflows/longhaul-monitor.yaml create mode 100644 test/longhaul/deploy/deployment.yaml create mode 100644 test/longhaul/deploy/rbac.yaml diff --git a/.github/workflows/longhaul-deploy.yml b/.github/workflows/longhaul-deploy.yml new file mode 100644 index 000000000..442ff8437 --- /dev/null +++ b/.github/workflows/longhaul-deploy.yml @@ -0,0 +1,155 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: LONGHAUL - Deploy Test Driver to AKS + +# Rolls a longhaul-test image onto the long-haul AKS cluster. +# +# Auth: long-lived ServiceAccount-token kubeconfig stored as the +# LONGHAUL_KUBECONFIG repo secret. Scoped to namespace +# documentdb-test-ns via Role/RoleBinding (see +# test/longhaul/deploy/rbac.yaml). No Azure auth on the runner. +# +# Triggers: +# * workflow_run — auto-deploys after a successful build on main, pinning +# to the immutable :sha- tag built by that run. +# * workflow_dispatch — manual roll/rollback to a specific tag (e.g. an +# earlier :sha-, or :main for a quick smoke). + +on: + workflow_run: + workflows: ["LONGHAUL - Build Test Driver Image"] + types: [completed] + branches: [main] + + workflow_dispatch: + inputs: + image_tag: + description: 'Image tag to deploy (e.g. sha-5c30d7b or main).' + required: true + default: 'main' + +permissions: + contents: read + +env: + NAMESPACE: documentdb-test-ns + DEPLOYMENT: longhaul-test + CONTAINER: driver + +jobs: + deploy: + name: Deploy to AKS + runs-on: ubuntu-22.04 + # When triggered by workflow_run, only proceed if the upstream build succeeded. + # workflow_dispatch always proceeds. + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Write kubeconfig + run: | + set -euo pipefail + install -m 700 -d "$RUNNER_TEMP/.kube" + KCFG="$RUNNER_TEMP/.kube/config" + printf '%s' "${{ secrets.LONGHAUL_KUBECONFIG }}" > "$KCFG" + chmod 600 "$KCFG" + echo "KUBECONFIG=$KCFG" >> "$GITHUB_ENV" + + - name: Verify cluster access + run: | + set -euo pipefail + kubectl version --client=false --output=yaml | head -20 || true + kubectl -n "$NAMESPACE" get all || true + + - name: Compute image ref + id: img + env: + # workflow_dispatch path: maintainer-supplied tag. + # workflow_run path: derive immutable :sha- from the upstream + # build's head SHA, so we deploy exactly what was just built. + DISPATCH_TAG: ${{ github.event.inputs.image_tag }} + UPSTREAM_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + set -euo pipefail + OWNER_LC="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" + REPO_NAME="$(basename "${{ github.repository }}")" + + if [ "${{ github.event_name }}" = "workflow_run" ]; then + SHORT_SHA="$(echo "${UPSTREAM_SHA}" | cut -c1-7)" + TAG="sha-${SHORT_SHA}" + else + TAG="${DISPATCH_TAG}" + fi + + IMAGE="ghcr.io/${OWNER_LC}/${REPO_NAME}/longhaul-test:${TAG}" + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + echo "image=${IMAGE}" >> "$GITHUB_OUTPUT" + echo "Resolved image: ${IMAGE}" + + - name: Apply Deployment manifest + # The Deployment manifest is the only thing the deploy workflow + # owns. Bootstrap resources (namespace, DocumentDB CR, credentials + # secret, driver ServiceAccount/Role/ClusterRole) live in + # setup.yaml + rbac.yaml and are applied once by a cluster admin + # (the deployer SA is intentionally namespace-scoped and cannot + # create those). + run: | + set -euo pipefail + OWNER_LC="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" + TAG='${{ steps.img.outputs.tag }}' + + sed -e "s|__OWNER__|${OWNER_LC}|g" \ + -e "s|__IMAGE_TAG__|${TAG}|g" \ + test/longhaul/deploy/deployment.yaml | kubectl apply -f - + + - name: Roll image + run: | + set -euo pipefail + IMAGE='${{ steps.img.outputs.image }}' + + # `set image` is a no-op if the manifest already pinned this image + # (apply step above), but harmless. The rollout restart below is + # what guarantees a re-pull when the tag is mutable (e.g. :main). + kubectl -n "$NAMESPACE" set image \ + "deployment/${DEPLOYMENT}" \ + "${CONTAINER}=${IMAGE}" \ + --record=false || true + + # Force a restart so the same tag (e.g. :main) re-pulls if image changed. + kubectl -n "$NAMESPACE" rollout restart "deployment/${DEPLOYMENT}" + + - name: Wait for rollout + run: | + set -euo pipefail + kubectl -n "$NAMESPACE" rollout status \ + "deployment/${DEPLOYMENT}" --timeout=5m + + - name: Pod status + if: always() + run: | + set -euo pipefail + kubectl -n "$NAMESPACE" get deployment "${DEPLOYMENT}" -o wide || true + kubectl -n "$NAMESPACE" get pods -l app.kubernetes.io/name=longhaul-test -o wide || true + POD="$(kubectl -n "$NAMESPACE" get pod -l app.kubernetes.io/name=longhaul-test \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + if [ -n "$POD" ]; then + echo "::group::Recent driver logs ($POD)" + kubectl -n "$NAMESPACE" logs "$POD" --tail=80 || true + echo "::endgroup::" + fi + + - name: Summary + if: always() + run: | + { + echo "## Long-haul deploy" + echo "" + echo "| Field | Value |" + echo "|-------|-------|" + echo "| Trigger | \`${{ github.event_name }}\` |" + echo "| Image | \`${{ steps.img.outputs.image }}\` |" + echo "| Namespace | \`$NAMESPACE\` |" + echo "| Deployment | \`$DEPLOYMENT\` |" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/longhaul-image-build.yml b/.github/workflows/longhaul-image-build.yml new file mode 100644 index 000000000..d141241e8 --- /dev/null +++ b/.github/workflows/longhaul-image-build.yml @@ -0,0 +1,105 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: LONGHAUL - Build Test Driver Image + +# Builds the long-haul test driver image (test/longhaul/Dockerfile) and pushes +# it to GHCR at: +# ghcr.io//documentdb-kubernetes-operator/longhaul-test: +# +# Auth: built-in GITHUB_TOKEN (no extra secrets needed). Same pattern as the +# operator/sidecar and database image build workflows in this repo. +# +# Tags pushed every run: +# :sha- immutable, deployable +# :main rolling pointer to the latest main build (convenience only; +# pin to :sha- for actual deployments) +# +# This workflow does NOT deploy. Long-haul deploy is a separate workflow +# (workflow_dispatch) that uses these tags. + +on: + push: + branches: + - main + paths: + - 'test/longhaul/**' + - '.github/workflows/longhaul-image-build.yml' + + workflow_dispatch: + inputs: + extra_tag: + description: 'Optional additional tag (e.g. dev-myfeature). Leave blank for none.' + required: false + default: '' + +permissions: + contents: read + packages: write # push to GHCR + +env: + IMAGE_NAME: longhaul-test + CONTEXT: test/longhaul + DOCKERFILE: test/longhaul/Dockerfile + +jobs: + build-and-push: + name: Build and push longhaul image + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Compute image refs + id: refs + run: | + set -euo pipefail + OWNER_LC="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" + REPO_NAME="$(basename "${{ github.repository }}")" + IMG="ghcr.io/${OWNER_LC}/${REPO_NAME}/${IMAGE_NAME}" + SHORT_SHA="$(echo "${GITHUB_SHA}" | cut -c1-7)" + + TAGS="${IMG}:sha-${SHORT_SHA},${IMG}:main" + if [ -n "${{ github.event.inputs.extra_tag }}" ]; then + TAGS="${TAGS},${IMG}:${{ github.event.inputs.extra_tag }}" + fi + + echo "image=${IMG}" >> "$GITHUB_OUTPUT" + echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + echo "tags=${TAGS}" >> "$GITHUB_OUTPUT" + echo "Will push: ${TAGS}" + + - name: Login to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ${{ env.CONTEXT }} + file: ${{ env.DOCKERFILE }} + platforms: linux/amd64 + push: true + tags: ${{ steps.refs.outputs.tags }} + provenance: false + cache-from: type=registry,ref=${{ steps.refs.outputs.image }}:buildcache + cache-to: type=registry,ref=${{ steps.refs.outputs.image }}:buildcache,mode=max + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.created=${{ github.event.repository.updated_at }} + + - name: Summary + run: | + { + echo "## Long-haul image pushed" + echo "" + echo "| Tag | Reference |" + echo "|-----|-----------|" + echo "| Immutable | \`${{ steps.refs.outputs.image }}:sha-${{ steps.refs.outputs.short_sha }}\` |" + echo "| Rolling | \`${{ steps.refs.outputs.image }}:main\` |" + echo "" + echo "Deploy by setting the Job spec image to the immutable tag." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/longhaul-monitor.yaml b/.github/workflows/longhaul-monitor.yaml new file mode 100644 index 000000000..c69f746f0 --- /dev/null +++ b/.github/workflows/longhaul-monitor.yaml @@ -0,0 +1,236 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Long Haul Test Monitor +# +# Polls the AKS cluster every hour to: +# 1. Check health: Job status, report staleness, test result +# 2. Auto-upgrade operator: If healthy, upgrade the operator to the latest +# version published to GHCR (via Helm). +# 3. Publish desired DocumentDB version: Write the latest published DocumentDB +# tag to the `longhaul-versions` ConfigMap. This workflow does NOT upgrade +# DocumentDB itself — the long-haul test driver reads the ConfigMap and +# performs the upgrade in-band as a load-aware operation, so continuous +# writers/verifiers can catch any data-integrity issues caused by it. +# +# Fails if: Job failed, ConfigMap reports FAIL, report is stale (>2h), or Job not found. +# +name: "Long Haul Monitor" + +on: + schedule: + - cron: "0 * * * *" # Every hour (matches LONGHAUL_REPORT_INTERVAL) + workflow_dispatch: {} # Allow manual trigger + +env: + NAMESPACE: documentdb-test-ns + DEPLOYMENT_NAME: longhaul-test + OPERATOR_NAMESPACE: documentdb-operator + +jobs: + check-longhaul: + runs-on: ubuntu-latest + steps: + - name: Set up kubeconfig + run: | + set -euo pipefail + install -m 700 -d "$RUNNER_TEMP/.kube" + KCFG="$RUNNER_TEMP/.kube/config" + # Secret is raw kubeconfig YAML (not base64). Same format the + # deploy workflow uses; keep these in sync. + printf '%s' "${{ secrets.LONGHAUL_KUBECONFIG }}" > "$KCFG" + chmod 600 "$KCFG" + echo "KUBECONFIG=$KCFG" >> "$GITHUB_ENV" + + - name: Check Deployment exists and is healthy + id: deploy-check + run: | + set -euo pipefail + # Returns empty string if the Deployment doesn't exist. + READY=$(kubectl get deployment "${{ env.DEPLOYMENT_NAME }}" -n "${{ env.NAMESPACE }}" \ + -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true) + DESIRED=$(kubectl get deployment "${{ env.DEPLOYMENT_NAME }}" -n "${{ env.NAMESPACE }}" \ + -o jsonpath='{.spec.replicas}' 2>/dev/null || true) + # Detect "rollout is failing" via the Progressing condition. + PROGRESSING_REASON=$(kubectl get deployment "${{ env.DEPLOYMENT_NAME }}" -n "${{ env.NAMESPACE }}" \ + -o jsonpath='{.status.conditions[?(@.type=="Progressing")].reason}' 2>/dev/null || true) + + if [ -z "$DESIRED" ]; then + echo "exists=false" >> "$GITHUB_OUTPUT" + else + echo "exists=true" >> "$GITHUB_OUTPUT" + fi + echo "ready=${READY:-0}" >> "$GITHUB_OUTPUT" + echo "desired=${DESIRED:-0}" >> "$GITHUB_OUTPUT" + echo "progressing_reason=${PROGRESSING_REASON}" >> "$GITHUB_OUTPUT" + + - name: Check ConfigMap report + id: report-check + run: | + RESULT=$(kubectl get configmap longhaul-report -n ${{ env.NAMESPACE }} -o jsonpath='{.data.result}' 2>/dev/null || echo "UNKNOWN") + echo "result=$RESULT" >> "$GITHUB_OUTPUT" + + LAST_UPDATED=$(kubectl get configmap longhaul-report -n ${{ env.NAMESPACE }} -o jsonpath='{.data.last-updated}' 2>/dev/null || echo "") + echo "last_updated=$LAST_UPDATED" >> "$GITHUB_OUTPUT" + + REPORT=$(kubectl get configmap longhaul-report -n ${{ env.NAMESPACE }} -o jsonpath='{.data.latest-report}' 2>/dev/null || echo "No report available") + echo "report<> "$GITHUB_OUTPUT" + echo "$REPORT" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + - name: Evaluate status + run: | + set -euo pipefail + echo "=== Long Haul Test Status ===" + echo "Deployment exists: ${{ steps.deploy-check.outputs.exists }}" + echo "Ready / desired: ${{ steps.deploy-check.outputs.ready }} / ${{ steps.deploy-check.outputs.desired }}" + echo "Progressing reason: ${{ steps.deploy-check.outputs.progressing_reason }}" + echo "Report result: ${{ steps.report-check.outputs.result }}" + echo "Last updated: ${{ steps.report-check.outputs.last_updated }}" + echo "" + echo "=== Latest Report ===" + echo "${{ steps.report-check.outputs.report }}" + echo "" + + # Fail if Deployment doesn't exist — test should always be running. + # If you need to fix/redeploy the long-haul test, disable this workflow first: + # gh workflow disable "Long Haul Monitor" + # Re-enable after redeployment: + # gh workflow enable "Long Haul Monitor" + if [ "${{ steps.deploy-check.outputs.exists }}" != "true" ]; then + echo "::error::No long-haul Deployment found — test is not running. Disable this workflow while fixing." + exit 1 + fi + + # Fail if rollout has stalled (e.g. ImagePullBackOff, CrashLoopBackOff + # past the progress deadline). + if [ "${{ steps.deploy-check.outputs.progressing_reason }}" = "ProgressDeadlineExceeded" ]; then + echo "::error::Long haul Deployment rollout stalled (ProgressDeadlineExceeded)" + exit 1 + fi + + # Fail if no replicas are ready. + if [ "${{ steps.deploy-check.outputs.ready }}" -lt 1 ]; then + echo "::error::Long haul Deployment has 0 ready replicas" + exit 1 + fi + + # Fail if report is stale (>2h old) — indicates the test may be hung + LAST="${{ steps.report-check.outputs.last_updated }}" + if [ -n "$LAST" ] && [ "$LAST" != "UNKNOWN" ]; then + LAST_EPOCH=$(date -d "$LAST" +%s 2>/dev/null || echo "0") + NOW_EPOCH=$(date +%s) + AGE_SECONDS=$((NOW_EPOCH - LAST_EPOCH)) + if [ "$AGE_SECONDS" -gt 7200 ]; then + echo "::error::Long haul report is stale (last updated ${AGE_SECONDS}s ago, threshold 7200s). Test may be hung." + exit 1 + fi + fi + + # Fail if ConfigMap reports failure + if [ "${{ steps.report-check.outputs.result }}" = "FAIL" ]; then + echo "::error::Long haul test reported FAIL in ConfigMap" + exit 1 + fi + + echo "✅ Long haul test is healthy" + + upgrade-if-needed: + runs-on: ubuntu-latest + needs: check-longhaul # Only upgrade if cluster is healthy + steps: + - uses: actions/checkout@v4 + + - name: Set up kubeconfig + run: | + set -euo pipefail + install -m 700 -d "$RUNNER_TEMP/.kube" + KCFG="$RUNNER_TEMP/.kube/config" + printf '%s' "${{ secrets.LONGHAUL_KUBECONFIG }}" > "$KCFG" + chmod 600 "$KCFG" + echo "KUBECONFIG=$KCFG" >> "$GITHUB_ENV" + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Check and upgrade operator + env: + GH_TOKEN: ${{ github.token }} + run: | + # Get latest published operator tag from GHCR (strict semver only) + EXPECTED=$(gh api /orgs/documentdb/packages/container/documentdb-operator/versions \ + --jq '[.[].metadata.container.tags[] | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))] | sort_by(. | split(".") | map(tonumber)) | last' 2>/dev/null || echo "") + + if [ -z "$EXPECTED" ]; then + # Fallback to Chart.yaml if GHCR query fails + EXPECTED=$(grep 'appVersion:' operator/documentdb-helm-chart/Chart.yaml | awk '{print $2}' | tr -d '"') + echo "ℹ️ GHCR query failed, using Chart.yaml: $EXPECTED" + fi + + # Running version from cluster + RUNNING=$(kubectl get deployment -n ${{ env.OPERATOR_NAMESPACE }} -l app.kubernetes.io/name=documentdb-operator \ + -o jsonpath='{.items[0].spec.template.spec.containers[0].image}' 2>/dev/null | sed 's/.*://') + + echo "Operator: expected=$EXPECTED running=$RUNNING" + + if [ -z "$RUNNING" ]; then + echo "::warning::Could not determine running operator version" + elif [ "$RUNNING" != "$EXPECTED" ]; then + echo "⬆️ Upgrading operator: $RUNNING → $EXPECTED" + + # Helm does NOT upgrade CRDs automatically. Apply them first so + # the new operator code does not reference fields the cluster's + # CRD schema lacks (e.g., schemaVersion). + echo "📦 Applying CRDs from chart..." + kubectl apply -f operator/documentdb-helm-chart/crds/ + + helm upgrade documentdb-operator operator/documentdb-helm-chart \ + -n ${{ env.OPERATOR_NAMESPACE }} \ + --set image.tag="$EXPECTED" \ + --wait --timeout 5m + + # Verify rollout completed + kubectl rollout status deployment -n ${{ env.OPERATOR_NAMESPACE }} -l app.kubernetes.io/name=documentdb-operator --timeout=3m + ACTUAL=$(kubectl get deployment -n ${{ env.OPERATOR_NAMESPACE }} -l app.kubernetes.io/name=documentdb-operator \ + -o jsonpath='{.items[0].spec.template.spec.containers[0].image}' | sed 's/.*://') + if [ "$ACTUAL" = "$EXPECTED" ]; then + echo "✅ Operator upgraded and verified: $ACTUAL" + else + echo "::error::Operator upgrade verification failed: expected=$EXPECTED actual=$ACTUAL" + exit 1 + fi + else + echo "✅ Operator is up to date ($RUNNING)" + fi + + - name: Publish desired DocumentDB version + env: + GH_TOKEN: ${{ github.token }} + run: | + # Determine the latest published DocumentDB tag and publish it to a + # ConfigMap. The long-haul test reads this and performs the actual + # upgrade as a load-aware operation (so continuous verifiers can + # detect any data-integrity issues caused by the upgrade). + EXPECTED=$(gh api '/orgs/documentdb/packages/container/documentdb-kubernetes-operator%2Fdocumentdb/versions' \ + --jq '[.[].metadata.container.tags[] | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))] | sort_by(. | split(".") | map(tonumber)) | last' 2>/dev/null || echo "") + + if [ -z "$EXPECTED" ]; then + EXPECTED=$(grep 'documentDbVersion:' operator/documentdb-helm-chart/values.yaml | awk '{print $2}' | tr -d '"') + echo "ℹ️ GHCR query failed, using values.yaml: $EXPECTED" + fi + + if [ -z "$EXPECTED" ]; then + echo "::warning::Could not determine desired DocumentDB version; skipping publish" + exit 0 + fi + + echo "📣 Publishing desired DocumentDB version: $EXPECTED" + kubectl create configmap longhaul-versions \ + -n ${{ env.NAMESPACE }} \ + --from-literal=desired-documentdb-version="$EXPECTED" \ + --from-literal=last-updated="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --dry-run=client -o yaml | kubectl apply -f - + + RUNNING=$(kubectl get documentdb documentdb-cluster -n ${{ env.NAMESPACE }} \ + -o jsonpath='{.status.documentDBImage}' 2>/dev/null | sed 's/.*://') + echo "DocumentDB: desired=$EXPECTED running=$RUNNING (test will upgrade if drifted)" diff --git a/test/longhaul/deploy/deployment.yaml b/test/longhaul/deploy/deployment.yaml new file mode 100644 index 000000000..88319de49 --- /dev/null +++ b/test/longhaul/deploy/deployment.yaml @@ -0,0 +1,94 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Long Haul Test - Kubernetes Deployment +# +# Why a Deployment (not a Job)? +# A Job's pod spec is immutable, so rolling a new image requires +# delete-and-recreate. Using Deployment(replicas=1) lets the deploy +# workflow do `kubectl set image deployment/longhaul-test ...` for +# one-line image rolls. The driver itself is restart-safe: it persists +# journal/checkpoint state to PVC and resumes from it. +# +# Strategy: Recreate (single-replica stateful driver — never run two +# pods concurrently against the same PVC). +# +# Failure semantics: on critical failure (data loss, policy violation) +# the driver exits non-zero. The Deployment auto-restarts the pod, which +# gives MTBF data; the alert workflow polls the report ConfigMap and +# pages on incident-count thresholds. +# +# Image refs are templated; the longhaul-deploy workflow substitutes: +# __OWNER__ -> lowercased ${{ github.repository_owner }} +# __IMAGE_TAG__ -> input.image_tag (e.g. sha-5c30d7b or main) +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhaul-test-config + namespace: documentdb-test-ns + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +data: + # Test duration. "0s" = run-until-failure (the design default for the + # long-haul cluster). Override to a finite value (e.g. "30m", "3h") only + # for short ad-hoc / smoke runs. + LONGHAUL_MAX_DURATION: "0s" + # Target DocumentDB cluster. + LONGHAUL_NAMESPACE: "documentdb-test-ns" + LONGHAUL_CLUSTER_NAME: "documentdb-cluster" + # Writer/verifier counts. + LONGHAUL_NUM_WRITERS: "5" + # Operation scheduling. + LONGHAUL_OP_COOLDOWN: "10m" + LONGHAUL_RECOVERY_TIMEOUT: "5m" + # Replica range for scale operations. + # Bounds for spec.instancesPerNode (CRD allows 1-3). Default Min=1, Max=3 + # exercises full scale-up/down cycle. Set Min=Max to disable scale ops. + LONGHAUL_MIN_INSTANCES: "1" + LONGHAUL_MAX_INSTANCES: "3" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: longhaul-test + namespace: documentdb-test-ns + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing + template: + metadata: + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing + spec: + serviceAccountName: longhaul-test + restartPolicy: Always + containers: + - name: driver + image: ghcr.io/__OWNER__/documentdb-kubernetes-operator/longhaul-test:__IMAGE_TAG__ + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: longhaul-test-config + env: + - name: LONGHAUL_DOCUMENTDB_URI + valueFrom: + secretKeyRef: + name: longhaul-documentdb-credentials + key: uri + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi diff --git a/test/longhaul/deploy/rbac.yaml b/test/longhaul/deploy/rbac.yaml new file mode 100644 index 000000000..d1a10cf02 --- /dev/null +++ b/test/longhaul/deploy/rbac.yaml @@ -0,0 +1,81 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# ServiceAccount for the long-haul test Job. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhaul-test + namespace: documentdb-test-ns + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +--- +# Role granting permissions within the target namespace. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: longhaul-test + namespace: documentdb-test-ns + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +rules: + # Read pod status for health monitoring. + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + # Read and patch DocumentDB CRs for health check and scale operations. + - apiGroups: ["documentdb.io"] + resources: ["dbs"] + verbs: ["get", "list", "patch"] + # Create/update ConfigMaps for periodic report persistence. + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: longhaul-test + namespace: documentdb-test-ns + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: longhaul-test +subjects: + - kind: ServiceAccount + name: longhaul-test + namespace: documentdb-test-ns +--- +# ClusterRole for metrics-server access (metrics API is cluster-scoped). +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhaul-test-metrics + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +rules: + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhaul-test-metrics + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhaul-test-metrics +subjects: + - kind: ServiceAccount + name: longhaul-test + namespace: documentdb-test-ns From fe3f0bdfa6a52a0fc44c48d1508dcb543893ef1b Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Tue, 30 Jun 2026 13:36:05 -0400 Subject: [PATCH 02/15] ci(longhaul): address CI/CD review feedback (15 fixes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-review found 15 issues across the three workflows + deploy manifest. Fixed in-place; also added a PR-time lint workflow so future regressions get caught before merge instead of post-merge against the live cluster. Critical: 1. longhaul-image-build: build context was test/longhaul, but the module's go.mod has `replace` directives for ../shared and ../../operator/src. Set context=. and file=test/longhaul/Dockerfile so the replace targets are inside the build context. 2. longhaul-monitor: operator version check used `-l app.kubernetes.io/name=documentdb-operator` + .items[0]; the chart applies that label to three Deployments (operator, sidecar injector, wal replica). Target the operator Deployment by name (env.OPERATOR_DEPLOYMENT=documentdb-operator). 3. longhaul-monitor: missing `packages: read` permission; the GHCR version lookups (`gh api /orgs/.../packages/...`) silently fall back to chart defaults without it. 4. longhaul-deploy: header comment claimed kubeconfig is scoped via rbac.yaml. rbac.yaml is the driver pod's in-cluster SA — orthogonal to the GHA kubeconfig, which the upgrade job needs cluster-admin for (helm upgrade in another ns + CRD apply). Rewrote header. Major: 5. All three workflows: secrets.LONGHAUL_KUBECONFIG was inlined into `run:` blocks via ${{ secrets.* }}. Switch to env: passthrough so `$` and backticks in the secret cannot shell-inject. 6. longhaul-monitor: ConfigMap longhaul-versions was applied with `kubectl create --dry-run | kubectl apply -f -`, which sends a full body and removes any keys the driver may have added (last-applied, attempt counters, etc.). Switch to server-side apply with --field-manager=longhaul-monitor so the monitor co-owns only the fields it sets. 7. longhaul-deploy: redundant `kubectl set image` + always rollout-restart after `kubectl apply` which already rolls a pod when the image changes. Dropped the set-image; gate the restart on mutable tags only (main / latest / sha-*); pinned semver tags never need a forced restart. 8. (deferred) workflow_run only fires on default branch — covered by the new lint-longhaul-workflows.yml which runs on PRs. 9. (verified, not a bug) GHCR documentdb image path matches the URL-encoded subpackage form used in the operator source (operator/src/internal/utils/constants.go). Minor: 10. Pinned runs-on to ubuntu-22.04 across all jobs (was ubuntu-latest in places) so future runner image roll-forward doesn't surprise us. 11. Added concurrency: groups to all three workflows (cancel-in-progress true for build/lint, false for deploy/monitor to avoid mid-rollout cancellation). 12. Added timeout-minutes to every job. 13. longhaul-image-build: replaced cache-{from,to}=type=registry,ref=...:buildcache (accumulates in GHCR, no GC) with type=gha so cache cycles automatically. 14. longhaul-monitor: added "Dump driver logs on failure" step that tails 200 lines from the long-haul pod + describes it whenever any prior step in check-longhaul fails. Run summaries now contain enough triage context without a kubectl session. 15. test/longhaul/deploy/deployment.yaml ConfigMap was missing the tunables LONGHAUL_STEADY_STATE_WAIT, LONGHAUL_REPORT_INTERVAL, LONGHAUL_RESET_DATA. Added with explicit values (60s, 1h, false) that match the in-code defaults and commentary about when to override. New CI gating: .github/workflows/lint-longhaul-workflows.yml runs on PRs touching any of the long-haul workflow files, Dockerfile, deploy manifests, or the operator chart: - actionlint (+ embedded shellcheck on run: blocks) - yamllint (strict) - kubeconform against k8s 1.30 schemas + a Datree CRD catalog fallback (skips DocumentDB which we don't ship a CRD schema for) - helm template smoke against operator/documentdb-helm-chart with a grep assertion that the rendered output still contains a Deployment named exactly `documentdb-operator` — protects against regressions of the fix in (2) - docker buildx with push:false, mirroring production build exactly to catch context/replace-directive bugs at PR time Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/lint-longhaul-workflows.yml | 113 ++++++++++++++++++ .github/workflows/longhaul-deploy.yml | 70 +++++++---- .github/workflows/longhaul-image-build.yml | 16 ++- .github/workflows/longhaul-monitor.yaml | 80 ++++++++++--- test/longhaul/deploy/deployment.yaml | 14 +++ 5 files changed, 252 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/lint-longhaul-workflows.yml diff --git a/.github/workflows/lint-longhaul-workflows.yml b/.github/workflows/lint-longhaul-workflows.yml new file mode 100644 index 000000000..06d8f4f9d --- /dev/null +++ b/.github/workflows/lint-longhaul-workflows.yml @@ -0,0 +1,113 @@ +# Pre-merge gating for the long-haul CI/CD bundle. +# +# The long-haul {build, deploy, monitor} workflows only run post-merge +# (push to main / scheduled / workflow_run), so a typo in any of them is +# normally not caught until they fire against the live AKS cluster. This +# workflow runs on PRs that touch any of those files (plus the deploy +# manifests + Dockerfile) and exercises them statically: +# +# - actionlint : GitHub Actions syntax, expression bugs, missing +# permissions, embedded shellcheck on `run:` blocks. +# - yamllint : strict YAML formatting. +# - kubeconform : Kubernetes manifest schema validation against the +# v1.30 schema set (matches our compatibility matrix). +# - helm template : ensure the operator chart still renders so the +# auto-upgrade job in monitor.yaml will succeed. +# - docker build : build the long-haul image with push:false to catch +# context / go.mod-replace problems before they break +# the scheduled image build. +name: Lint Long-Haul Workflows + +on: + pull_request: + paths: + - ".github/workflows/longhaul-*.y*ml" + - ".github/workflows/lint-longhaul-workflows.yml" + - "test/longhaul/Dockerfile" + - "test/longhaul/deploy/**" + - "operator/documentdb-helm-chart/**" + +permissions: + contents: read + +concurrency: + group: lint-longhaul-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: actionlint + # reviewdog/action-actionlint scans .github/workflows/ by default + # and embeds shellcheck so `run:` blocks are linted too. We rely + # on the default discovery — there is no point linting only the + # long-haul subset since neighbouring workflows share a runner + # image and any syntax problem would block CI anyway. + uses: reviewdog/action-actionlint@v1 + with: + reporter: github-pr-check + fail_on_error: "true" + + - name: yamllint + run: | + set -euo pipefail + pip install --quiet yamllint + yamllint -d '{extends: default, rules: {line-length: {max: 200}, comments-indentation: disable, document-start: disable, truthy: {check-keys: false}}}' \ + .github/workflows/longhaul-image-build.yml \ + .github/workflows/longhaul-deploy.yml \ + .github/workflows/longhaul-monitor.yaml \ + test/longhaul/deploy/ + + - name: kubeconform manifests + run: | + set -euo pipefail + curl -sSL https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz \ + | tar -xz -C /usr/local/bin kubeconform + # Skip CRD-typed resources (DocumentDB) since their schemas live + # in operator/documentdb-helm-chart/crds — pass via -schema-location. + kubeconform -strict -summary \ + -kubernetes-version 1.30.0 \ + -schema-location default \ + -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \ + -skip 'DocumentDB' \ + test/longhaul/deploy/ + + - name: helm template smoke + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + # If the chart fails to template, the monitor's auto-upgrade + # job will fail at runtime against the live cluster. + helm template documentdb-operator operator/documentdb-helm-chart \ + --namespace documentdb-operator \ + > /tmp/rendered.yaml + # Sanity: rendered output must include the operator Deployment we + # target by name in monitor.yaml. + grep -q 'name: documentdb-operator$' /tmp/rendered.yaml \ + || (echo "::error::Rendered chart missing 'documentdb-operator' Deployment — monitor.yaml's name-based selector will fail" && exit 1) + + image-build-smoke: + # Mirrors the production longhaul-image-build.yml exactly (context, file, + # platforms) but with push:false. Catches context/replace-directive bugs + # at PR time rather than on the main-branch scheduled run. + runs-on: ubuntu-22.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build long-haul image (no push) + uses: docker/build-push-action@v6 + with: + context: . + file: test/longhaul/Dockerfile + push: false + platforms: linux/amd64 + cache-from: type=gha,scope=longhaul-pr + cache-to: type=gha,scope=longhaul-pr,mode=max diff --git a/.github/workflows/longhaul-deploy.yml b/.github/workflows/longhaul-deploy.yml index 442ff8437..8aa9abcb6 100644 --- a/.github/workflows/longhaul-deploy.yml +++ b/.github/workflows/longhaul-deploy.yml @@ -5,10 +5,18 @@ name: LONGHAUL - Deploy Test Driver to AKS # Rolls a longhaul-test image onto the long-haul AKS cluster. # -# Auth: long-lived ServiceAccount-token kubeconfig stored as the -# LONGHAUL_KUBECONFIG repo secret. Scoped to namespace -# documentdb-test-ns via Role/RoleBinding (see -# test/longhaul/deploy/rbac.yaml). No Azure auth on the runner. +# Auth: a kubeconfig for the long-haul AKS cluster, provisioned out of band +# and stored as the LONGHAUL_KUBECONFIG repo secret. The kubeconfig MUST be +# privileged enough to: +# * apply Deployments + ConfigMaps in documentdb-test-ns (this workflow) +# * roll the operator via Helm in documentdb-operator + apply CRDs +# cluster-wide (longhaul-monitor.yaml's upgrade job) +# In practice that means cluster-admin or a sufficiently broad ClusterRole. +# +# Note: test/longhaul/deploy/rbac.yaml is the in-cluster RBAC granted to +# the long-haul *driver pod* (its ServiceAccount), not to this workflow. +# The two are independent: the pod's SA only needs namespace-scoped pod / +# documentdb / configmap access; this workflow needs more. # # Triggers: # * workflow_run — auto-deploys after a successful build on main, pinning @@ -37,10 +45,15 @@ env: DEPLOYMENT: longhaul-test CONTAINER: driver +concurrency: + group: longhaul-deploy + cancel-in-progress: false + jobs: deploy: name: Deploy to AKS runs-on: ubuntu-22.04 + timeout-minutes: 15 # When triggered by workflow_run, only proceed if the upstream build succeeded. # workflow_dispatch always proceeds. if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} @@ -49,11 +62,18 @@ jobs: uses: actions/checkout@v4 - name: Write kubeconfig + # Pass the secret via env, not ${{ }} expression. Expression-style + # interpolation pastes the secret value directly into the shell + # script before execution, which can lead to subtle shell-injection + # if the secret contains characters like $ or backticks. Using env: + # keeps the value as a literal string in the runner's environment. + env: + KUBECONFIG_CONTENT: ${{ secrets.LONGHAUL_KUBECONFIG }} run: | set -euo pipefail install -m 700 -d "$RUNNER_TEMP/.kube" KCFG="$RUNNER_TEMP/.kube/config" - printf '%s' "${{ secrets.LONGHAUL_KUBECONFIG }}" > "$KCFG" + printf '%s' "$KUBECONFIG_CONTENT" > "$KCFG" chmod 600 "$KCFG" echo "KUBECONFIG=$KCFG" >> "$GITHUB_ENV" @@ -79,22 +99,32 @@ jobs: if [ "${{ github.event_name }}" = "workflow_run" ]; then SHORT_SHA="$(echo "${UPSTREAM_SHA}" | cut -c1-7)" TAG="sha-${SHORT_SHA}" + MUTABLE=false else TAG="${DISPATCH_TAG}" + # Treat anything that isn't an immutable sha-* tag as mutable + # (e.g. :main) so we force a rollout restart to re-pull. + case "$TAG" in + sha-*) MUTABLE=false ;; + *) MUTABLE=true ;; + esac fi IMAGE="ghcr.io/${OWNER_LC}/${REPO_NAME}/longhaul-test:${TAG}" - echo "tag=${TAG}" >> "$GITHUB_OUTPUT" - echo "image=${IMAGE}" >> "$GITHUB_OUTPUT" - echo "Resolved image: ${IMAGE}" + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + echo "image=${IMAGE}" >> "$GITHUB_OUTPUT" + echo "mutable=${MUTABLE}" >> "$GITHUB_OUTPUT" + echo "Resolved image: ${IMAGE} (mutable=${MUTABLE})" - name: Apply Deployment manifest # The Deployment manifest is the only thing the deploy workflow # owns. Bootstrap resources (namespace, DocumentDB CR, credentials # secret, driver ServiceAccount/Role/ClusterRole) live in - # setup.yaml + rbac.yaml and are applied once by a cluster admin - # (the deployer SA is intentionally namespace-scoped and cannot - # create those). + # setup.yaml + rbac.yaml and are applied once by a cluster admin. + # + # `kubectl apply` with the substituted image tag is sufficient to + # trigger a rollout when the tag changes — no extra `set image` + # call needed. run: | set -euo pipefail OWNER_LC="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" @@ -104,20 +134,14 @@ jobs: -e "s|__IMAGE_TAG__|${TAG}|g" \ test/longhaul/deploy/deployment.yaml | kubectl apply -f - - - name: Roll image + - name: Force re-pull for mutable tags + # `kubectl apply` only triggers a rollout when the pod-spec changes. + # For mutable tags like :main, the spec stays identical even when + # the underlying image moves — so we need an explicit restart to + # re-pull. Immutable :sha- tags don't need this. + if: steps.img.outputs.mutable == 'true' run: | set -euo pipefail - IMAGE='${{ steps.img.outputs.image }}' - - # `set image` is a no-op if the manifest already pinned this image - # (apply step above), but harmless. The rollout restart below is - # what guarantees a re-pull when the tag is mutable (e.g. :main). - kubectl -n "$NAMESPACE" set image \ - "deployment/${DEPLOYMENT}" \ - "${CONTAINER}=${IMAGE}" \ - --record=false || true - - # Force a restart so the same tag (e.g. :main) re-pulls if image changed. kubectl -n "$NAMESPACE" rollout restart "deployment/${DEPLOYMENT}" - name: Wait for rollout diff --git a/.github/workflows/longhaul-image-build.yml b/.github/workflows/longhaul-image-build.yml index d141241e8..219adb555 100644 --- a/.github/workflows/longhaul-image-build.yml +++ b/.github/workflows/longhaul-image-build.yml @@ -39,13 +39,21 @@ permissions: env: IMAGE_NAME: longhaul-test - CONTEXT: test/longhaul + # Build context MUST be the repo root, not test/longhaul/, because + # test/longhaul/go.mod has `replace` directives pointing to + # ../shared and ../../operator/src (see Dockerfile header at line 5-8). + CONTEXT: . DOCKERFILE: test/longhaul/Dockerfile +concurrency: + group: longhaul-image-build + cancel-in-progress: false + jobs: build-and-push: name: Build and push longhaul image runs-on: ubuntu-22.04 + timeout-minutes: 30 steps: - name: Checkout uses: actions/checkout@v4 @@ -84,8 +92,10 @@ jobs: push: true tags: ${{ steps.refs.outputs.tags }} provenance: false - cache-from: type=registry,ref=${{ steps.refs.outputs.image }}:buildcache - cache-to: type=registry,ref=${{ steps.refs.outputs.image }}:buildcache,mode=max + # GHA cache instead of a :buildcache image tag — keeps GHCR clean + # and is automatically GC'd by GitHub. + cache-from: type=gha,scope=longhaul-test + cache-to: type=gha,scope=longhaul-test,mode=max labels: | org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} org.opencontainers.image.revision=${{ github.sha }} diff --git a/.github/workflows/longhaul-monitor.yaml b/.github/workflows/longhaul-monitor.yaml index c69f746f0..d89217ed8 100644 --- a/.github/workflows/longhaul-monitor.yaml +++ b/.github/workflows/longhaul-monitor.yaml @@ -22,23 +22,35 @@ on: - cron: "0 * * * *" # Every hour (matches LONGHAUL_REPORT_INTERVAL) workflow_dispatch: {} # Allow manual trigger +permissions: + contents: read + packages: read # Required: gh api /orgs/.../packages/container/.../versions + +concurrency: + group: longhaul-monitor + cancel-in-progress: false + env: NAMESPACE: documentdb-test-ns DEPLOYMENT_NAME: longhaul-test OPERATOR_NAMESPACE: documentdb-operator + OPERATOR_DEPLOYMENT: documentdb-operator jobs: check-longhaul: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 + timeout-minutes: 10 steps: - name: Set up kubeconfig + env: + KUBECONFIG_CONTENT: ${{ secrets.LONGHAUL_KUBECONFIG }} run: | set -euo pipefail install -m 700 -d "$RUNNER_TEMP/.kube" KCFG="$RUNNER_TEMP/.kube/config" # Secret is raw kubeconfig YAML (not base64). Same format the # deploy workflow uses; keep these in sync. - printf '%s' "${{ secrets.LONGHAUL_KUBECONFIG }}" > "$KCFG" + printf '%s' "$KUBECONFIG_CONTENT" > "$KCFG" chmod 600 "$KCFG" echo "KUBECONFIG=$KCFG" >> "$GITHUB_ENV" @@ -135,18 +147,42 @@ jobs: echo "✅ Long haul test is healthy" + - name: Dump driver logs on failure + # Tail recent driver logs whenever the evaluate step (or any prior + # step) fails, so the run summary contains enough context to + # triage without a kubectl session. + if: failure() + run: | + set +e + POD="$(kubectl -n "${{ env.NAMESPACE }}" get pod \ + -l app.kubernetes.io/name=longhaul-test \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)" + if [ -n "$POD" ]; then + echo "::group::Driver logs ($POD, last 200 lines)" + kubectl -n "${{ env.NAMESPACE }}" logs "$POD" --tail=200 || true + echo "::endgroup::" + echo "::group::Pod describe" + kubectl -n "${{ env.NAMESPACE }}" describe pod "$POD" || true + echo "::endgroup::" + else + echo "No long-haul driver pod found in ${{ env.NAMESPACE }}." + fi + upgrade-if-needed: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 + timeout-minutes: 15 needs: check-longhaul # Only upgrade if cluster is healthy steps: - uses: actions/checkout@v4 - name: Set up kubeconfig + env: + KUBECONFIG_CONTENT: ${{ secrets.LONGHAUL_KUBECONFIG }} run: | set -euo pipefail install -m 700 -d "$RUNNER_TEMP/.kube" KCFG="$RUNNER_TEMP/.kube/config" - printf '%s' "${{ secrets.LONGHAUL_KUBECONFIG }}" > "$KCFG" + printf '%s' "$KUBECONFIG_CONTENT" > "$KCFG" chmod 600 "$KCFG" echo "KUBECONFIG=$KCFG" >> "$GITHUB_ENV" @@ -157,6 +193,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | + set -euo pipefail # Get latest published operator tag from GHCR (strict semver only) EXPECTED=$(gh api /orgs/documentdb/packages/container/documentdb-operator/versions \ --jq '[.[].metadata.container.tags[] | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))] | sort_by(. | split(".") | map(tonumber)) | last' 2>/dev/null || echo "") @@ -167,9 +204,14 @@ jobs: echo "ℹ️ GHCR query failed, using Chart.yaml: $EXPECTED" fi - # Running version from cluster - RUNNING=$(kubectl get deployment -n ${{ env.OPERATOR_NAMESPACE }} -l app.kubernetes.io/name=documentdb-operator \ - -o jsonpath='{.items[0].spec.template.spec.containers[0].image}' 2>/dev/null | sed 's/.*://') + # Running version from cluster. Target the operator Deployment by + # NAME, not by app.kubernetes.io/name label — the chart applies + # the same label to three Deployments (operator, sidecar-injector + # in cnpg-system, wal-replica in cnpg-system); even though the + # ns filter excludes the cnpg-system ones today, naming is safer. + RUNNING=$(kubectl get deployment "${{ env.OPERATOR_DEPLOYMENT }}" \ + -n "${{ env.OPERATOR_NAMESPACE }}" \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null | sed 's/.*://') echo "Operator: expected=$EXPECTED running=$RUNNING" @@ -185,14 +227,16 @@ jobs: kubectl apply -f operator/documentdb-helm-chart/crds/ helm upgrade documentdb-operator operator/documentdb-helm-chart \ - -n ${{ env.OPERATOR_NAMESPACE }} \ + -n "${{ env.OPERATOR_NAMESPACE }}" \ --set image.tag="$EXPECTED" \ --wait --timeout 5m - # Verify rollout completed - kubectl rollout status deployment -n ${{ env.OPERATOR_NAMESPACE }} -l app.kubernetes.io/name=documentdb-operator --timeout=3m - ACTUAL=$(kubectl get deployment -n ${{ env.OPERATOR_NAMESPACE }} -l app.kubernetes.io/name=documentdb-operator \ - -o jsonpath='{.items[0].spec.template.spec.containers[0].image}' | sed 's/.*://') + # Verify rollout completed on the operator Deployment specifically. + kubectl rollout status deployment "${{ env.OPERATOR_DEPLOYMENT }}" \ + -n "${{ env.OPERATOR_NAMESPACE }}" --timeout=3m + ACTUAL=$(kubectl get deployment "${{ env.OPERATOR_DEPLOYMENT }}" \ + -n "${{ env.OPERATOR_NAMESPACE }}" \ + -o jsonpath='{.spec.template.spec.containers[0].image}' | sed 's/.*://') if [ "$ACTUAL" = "$EXPECTED" ]; then echo "✅ Operator upgraded and verified: $ACTUAL" else @@ -207,6 +251,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | + set -euo pipefail # Determine the latest published DocumentDB tag and publish it to a # ConfigMap. The long-haul test reads this and performs the actual # upgrade as a load-aware operation (so continuous verifiers can @@ -225,12 +270,17 @@ jobs: fi echo "📣 Publishing desired DocumentDB version: $EXPECTED" + # Use server-side apply so we can co-own this ConfigMap with the + # driver (the driver may add keys like last-applied-version or + # upgrade-attempt counters). Client-side apply with a full + # generated body would clobber any field we don't set here. kubectl create configmap longhaul-versions \ - -n ${{ env.NAMESPACE }} \ + -n "${{ env.NAMESPACE }}" \ --from-literal=desired-documentdb-version="$EXPECTED" \ --from-literal=last-updated="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - --dry-run=client -o yaml | kubectl apply -f - + --dry-run=client -o yaml \ + | kubectl apply --server-side --field-manager=longhaul-monitor -f - - RUNNING=$(kubectl get documentdb documentdb-cluster -n ${{ env.NAMESPACE }} \ + RUNNING=$(kubectl get documentdb documentdb-cluster -n "${{ env.NAMESPACE }}" \ -o jsonpath='{.status.documentDBImage}' 2>/dev/null | sed 's/.*://') echo "DocumentDB: desired=$EXPECTED running=$RUNNING (test will upgrade if drifted)" diff --git a/test/longhaul/deploy/deployment.yaml b/test/longhaul/deploy/deployment.yaml index 88319de49..4e4dad3d3 100644 --- a/test/longhaul/deploy/deployment.yaml +++ b/test/longhaul/deploy/deployment.yaml @@ -42,6 +42,20 @@ data: # Operation scheduling. LONGHAUL_OP_COOLDOWN: "10m" LONGHAUL_RECOVERY_TIMEOUT: "5m" + # How long the cluster must be observed healthy before the next + # operation fires. Matches the in-code default (60s) and the design doc + # (Section "Steady-state gate"). Keep explicit so operators can lower + # it for chaos runs or raise it during incident triage without a + # rebuild. + LONGHAUL_STEADY_STATE_WAIT: "60s" + # Checkpoint report cadence. Matches the monitor workflow cron (hourly) + # — bump both together if changed. + LONGHAUL_REPORT_INTERVAL: "1h" + # Whether to drop the workload collection on startup. MUST stay "false" + # on the long-haul cluster — restarts of the driver pod (rolling image + # updates, node drains) need to resume durability state from PVC. + # Override to "true" only for local/dev iterations. + LONGHAUL_RESET_DATA: "false" # Replica range for scale operations. # Bounds for spec.instancesPerNode (CRD allows 1-3). Default Min=1, Max=3 # exercises full scale-up/down cycle. Set Min=Max to disable scale ops. From 4e1d8d4a4c1bd015bc71d5733c7d8c0b711e8af2 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Tue, 30 Jun 2026 15:32:04 -0400 Subject: [PATCH 03/15] ci(longhaul): fix Lint Long-Haul Workflows failures - lint-longhaul-workflows.yml: replace deprecated reviewdog input fail_on_error with fail_level: error. - Group consecutive $GITHUB_OUTPUT redirects with { ...; } >> file to satisfy shellcheck SC2129 (image-build, deploy, monitor). - monitor: bind ready output to a shell variable before the numeric -lt comparison to satisfy shellcheck SC2170. - image-build: drop extra spaces after cache-to: colon (yamllint). - monitor: use two spaces before inline comments (yamllint). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/lint-longhaul-workflows.yml | 2 +- .github/workflows/longhaul-deploy.yml | 8 +++--- .github/workflows/longhaul-image-build.yml | 10 +++++--- .github/workflows/longhaul-monitor.yaml | 25 +++++++++++-------- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.github/workflows/lint-longhaul-workflows.yml b/.github/workflows/lint-longhaul-workflows.yml index 06d8f4f9d..c689dbc04 100644 --- a/.github/workflows/lint-longhaul-workflows.yml +++ b/.github/workflows/lint-longhaul-workflows.yml @@ -50,7 +50,7 @@ jobs: uses: reviewdog/action-actionlint@v1 with: reporter: github-pr-check - fail_on_error: "true" + fail_level: error - name: yamllint run: | diff --git a/.github/workflows/longhaul-deploy.yml b/.github/workflows/longhaul-deploy.yml index 8aa9abcb6..261fc051a 100644 --- a/.github/workflows/longhaul-deploy.yml +++ b/.github/workflows/longhaul-deploy.yml @@ -111,9 +111,11 @@ jobs: fi IMAGE="ghcr.io/${OWNER_LC}/${REPO_NAME}/longhaul-test:${TAG}" - echo "tag=${TAG}" >> "$GITHUB_OUTPUT" - echo "image=${IMAGE}" >> "$GITHUB_OUTPUT" - echo "mutable=${MUTABLE}" >> "$GITHUB_OUTPUT" + { + echo "tag=${TAG}" + echo "image=${IMAGE}" + echo "mutable=${MUTABLE}" + } >> "$GITHUB_OUTPUT" echo "Resolved image: ${IMAGE} (mutable=${MUTABLE})" - name: Apply Deployment manifest diff --git a/.github/workflows/longhaul-image-build.yml b/.github/workflows/longhaul-image-build.yml index 219adb555..8e7e3e74b 100644 --- a/.github/workflows/longhaul-image-build.yml +++ b/.github/workflows/longhaul-image-build.yml @@ -72,9 +72,11 @@ jobs: TAGS="${TAGS},${IMG}:${{ github.event.inputs.extra_tag }}" fi - echo "image=${IMG}" >> "$GITHUB_OUTPUT" - echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT" - echo "tags=${TAGS}" >> "$GITHUB_OUTPUT" + { + echo "image=${IMG}" + echo "short_sha=${SHORT_SHA}" + echo "tags=${TAGS}" + } >> "$GITHUB_OUTPUT" echo "Will push: ${TAGS}" - name: Login to GHCR @@ -95,7 +97,7 @@ jobs: # GHA cache instead of a :buildcache image tag — keeps GHCR clean # and is automatically GC'd by GitHub. cache-from: type=gha,scope=longhaul-test - cache-to: type=gha,scope=longhaul-test,mode=max + cache-to: type=gha,scope=longhaul-test,mode=max labels: | org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} org.opencontainers.image.revision=${{ github.sha }} diff --git a/.github/workflows/longhaul-monitor.yaml b/.github/workflows/longhaul-monitor.yaml index d89217ed8..a783b0f93 100644 --- a/.github/workflows/longhaul-monitor.yaml +++ b/.github/workflows/longhaul-monitor.yaml @@ -19,8 +19,8 @@ name: "Long Haul Monitor" on: schedule: - - cron: "0 * * * *" # Every hour (matches LONGHAUL_REPORT_INTERVAL) - workflow_dispatch: {} # Allow manual trigger + - cron: "0 * * * *" # Every hour (matches LONGHAUL_REPORT_INTERVAL) + workflow_dispatch: {} # Allow manual trigger permissions: contents: read @@ -72,9 +72,11 @@ jobs: else echo "exists=true" >> "$GITHUB_OUTPUT" fi - echo "ready=${READY:-0}" >> "$GITHUB_OUTPUT" - echo "desired=${DESIRED:-0}" >> "$GITHUB_OUTPUT" - echo "progressing_reason=${PROGRESSING_REASON}" >> "$GITHUB_OUTPUT" + { + echo "ready=${READY:-0}" + echo "desired=${DESIRED:-0}" + echo "progressing_reason=${PROGRESSING_REASON}" + } >> "$GITHUB_OUTPUT" - name: Check ConfigMap report id: report-check @@ -86,9 +88,11 @@ jobs: echo "last_updated=$LAST_UPDATED" >> "$GITHUB_OUTPUT" REPORT=$(kubectl get configmap longhaul-report -n ${{ env.NAMESPACE }} -o jsonpath='{.data.latest-report}' 2>/dev/null || echo "No report available") - echo "report<> "$GITHUB_OUTPUT" - echo "$REPORT" >> "$GITHUB_OUTPUT" - echo "EOF" >> "$GITHUB_OUTPUT" + { + echo "report<> "$GITHUB_OUTPUT" - name: Evaluate status run: | @@ -122,7 +126,8 @@ jobs: fi # Fail if no replicas are ready. - if [ "${{ steps.deploy-check.outputs.ready }}" -lt 1 ]; then + READY="${{ steps.deploy-check.outputs.ready }}" + if [ "${READY:-0}" -lt 1 ]; then echo "::error::Long haul Deployment has 0 ready replicas" exit 1 fi @@ -171,7 +176,7 @@ jobs: upgrade-if-needed: runs-on: ubuntu-22.04 timeout-minutes: 15 - needs: check-longhaul # Only upgrade if cluster is healthy + needs: check-longhaul # Only upgrade if cluster is healthy steps: - uses: actions/checkout@v4 From 60b0907608016b47ba00cb153fc02d6aa1c7328f Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Tue, 30 Jun 2026 15:34:12 -0400 Subject: [PATCH 04/15] ci(longhaul): fix helm template smoke in lint workflow Build chart dependencies (cloudnative-pg) before templating and pass --api-versions cert-manager.io/v1/Certificate so the cert-manager preflight check passes during offline rendering. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/lint-longhaul-workflows.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/lint-longhaul-workflows.yml b/.github/workflows/lint-longhaul-workflows.yml index c689dbc04..37f4e3f7a 100644 --- a/.github/workflows/lint-longhaul-workflows.yml +++ b/.github/workflows/lint-longhaul-workflows.yml @@ -80,10 +80,14 @@ jobs: run: | set -euo pipefail curl -sSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + # Fetch chart dependencies (e.g. cloudnative-pg) so the chart can + # be templated — mirrors the release workflows. + helm dependency update operator/documentdb-helm-chart # If the chart fails to template, the monitor's auto-upgrade # job will fail at runtime against the live cluster. helm template documentdb-operator operator/documentdb-helm-chart \ --namespace documentdb-operator \ + --api-versions cert-manager.io/v1/Certificate \ > /tmp/rendered.yaml # Sanity: rendered output must include the operator Deployment we # target by name in monitor.yaml. From 01e5032dc18358fc1b8aa87ba816cfacac17a1ee Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Sat, 4 Jul 2026 21:14:35 -0400 Subject: [PATCH 05/15] fix: use --force-conflicts in longhaul-monitor version publish The 'Publish desired DocumentDB version' step performs a server-side apply on the longhaul-versions ConfigMap with field manager 'longhaul-monitor'. When the ConfigMap already exists and its fields (e.g. .data.last-updated) are owned by another manager such as 'kubectl-client-side-apply' (from an earlier 'kubectl create'/client-side apply), the SSA fails with a field ownership conflict: error: Apply failed with 1 conflict: conflict with "kubectl-client-side-apply" using v1: .data.last-updated Add --force-conflicts so the monitor, which is the intended authoritative writer of desired-documentdb-version and last-updated, cleanly takes ownership of those fields. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/longhaul-monitor.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/longhaul-monitor.yaml b/.github/workflows/longhaul-monitor.yaml index a783b0f93..74ecf28e0 100644 --- a/.github/workflows/longhaul-monitor.yaml +++ b/.github/workflows/longhaul-monitor.yaml @@ -279,12 +279,16 @@ jobs: # driver (the driver may add keys like last-applied-version or # upgrade-attempt counters). Client-side apply with a full # generated body would clobber any field we don't set here. + # --force-conflicts lets longhaul-monitor take ownership of the + # fields it manages (desired-documentdb-version, last-updated) even + # if the ConfigMap was originally created by a different field + # manager (e.g. an earlier `kubectl create`/client-side apply). kubectl create configmap longhaul-versions \ -n "${{ env.NAMESPACE }}" \ --from-literal=desired-documentdb-version="$EXPECTED" \ --from-literal=last-updated="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ --dry-run=client -o yaml \ - | kubectl apply --server-side --field-manager=longhaul-monitor -f - + | kubectl apply --server-side --force-conflicts --field-manager=longhaul-monitor -f - RUNNING=$(kubectl get documentdb documentdb-cluster -n "${{ env.NAMESPACE }}" \ -o jsonpath='{.status.documentDBImage}' 2>/dev/null | sed 's/.*://') From 7947ccdf1ea83eda44516569b7566b159cf95477 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Tue, 7 Jul 2026 11:01:41 -0400 Subject: [PATCH 06/15] docs(longhaul): add MIT license header to lint workflow The lint-longhaul-workflows.yml file was missing the standard MIT license header present in the other longhaul workflow files. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/lint-longhaul-workflows.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/lint-longhaul-workflows.yml b/.github/workflows/lint-longhaul-workflows.yml index 37f4e3f7a..0d1da3944 100644 --- a/.github/workflows/lint-longhaul-workflows.yml +++ b/.github/workflows/lint-longhaul-workflows.yml @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + # Pre-merge gating for the long-haul CI/CD bundle. # # The long-haul {build, deploy, monitor} workflows only run post-merge From b213cefef5f369e49c594ce1b396f78b35e44158 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Tue, 7 Jul 2026 11:09:56 -0400 Subject: [PATCH 07/15] ci(longhaul): broaden lint triggers, add go tests + manifest guard Strengthen the pre-merge long-haul gate: - Broaden the PR path filter to test/longhaul/** so edits to the driver sources (not just Dockerfile/deploy) trigger the gate. Previously a change under test/longhaul/workload, operations, report, etc. would skip the build-smoke and tests entirely. - Add a go-test job that runs 'go vet ./...' and 'go test ./...' for the driver module. The driver has real unit tests (config, journal, monitor, operations, report) that the image build only compiled, never executed, so logic regressions could slip through. - Add a manifest sanity guard enforcing a 512Mi memory-limit floor on the driver Deployment, and bump the manifest from the 256Mi limit (which OOM-kills the driver) to 1Gi. Schema validation cannot catch this. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/lint-longhaul-workflows.yml | 65 +++++++++++++++++-- test/longhaul/deploy/deployment.yaml | 4 +- 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint-longhaul-workflows.yml b/.github/workflows/lint-longhaul-workflows.yml index 0d1da3944..8560d7718 100644 --- a/.github/workflows/lint-longhaul-workflows.yml +++ b/.github/workflows/lint-longhaul-workflows.yml @@ -6,16 +6,23 @@ # The long-haul {build, deploy, monitor} workflows only run post-merge # (push to main / scheduled / workflow_run), so a typo in any of them is # normally not caught until they fire against the live AKS cluster. This -# workflow runs on PRs that touch any of those files (plus the deploy -# manifests + Dockerfile) and exercises them statically: +# workflow runs on PRs that touch any of those files (plus the long-haul +# driver sources, deploy manifests, and Dockerfile) and exercises them +# statically: # # - actionlint : GitHub Actions syntax, expression bugs, missing # permissions, embedded shellcheck on `run:` blocks. # - yamllint : strict YAML formatting. # - kubeconform : Kubernetes manifest schema validation against the # v1.30 schema set (matches our compatibility matrix). +# - manifest guard: sanity checks that schema validation can't catch, +# e.g. the driver memory limit must clear a floor so we +# never re-ship the 256Mi limit that OOM-kills the driver. # - helm template : ensure the operator chart still renders so the # auto-upgrade job in monitor.yaml will succeed. +# - go vet/test : the driver has real unit tests (config, journal, +# monitor, operations, report); compiling in the image +# build is not enough, so run them here. # - docker build : build the long-haul image with push:false to catch # context / go.mod-replace problems before they break # the scheduled image build. @@ -26,8 +33,7 @@ on: paths: - ".github/workflows/longhaul-*.y*ml" - ".github/workflows/lint-longhaul-workflows.yml" - - "test/longhaul/Dockerfile" - - "test/longhaul/deploy/**" + - "test/longhaul/**" - "operator/documentdb-helm-chart/**" permissions: @@ -79,6 +85,33 @@ jobs: -skip 'DocumentDB' \ test/longhaul/deploy/ + - name: manifest sanity guard + # Schema validation (kubeconform) proves the YAML is well-formed but + # not that the values are sane. Encode the hard-won lessons here: + # * the driver was OOM-killed at 256Mi — enforce a memory floor so + # a future edit can't silently reintroduce it. + run: | + set -euo pipefail + DEPLOY=test/longhaul/deploy/deployment.yaml + # Extract the container memory limit (e.g. "1Gi", "512Mi"). + LIMIT=$(grep -A4 'limits:' "$DEPLOY" | grep -m1 'memory:' | awk '{print $2}') + if [ -z "$LIMIT" ]; then + echo "::error file=$DEPLOY::Could not find a container memory limit" + exit 1 + fi + # Normalise to MiB for comparison. + case "$LIMIT" in + *Gi) MIB=$(( ${LIMIT%Gi} * 1024 )) ;; + *Mi) MIB=${LIMIT%Mi} ;; + *) echo "::error file=$DEPLOY::Unexpected memory unit in '$LIMIT' (want Mi/Gi)"; exit 1 ;; + esac + FLOOR_MIB=512 + echo "driver memory limit = ${LIMIT} (${MIB}Mi); floor = ${FLOOR_MIB}Mi" + if [ "$MIB" -lt "$FLOOR_MIB" ]; then + echo "::error file=$DEPLOY::Driver memory limit ${LIMIT} is below the ${FLOOR_MIB}Mi floor; 256Mi OOM-kills the driver." + exit 1 + fi + - name: helm template smoke run: | set -euo pipefail @@ -97,6 +130,30 @@ jobs: grep -q 'name: documentdb-operator$' /tmp/rendered.yaml \ || (echo "::error::Rendered chart missing 'documentdb-operator' Deployment — monitor.yaml's name-based selector will fail" && exit 1) + go-test: + # The driver has real unit tests (config, journal, monitor, operations, + # report). The image build only compiles them — run them here so a logic + # regression in the reporter/thresholds is caught at PR time. + runs-on: ubuntu-22.04 + timeout-minutes: 15 + defaults: + run: + working-directory: test/longhaul + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: test/longhaul/go.mod + cache-dependency-path: test/longhaul/go.sum + + - name: go vet + run: go vet ./... + + - name: go test + run: go test ./... -count=1 + image-build-smoke: # Mirrors the production longhaul-image-build.yml exactly (context, file, # platforms) but with push:false. Catches context/replace-directive bugs diff --git a/test/longhaul/deploy/deployment.yaml b/test/longhaul/deploy/deployment.yaml index 4e4dad3d3..b07f0cde3 100644 --- a/test/longhaul/deploy/deployment.yaml +++ b/test/longhaul/deploy/deployment.yaml @@ -102,7 +102,7 @@ spec: resources: requests: cpu: 100m - memory: 128Mi + memory: 256Mi limits: cpu: 500m - memory: 256Mi + memory: 1Gi From d032131c5c0f6f1c7e6a3bf623cf8cac4d681c2a Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Tue, 7 Jul 2026 11:25:15 -0400 Subject: [PATCH 08/15] perf(longhaul): stream verify scan to keep driver lightweight (256Mi) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verifier's fetchDocs materialised an entire per-cycle window of documents into a slice before auditing them. Steady-state that window is small, but after a disruption pause the verifier catches up on tens of thousands of docs at once; the slice plus BSON-decode buffers spike the heap and OOM-kill the driver against a hard 256Mi cgroup limit — even though the driver's own bookkeeping is only a few MB. Refactor the audit into an incremental auditor (step/finish) and stream the cursor through it one doc at a time, so verify memory is O(1) per cycle instead of O(window). auditDocs is retained as a thin wrapper so the existing table tests are unchanged. Also surface cursor.Err() so a truncated read is reported instead of being silently mistaken for tail loss. With the streaming scan the driver is genuinely lightweight, so: - restore the Deployment memory limit to 256Mi (request 128Mi), - add GOMEMLIMIT=200MiB as a soft GC ceiling below the hard limit, and - drop the now-unnecessary memory-floor guard from the lint workflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/lint-longhaul-workflows.yml | 30 ----- test/longhaul/deploy/deployment.yaml | 11 +- test/longhaul/workload/verifier.go | 127 +++++++++++------- 3 files changed, 89 insertions(+), 79 deletions(-) diff --git a/.github/workflows/lint-longhaul-workflows.yml b/.github/workflows/lint-longhaul-workflows.yml index 8560d7718..18f494b60 100644 --- a/.github/workflows/lint-longhaul-workflows.yml +++ b/.github/workflows/lint-longhaul-workflows.yml @@ -15,9 +15,6 @@ # - yamllint : strict YAML formatting. # - kubeconform : Kubernetes manifest schema validation against the # v1.30 schema set (matches our compatibility matrix). -# - manifest guard: sanity checks that schema validation can't catch, -# e.g. the driver memory limit must clear a floor so we -# never re-ship the 256Mi limit that OOM-kills the driver. # - helm template : ensure the operator chart still renders so the # auto-upgrade job in monitor.yaml will succeed. # - go vet/test : the driver has real unit tests (config, journal, @@ -85,33 +82,6 @@ jobs: -skip 'DocumentDB' \ test/longhaul/deploy/ - - name: manifest sanity guard - # Schema validation (kubeconform) proves the YAML is well-formed but - # not that the values are sane. Encode the hard-won lessons here: - # * the driver was OOM-killed at 256Mi — enforce a memory floor so - # a future edit can't silently reintroduce it. - run: | - set -euo pipefail - DEPLOY=test/longhaul/deploy/deployment.yaml - # Extract the container memory limit (e.g. "1Gi", "512Mi"). - LIMIT=$(grep -A4 'limits:' "$DEPLOY" | grep -m1 'memory:' | awk '{print $2}') - if [ -z "$LIMIT" ]; then - echo "::error file=$DEPLOY::Could not find a container memory limit" - exit 1 - fi - # Normalise to MiB for comparison. - case "$LIMIT" in - *Gi) MIB=$(( ${LIMIT%Gi} * 1024 )) ;; - *Mi) MIB=${LIMIT%Mi} ;; - *) echo "::error file=$DEPLOY::Unexpected memory unit in '$LIMIT' (want Mi/Gi)"; exit 1 ;; - esac - FLOOR_MIB=512 - echo "driver memory limit = ${LIMIT} (${MIB}Mi); floor = ${FLOOR_MIB}Mi" - if [ "$MIB" -lt "$FLOOR_MIB" ]; then - echo "::error file=$DEPLOY::Driver memory limit ${LIMIT} is below the ${FLOOR_MIB}Mi floor; 256Mi OOM-kills the driver." - exit 1 - fi - - name: helm template smoke run: | set -euo pipefail diff --git a/test/longhaul/deploy/deployment.yaml b/test/longhaul/deploy/deployment.yaml index b07f0cde3..d593dc3b8 100644 --- a/test/longhaul/deploy/deployment.yaml +++ b/test/longhaul/deploy/deployment.yaml @@ -99,10 +99,17 @@ spec: secretKeyRef: name: longhaul-documentdb-credentials key: uri + # Soft heap ceiling below the cgroup limit so Go's GC turns + # aggressive before the hard limit, preventing OOMKills under a + # transient allocation spike. The verify path streams its cursor + # (O(1) memory), so steady-state usage is a few MB; this is a + # belt-and-suspenders guard, not a working-set requirement. + - name: GOMEMLIMIT + value: "200MiB" resources: requests: cpu: 100m - memory: 256Mi + memory: 128Mi limits: cpu: 500m - memory: 1Gi + memory: 256Mi diff --git a/test/longhaul/workload/verifier.go b/test/longhaul/workload/verifier.go index 34787673b..10addf7d0 100644 --- a/test/longhaul/workload/verifier.go +++ b/test/longhaul/workload/verifier.go @@ -115,55 +115,80 @@ type auditResult struct { findings []finding } -// auditDocs is the pure decision core of verifyWriter. Given the docs the -// verifier read (sorted by seq ascending) plus the writer's expected starting -// seq and current tip, it returns the new expected seq, the gap/checksum/tail -// counters, and a list of findings for the caller to log. +// auditor is the incremental decision core of verifyWriter. Feed it the docs +// for a writer in ascending seq order via step(); it accumulates gap/checksum +// counters and findings while holding only the running expectedSeq — never the +// docs themselves. finish(maxSeq) applies trailing tail-loss detection and +// returns the aggregate. Keeping state O(1) (not O(docs)) is what lets the +// production scan stream a cursor instead of buffering a whole cycle's window, +// which after a disruption pause can be tens of thousands of docs. // // Invariants checked: // - For each doc, if doc.Seq > expectedSeq, the slots in [expectedSeq, doc.Seq) // are missing (internal gap). // - For each doc, checksum is recomputed and compared. -// - After processing all docs, if expectedSeq <= maxSeq the trailing slots +// - After all docs, if expectedSeq <= maxSeq the trailing slots // [expectedSeq, maxSeq] are missing (tail loss). -// - On exit, newExpectedSeq is always maxSeq+1 when maxSeq >= initial +// - On finish, newExpectedSeq is always maxSeq+1 when maxSeq >= initial // expectedSeq, so the next cycle accounts for everything past maxSeq. -func auditDocs(writerID string, docs []WriteDocument, expectedSeq, maxSeq int64) auditResult { - var r auditResult - for _, doc := range docs { - if doc.Seq > expectedSeq { - gaps := doc.Seq - expectedSeq - r.internalGaps += gaps - r.findings = append(r.findings, finding{ - kind: findingGap, writerID: writerID, - seq: expectedSeq, endSeq: doc.Seq, count: gaps, - }) - } - expectedSeq = doc.Seq + 1 - - want := computeChecksum(doc.WriterID, doc.Seq, doc.Payload) - if doc.Checksum != want { - r.checksumErrors++ - r.findings = append(r.findings, finding{ - kind: findingChecksum, writerID: writerID, - seq: doc.Seq, count: 1, - stored: doc.Checksum, computed: want, - }) - } +type auditor struct { + writerID string + expectedSeq int64 + r auditResult +} + +func newAuditor(writerID string, expectedSeq int64) *auditor { + return &auditor{writerID: writerID, expectedSeq: expectedSeq} +} + +// step folds a single doc into the running audit state. +func (a *auditor) step(doc WriteDocument) { + if doc.Seq > a.expectedSeq { + gaps := doc.Seq - a.expectedSeq + a.r.internalGaps += gaps + a.r.findings = append(a.r.findings, finding{ + kind: findingGap, writerID: a.writerID, + seq: a.expectedSeq, endSeq: doc.Seq, count: gaps, + }) } + a.expectedSeq = doc.Seq + 1 - if expectedSeq <= maxSeq { - tail := maxSeq - expectedSeq + 1 - r.tailLoss = tail - r.findings = append(r.findings, finding{ - kind: findingTail, writerID: writerID, - seq: expectedSeq, endSeq: maxSeq, count: tail, + want := computeChecksum(doc.WriterID, doc.Seq, doc.Payload) + if doc.Checksum != want { + a.r.checksumErrors++ + a.r.findings = append(a.r.findings, finding{ + kind: findingChecksum, writerID: a.writerID, + seq: doc.Seq, count: 1, + stored: doc.Checksum, computed: want, }) - expectedSeq = maxSeq + 1 } +} - r.newExpectedSeq = expectedSeq - return r +// finish applies tail-loss detection for [expectedSeq, maxSeq] and returns the +// aggregate result. +func (a *auditor) finish(maxSeq int64) auditResult { + if a.expectedSeq <= maxSeq { + tail := maxSeq - a.expectedSeq + 1 + a.r.tailLoss = tail + a.r.findings = append(a.r.findings, finding{ + kind: findingTail, writerID: a.writerID, + seq: a.expectedSeq, endSeq: maxSeq, count: tail, + }) + a.expectedSeq = maxSeq + 1 + } + a.r.newExpectedSeq = a.expectedSeq + return a.r +} + +// auditDocs is the pure, table-testable entry point: it replays a slice of docs +// through an auditor. Production code uses the streaming path in scanWriter, +// which feeds cursor-decoded docs to an auditor one at a time. +func auditDocs(writerID string, docs []WriteDocument, expectedSeq, maxSeq int64) auditResult { + a := newAuditor(writerID, expectedSeq) + for _, doc := range docs { + a.step(doc) + } + return a.finish(maxSeq) } func (v *Verifier) verifyWriter(ctx context.Context, w *Writer) { @@ -185,13 +210,12 @@ func (v *Verifier) verifyWriter(ctx context.Context, w *Writer) { return } - docs, err := v.fetchDocs(ctx, writerID, expectedSeq, maxSeq) + r, err := v.scanWriter(ctx, writerID, expectedSeq, maxSeq) if err != nil { v.journal.Warn("verifier", fmt.Sprintf("query failed for writer %s: %v", writerID, err)) return } - r := auditDocs(writerID, docs, expectedSeq, maxSeq) v.metrics.VerifyGapsDetected.Add(r.internalGaps + r.tailLoss) v.metrics.ChecksumErrors.Add(r.checksumErrors) for _, f := range r.findings { @@ -201,10 +225,12 @@ func (v *Verifier) verifyWriter(ctx context.Context, w *Writer) { v.nextSeq[writerID] = r.newExpectedSeq } -// fetchDocs reads all docs for writerID with seq in [expectedSeq, maxSeq], -// sorted by seq ascending. Decode errors are logged but skipped (the rest of -// the scan continues; a skipped doc looks like a gap to auditDocs). -func (v *Verifier) fetchDocs(ctx context.Context, writerID string, expectedSeq, maxSeq int64) ([]WriteDocument, error) { +// scanWriter streams all docs for writerID with seq in [expectedSeq, maxSeq] +// (sorted by seq ascending) through an auditor, holding only one decoded doc at +// a time. This keeps verify memory O(1) per cycle regardless of how many docs +// accumulated since the last tick — which can be tens of thousands after a +// disruption pause — instead of materialising the whole window in a slice. +func (v *Verifier) scanWriter(ctx context.Context, writerID string, expectedSeq, maxSeq int64) (auditResult, error) { opts := options.Find().SetSort(bson.D{{Key: "seq", Value: 1}}) filter := bson.D{ {Key: "writer_id", Value: writerID}, @@ -215,20 +241,27 @@ func (v *Verifier) fetchDocs(ctx context.Context, writerID string, expectedSeq, } cursor, err := v.collection.Find(ctx, filter, opts) if err != nil { - return nil, err + return auditResult{}, err } defer cursor.Close(ctx) - var out []WriteDocument + a := newAuditor(writerID, expectedSeq) for cursor.Next(ctx) { var doc WriteDocument if err := cursor.Decode(&doc); err != nil { + // Decode errors are logged but skipped (the rest of the scan + // continues; a skipped doc looks like a gap to the auditor). v.journal.Warn("verifier", fmt.Sprintf("decode error for writer %s: %v", writerID, err)) continue } - out = append(out, doc) + a.step(doc) + } + // Surface iteration errors rather than silently treating a truncated read as + // tail loss, which would be a false positive. + if err := cursor.Err(); err != nil { + return auditResult{}, err } - return out, nil + return a.finish(maxSeq), nil } func (v *Verifier) logFinding(f finding) { From 2149518df94262c2e7e84d8e1cf30bf48391c6ad Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Tue, 7 Jul 2026 11:47:55 -0400 Subject: [PATCH 09/15] test: add kind-based long-haul smoke gate Add longhaul-smoke.yml, a PR gate that stands up a real DocumentDB on an ephemeral kind cluster and runs the actual long-haul driver against it for a short bounded window. It reuses test/longhaul/deploy/{deployment,rbac}.yaml unmodified (only patching runtime knobs on the shipped ConfigMap for a bounded, deterministic run) so manifest/driver regressions fail the PR. Flow: build driver image -> kind -> cert-manager -> CSI hostpath -> operator via public OCI chart -> single-instance ClusterIP DocumentDB -> run driver (LONGHAUL_MAX_DURATION bounded, scale ops disabled) -> assert container exit 0 AND longhaul-report ConfigMap result == PASS. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/longhaul-smoke.yml | 355 +++++++++++++++++++++++++++ 1 file changed, 355 insertions(+) create mode 100644 .github/workflows/longhaul-smoke.yml diff --git a/.github/workflows/longhaul-smoke.yml b/.github/workflows/longhaul-smoke.yml new file mode 100644 index 000000000..dd1fec7c9 --- /dev/null +++ b/.github/workflows/longhaul-smoke.yml @@ -0,0 +1,355 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Long-Haul Smoke Gate +# +# Purpose +# A PR gate that stands up a *real* DocumentDB on an ephemeral kind cluster +# and runs the actual long-haul driver against it for a short, bounded +# window. Unlike lint-longhaul-workflows.yml (which statically validates the +# YAML/manifests) this workflow proves the driver + deploy manifests actually +# function end-to-end: the image builds, the container starts, it connects to +# DocumentDB over the gateway, writers + verifier run, and the driver reports +# PASS with a clean exit code. +# +# Why reuse test/longhaul/deploy/{deployment,rbac}.yaml as-is +# The whole point of the gate is to exercise the SAME manifests the long-haul +# cluster ships, so a regression in the ConfigMap/Deployment (bad env key, +# too-low memory limit, wrong secret ref) fails the PR. We only override a +# handful of ConfigMap knobs (via a JSON merge patch) to make the run bounded +# and deterministic in CI — we never fork the manifest. +# +# Why not a Job +# deployment.yaml is a Deployment (restartPolicy: Always) so a bounded run +# that exits 0 would auto-restart. We handle that by scaling the Deployment +# to 0 the moment the driver's container first terminates, freezing the +# verdict, then asserting on BOTH the container exit code AND the +# longhaul-report ConfigMap's result field. +# +# CI budget +# Scale/upgrade disruption ops are disabled for the smoke run +# (LONGHAUL_MIN_INSTANCES == LONGHAUL_MAX_INSTANCES) so the gate is a fast, +# deterministic data-durability check (writers + verifier) that fits a +# 7 GB GitHub-hosted runner and finishes in a few minutes. + +name: Long-Haul Smoke Gate + +on: + pull_request: + paths: + - "test/longhaul/**" + - "test/shared/**" + - ".github/workflows/longhaul-smoke.yml" + workflow_dispatch: + inputs: + max_duration: + description: "Bounded driver run length (Go duration, e.g. 3m)" + required: false + default: "3m" + +permissions: + contents: read + +concurrency: + group: longhaul-smoke-${{ github.ref }} + cancel-in-progress: true + +env: + # Namespaces / names are aligned with the hardcoded values baked into + # test/longhaul/deploy/*.yaml so those manifests apply unmodified. + DB_NS: documentdb-test-ns + DB_NAME: documentdb-cluster + OPERATOR_NS: documentdb-operator + CERT_MANAGER_NS: cert-manager + DB_USERNAME: docdbuser + # Smoke-only credential for an ephemeral, throwaway kind cluster that is + # destroyed at the end of the job. Not a secret; never reused elsewhere. + DB_PASSWORD: SmokeTest123! + GATEWAY_PORT: "10260" + KIND_CLUSTER: longhaul-smoke + DRIVER_IMAGE: longhaul-test:smoke + MAX_DURATION: ${{ github.event.inputs.max_duration || '3m' }} + +jobs: + smoke: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build long-haul driver image + run: | + # Context MUST be the repo root: the Dockerfile copies ../shared and + # ../../operator/src (see test/longhaul/Dockerfile header). + docker build \ + -f test/longhaul/Dockerfile \ + -t "${DRIVER_IMAGE}" \ + . + + - name: Create kind cluster + uses: helm/kind-action@v1.12.0 + with: + version: v0.31.0 + node_image: kindest/node:v1.30.4 + cluster_name: ${{ env.KIND_CLUSTER }} + + - name: Load driver image into kind + run: kind load docker-image "${DRIVER_IMAGE}" --name "${KIND_CLUSTER}" + + - name: Install cert-manager + run: | + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm install cert-manager jetstack/cert-manager \ + --namespace "${CERT_MANAGER_NS}" \ + --create-namespace \ + --version v1.15.3 \ + --set installCRDs=true \ + --wait --timeout=5m + + - name: Deploy CSI hostpath driver + run: | + chmod +x ./operator/src/scripts/test-scripts/deploy-csi-driver.sh + ./operator/src/scripts/test-scripts/deploy-csi-driver.sh + + - name: Install DocumentDB operator (public OCI chart) + env: + GHCR_LOGIN_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_LOGIN_USERNAME: ${{ github.actor }} + run: | + # The released chart is published from the canonical documentdb org + # and is public, so forks install it anonymously — no operator build + # required for the smoke gate. + OCI_CHART="oci://ghcr.io/documentdb/documentdb-operator" + + if [[ -n "$GHCR_LOGIN_TOKEN" ]]; then + printf '%s' "$GHCR_LOGIN_TOKEN" \ + | helm registry login ghcr.io --username "$GHCR_LOGIN_USERNAME" --password-stdin \ + || echo "GHCR login failed; continuing anonymously for the public chart." + fi + + # Resolve the highest stable released chart version from GHCR OCI tags + # (OCI registries don't support `helm search repo`). + GHCR_TOKEN=$(curl -sSL 'https://ghcr.io/token?scope=repository:documentdb/documentdb-operator:pull' | jq -r '.token // empty' 2>/dev/null || true) + CURL_ARGS=(-sSL "https://ghcr.io/v2/documentdb/documentdb-operator/tags/list") + if [[ -n "$GHCR_TOKEN" ]]; then + CURL_ARGS+=(-H "Authorization: Bearer $GHCR_TOKEN") + fi + CHART_VERSION=$(curl "${CURL_ARGS[@]}" 2>/dev/null \ + | jq -r '.tags[]?' 2>/dev/null \ + | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' \ + | sort -V | tail -1 || echo "") + if [[ -z "$CHART_VERSION" ]]; then + echo "::error::Could not resolve a released operator chart version from GHCR." + exit 1 + fi + echo "Installing operator chart version ${CHART_VERSION}" + + helm install documentdb-operator "$OCI_CHART" \ + --namespace "${OPERATOR_NS}" \ + --create-namespace \ + --version "$CHART_VERSION" \ + --wait --timeout=15m + + kubectl wait --for=condition=Available deployment/documentdb-operator \ + -n "${OPERATOR_NS}" --timeout=300s + + - name: Create DocumentDB credentials secret + run: | + kubectl create namespace "${DB_NS}" --dry-run=client -o yaml | kubectl apply -f - + # Sidecar injector plugin requires a secret named documentdb-credentials + # with username/password keys in the database namespace. + cat </dev/null \ + | jq '[.items[] | select(.status.phase=="Running" and ([.status.containerStatuses[]?.ready] | all))] | length' 2>/dev/null || echo 0) + echo " attempt ${i}: ${ready} ready pod(s)" + if [[ "${ready}" -ge 1 ]]; then + echo "DocumentDB pod is Ready." + break + fi + if [[ "${i}" -eq 40 ]]; then + echo "::error::DocumentDB cluster did not become ready in time." + kubectl get pods -n "${DB_NS}" -o wide || true + kubectl describe documentdb "${DB_NAME}" -n "${DB_NS}" || true + kubectl logs -n "${OPERATOR_NS}" deployment/documentdb-operator --tail=50 || true + exit 1 + fi + sleep 15 + done + + # The gateway sidecar needs a moment after the pod is Ready before it + # accepts Mongo-wire connections; wait for the service endpoints. + echo "Waiting for gateway service endpoints..." + for i in $(seq 1 20); do + eps=$(kubectl get endpoints "documentdb-service-${DB_NAME}" -n "${DB_NS}" \ + -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || echo "") + if [[ -n "${eps}" ]]; then + echo "Gateway endpoints ready: ${eps}" + break + fi + sleep 6 + done + + - name: Create driver connection secret (longhaul-documentdb-credentials) + run: | + # In-cluster DNS for the ClusterIP gateway service the operator creates. + HOST="documentdb-service-${DB_NAME}.${DB_NS}.svc" + URI="mongodb://${DB_USERNAME}:${DB_PASSWORD}@${HOST}:${GATEWAY_PORT}/?directConnection=true&authMechanism=SCRAM-SHA-256&tls=true&tlsAllowInvalidCertificates=true&replicaSet=rs0" + kubectl create secret generic longhaul-documentdb-credentials \ + -n "${DB_NS}" \ + --from-literal=uri="${URI}" \ + --dry-run=client -o yaml | kubectl apply -f - + + - name: Deploy long-haul driver (real manifests, bounded override) + run: | + # RBAC applies unmodified (namespace matches DB_NS). + kubectl apply -f test/longhaul/deploy/rbac.yaml + + # Apply the real ConfigMap + Deployment, substituting only the + # templated image ref with the locally built image loaded into kind. + sed -e "s|ghcr.io/__OWNER__/documentdb-kubernetes-operator/longhaul-test:__IMAGE_TAG__|${DRIVER_IMAGE}|g" \ + test/longhaul/deploy/deployment.yaml | kubectl apply -f - + + # Bounded, deterministic smoke override — patch ONLY runtime knobs on + # the shipped ConfigMap; the manifest structure is unchanged. + # - MAX_DURATION: finite run + # - RESET_DATA: fresh collection each CI run + # - MIN==MAX instances: disable disruptive scale ops (fast + stable) + # - short cadences so the verifier gets several cycles in the window + PATCH=$(jq -nc \ + --arg dur "${MAX_DURATION}" \ + '{data: { + LONGHAUL_MAX_DURATION: $dur, + LONGHAUL_RESET_DATA: "true", + LONGHAUL_NUM_WRITERS: "2", + LONGHAUL_OP_COOLDOWN: "30s", + LONGHAUL_STEADY_STATE_WAIT: "10s", + LONGHAUL_RECOVERY_TIMEOUT: "2m", + LONGHAUL_REPORT_INTERVAL: "30s", + LONGHAUL_MIN_INSTANCES: "1", + LONGHAUL_MAX_INSTANCES: "1" + }}') + kubectl patch configmap longhaul-test-config -n "${DB_NS}" --type merge -p "${PATCH}" + + # Restart so the pod picks up the patched ConfigMap (envFrom is only + # read at container start). + kubectl rollout restart deployment/longhaul-test -n "${DB_NS}" + kubectl rollout status deployment/longhaul-test -n "${DB_NS}" --timeout=120s + + - name: Wait for driver to complete one bounded run + id: wait + run: | + set -euo pipefail + deadline=$(( $(date +%s) + 900 )) # 15 min hard cap + exit_code="" + while [[ $(date +%s) -lt ${deadline} ]]; do + pod=$(kubectl get pods -n "${DB_NS}" \ + -l app.kubernetes.io/name=longhaul-test \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") + if [[ -z "${pod}" ]]; then + sleep 5; continue + fi + + # Look for a terminated driver container (current or previous + # restart) — that is the terminal verdict of a bounded run. + term_cur=$(kubectl get pod "${pod}" -n "${DB_NS}" \ + -o jsonpath='{.status.containerStatuses[0].state.terminated.exitCode}' 2>/dev/null || echo "") + term_prev=$(kubectl get pod "${pod}" -n "${DB_NS}" \ + -o jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}' 2>/dev/null || echo "") + + if [[ -n "${term_cur}" ]]; then + exit_code="${term_cur}" + elif [[ -n "${term_prev}" ]]; then + exit_code="${term_prev}" + fi + + if [[ -n "${exit_code}" ]]; then + echo "Driver container terminated with exit code ${exit_code}." + # Freeze state before the Deployment restarts the pod again. + kubectl scale deployment/longhaul-test -n "${DB_NS}" --replicas=0 || true + break + fi + echo " driver still running..." + sleep 10 + done + + if [[ -z "${exit_code}" ]]; then + echo "::error::Driver did not terminate within the smoke window." + exit 1 + fi + echo "exit_code=${exit_code}" >> "${GITHUB_OUTPUT}" + + - name: Assert PASS verdict + run: | + exit_code="${{ steps.wait.outputs.exit_code }}" + result=$(kubectl get configmap longhaul-report -n "${DB_NS}" \ + -o jsonpath='{.data.result}' 2>/dev/null || echo "MISSING") + echo "Driver exit code : ${exit_code}" + echo "Report result : ${result}" + + if [[ "${exit_code}" != "0" ]]; then + echo "::error::Driver exited non-zero (${exit_code})." + exit 1 + fi + if [[ "${result}" != "PASS" ]]; then + echo "::error::longhaul-report result is '${result}', expected PASS." + exit 1 + fi + echo "✅ Long-haul smoke gate passed (exit 0, report PASS)." + + - name: Diagnostics on failure + if: failure() + run: | + echo "===== DocumentDB pods =====" + kubectl get pods -n "${DB_NS}" -o wide || true + echo "===== Driver logs (current) =====" + kubectl logs -n "${DB_NS}" -l app.kubernetes.io/name=longhaul-test --tail=200 || true + echo "===== Driver logs (previous) =====" + kubectl logs -n "${DB_NS}" -l app.kubernetes.io/name=longhaul-test --previous --tail=200 || true + echo "===== longhaul-report ConfigMap =====" + kubectl get configmap longhaul-report -n "${DB_NS}" -o yaml || true + echo "===== DocumentDB describe =====" + kubectl describe documentdb "${DB_NAME}" -n "${DB_NS}" || true + echo "===== Operator logs =====" + kubectl logs -n "${OPERATOR_NS}" deployment/documentdb-operator --tail=100 || true From 1926d34b2d77aea0c576051c4db4577698972715 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Tue, 7 Jul 2026 12:00:38 -0400 Subject: [PATCH 10/15] test: reuse setup-test-environment for long-haul smoke gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor longhaul-smoke.yml to delegate environment bring-up (kind, cert-manager, CSI, operator install, DocumentDB deploy + readiness) to the shared .github/actions/setup-test-environment composite action instead of duplicating that logic inline. To make the action usable on a fork PR without building operator/database images, add a 'released-chart' fast path: when released-chart-version is set, skip the operator/sidecar image download/verify/kind-load steps and omit the DocumentDB CR image block so the released operator uses its own tested default (public) documentdb/gateway images. This is fully backward compatible — the new guards are keyed on released-chart-version, which existing callers (test-e2e.yml) never set, so their behavior is unchanged. The smoke job then builds only the lightweight driver image, loads it into the action's kind cluster, wires the connection secret, applies the real deploy/{rbac,deployment}.yaml (patching only ConfigMap runtime knobs for a bounded run), and asserts container exit 0 AND longhaul-report result == PASS. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .../actions/setup-test-environment/action.yml | 66 ++++--- .github/workflows/longhaul-smoke.yml | 183 ++++-------------- 2 files changed, 83 insertions(+), 166 deletions(-) diff --git a/.github/actions/setup-test-environment/action.yml b/.github/actions/setup-test-environment/action.yml index ccac467fe..debc0198c 100644 --- a/.github/actions/setup-test-environment/action.yml +++ b/.github/actions/setup-test-environment/action.yml @@ -111,7 +111,7 @@ runs: echo "✅ Architecture validation passed: $EXPECTED_ARCH" - name: Download and load Docker images (local build) - if: inputs.use-external-images == 'false' + if: inputs.use-external-images == 'false' && inputs.released-chart-version == '' shell: bash run: | echo "Loading platform-specific Docker images from artifacts for ${{ inputs.architecture }}..." @@ -156,7 +156,7 @@ runs: echo "✓ All required Docker images for $ARCH architecture are available" - name: Verify external Docker images (external images) - if: inputs.use-external-images == 'true' + if: inputs.use-external-images == 'true' && inputs.released-chart-version == '' shell: bash run: | echo "Using external Docker images with tag: ${{ inputs.image-tag }}" @@ -263,7 +263,7 @@ runs: echo "Resolved Gateway image: $GW_IMAGE" - name: Load Docker images into kind cluster (local build) - if: inputs.use-external-images == 'false' + if: inputs.use-external-images == 'false' && inputs.released-chart-version == '' shell: bash run: | echo "Loading local Docker images into kind cluster..." @@ -302,7 +302,7 @@ runs: echo "✓ All Docker images loaded into kind cluster successfully" - name: Pre-pull external images for kind cluster (external images) - if: inputs.use-external-images == 'true' + if: inputs.use-external-images == 'true' && inputs.released-chart-version == '' shell: bash run: | echo "Pre-pulling external Docker images for kind cluster..." @@ -726,27 +726,45 @@ runs: run: | echo "Deploying DocumentDB cluster on ${{ inputs.architecture }} architecture..." echo "Configuration: ${{ inputs.test-scenario-name }} (${{ inputs.node-count }} nodes, ${{ inputs.instances-per-node }} instances per node)" - + + # Decide whether to pin documentdb/gateway images on the CR. + # + # In released-chart mode (released-chart-version != '') with no explicit + # image overrides, we omit the image block so the released operator uses + # its own tested default (public) documentdb/gateway images. That is what + # lets a fork stand up a cluster without building/loading database images. + # In all other modes (the artifact/local/external build pipelines) we keep + # the resolved, loaded-into-kind images so behavior is unchanged. + INCLUDE_IMAGE_BLOCK="true" + if [[ -n "${{ inputs.released-chart-version }}" \ + && -z "${{ inputs.documentdb-image }}" \ + && -z "${{ inputs.gateway-image }}" ]]; then + INCLUDE_IMAGE_BLOCK="false" + echo "Released-chart mode with no image overrides: operator default images will be used." + fi + # Create DocumentDB resource - cat <-- + KIND_CLUSTER: documentdb-longhaul-smoke-amd64-smoke MAX_DURATION: ${{ github.event.inputs.max_duration || '3m' }} jobs: smoke: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 40 steps: - name: Checkout uses: actions/checkout@v4 @@ -81,156 +95,41 @@ jobs: - name: Build long-haul driver image run: | # Context MUST be the repo root: the Dockerfile copies ../shared and - # ../../operator/src (see test/longhaul/Dockerfile header). + # ../../operator/src (see test/longhaul/Dockerfile header). Built now, + # before the cluster exists, and loaded into kind after setup. docker build \ -f test/longhaul/Dockerfile \ -t "${DRIVER_IMAGE}" \ . - - name: Create kind cluster - uses: helm/kind-action@v1.12.0 + - name: Set up DocumentDB test environment (kind + operator + cluster) + uses: ./.github/actions/setup-test-environment with: - version: v0.31.0 - node_image: kindest/node:v1.30.4 - cluster_name: ${{ env.KIND_CLUSTER }} + test-type: "longhaul" + architecture: "amd64" + runner: "ubuntu-latest" + test-scenario-name: "smoke" + node-count: "1" + instances-per-node: "1" + cert-manager-namespace: ${{ env.CERT_MANAGER_NS }} + operator-namespace: ${{ env.OPERATOR_NS }} + db-namespace: ${{ env.DB_NS }} + db-cluster-name: ${{ env.DB_NAME }} + db-username: ${{ env.DB_USERNAME }} + db-password: ${{ env.DB_PASSWORD }} + db-port: ${{ env.GATEWAY_PORT }} + # image-tag / chart-version are required inputs but unused in + # released-chart mode; pass placeholders. + image-tag: "released" + chart-version: "released" + use-external-images: "true" + released-chart-version: "latest" + github-token: ${{ secrets.GITHUB_TOKEN }} + repository-owner: ${{ github.repository_owner }} - name: Load driver image into kind run: kind load docker-image "${DRIVER_IMAGE}" --name "${KIND_CLUSTER}" - - name: Install cert-manager - run: | - helm repo add jetstack https://charts.jetstack.io - helm repo update - helm install cert-manager jetstack/cert-manager \ - --namespace "${CERT_MANAGER_NS}" \ - --create-namespace \ - --version v1.15.3 \ - --set installCRDs=true \ - --wait --timeout=5m - - - name: Deploy CSI hostpath driver - run: | - chmod +x ./operator/src/scripts/test-scripts/deploy-csi-driver.sh - ./operator/src/scripts/test-scripts/deploy-csi-driver.sh - - - name: Install DocumentDB operator (public OCI chart) - env: - GHCR_LOGIN_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GHCR_LOGIN_USERNAME: ${{ github.actor }} - run: | - # The released chart is published from the canonical documentdb org - # and is public, so forks install it anonymously — no operator build - # required for the smoke gate. - OCI_CHART="oci://ghcr.io/documentdb/documentdb-operator" - - if [[ -n "$GHCR_LOGIN_TOKEN" ]]; then - printf '%s' "$GHCR_LOGIN_TOKEN" \ - | helm registry login ghcr.io --username "$GHCR_LOGIN_USERNAME" --password-stdin \ - || echo "GHCR login failed; continuing anonymously for the public chart." - fi - - # Resolve the highest stable released chart version from GHCR OCI tags - # (OCI registries don't support `helm search repo`). - GHCR_TOKEN=$(curl -sSL 'https://ghcr.io/token?scope=repository:documentdb/documentdb-operator:pull' | jq -r '.token // empty' 2>/dev/null || true) - CURL_ARGS=(-sSL "https://ghcr.io/v2/documentdb/documentdb-operator/tags/list") - if [[ -n "$GHCR_TOKEN" ]]; then - CURL_ARGS+=(-H "Authorization: Bearer $GHCR_TOKEN") - fi - CHART_VERSION=$(curl "${CURL_ARGS[@]}" 2>/dev/null \ - | jq -r '.tags[]?' 2>/dev/null \ - | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' \ - | sort -V | tail -1 || echo "") - if [[ -z "$CHART_VERSION" ]]; then - echo "::error::Could not resolve a released operator chart version from GHCR." - exit 1 - fi - echo "Installing operator chart version ${CHART_VERSION}" - - helm install documentdb-operator "$OCI_CHART" \ - --namespace "${OPERATOR_NS}" \ - --create-namespace \ - --version "$CHART_VERSION" \ - --wait --timeout=15m - - kubectl wait --for=condition=Available deployment/documentdb-operator \ - -n "${OPERATOR_NS}" --timeout=300s - - - name: Create DocumentDB credentials secret - run: | - kubectl create namespace "${DB_NS}" --dry-run=client -o yaml | kubectl apply -f - - # Sidecar injector plugin requires a secret named documentdb-credentials - # with username/password keys in the database namespace. - cat </dev/null \ - | jq '[.items[] | select(.status.phase=="Running" and ([.status.containerStatuses[]?.ready] | all))] | length' 2>/dev/null || echo 0) - echo " attempt ${i}: ${ready} ready pod(s)" - if [[ "${ready}" -ge 1 ]]; then - echo "DocumentDB pod is Ready." - break - fi - if [[ "${i}" -eq 40 ]]; then - echo "::error::DocumentDB cluster did not become ready in time." - kubectl get pods -n "${DB_NS}" -o wide || true - kubectl describe documentdb "${DB_NAME}" -n "${DB_NS}" || true - kubectl logs -n "${OPERATOR_NS}" deployment/documentdb-operator --tail=50 || true - exit 1 - fi - sleep 15 - done - - # The gateway sidecar needs a moment after the pod is Ready before it - # accepts Mongo-wire connections; wait for the service endpoints. - echo "Waiting for gateway service endpoints..." - for i in $(seq 1 20); do - eps=$(kubectl get endpoints "documentdb-service-${DB_NAME}" -n "${DB_NS}" \ - -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || echo "") - if [[ -n "${eps}" ]]; then - echo "Gateway endpoints ready: ${eps}" - break - fi - sleep 6 - done - - name: Create driver connection secret (longhaul-documentdb-credentials) run: | # In-cluster DNS for the ClusterIP gateway service the operator creates. From 05200fcffb8c41a254c9744b02f29a9f78ec7b92 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Tue, 7 Jul 2026 12:17:50 -0400 Subject: [PATCH 11/15] test(longhaul): rework smoke gate to build-artifact pattern Rewrite longhaul-smoke.yml to mirror test-e2e.yml: a reusable build job (test-build-and-package.yml) produces operator+sidecar from PR source while documentdb/gateway stay on public images via registry-mode probe, then the smoke job loads those artifacts into kind through the shared setup-test-environment action (use-external-images: false) and runs the real long-haul driver against a real DocumentDB for a bounded window, asserting exit 0 + longhaul-report result=PASS. Revert the earlier setup-test-environment/action.yml changes: the released-chart guards were only needed for fork verification, which is no longer the approach. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .../actions/setup-test-environment/action.yml | 66 +++++--------- .github/workflows/longhaul-smoke.yml | 91 ++++++++++++++----- 2 files changed, 90 insertions(+), 67 deletions(-) diff --git a/.github/actions/setup-test-environment/action.yml b/.github/actions/setup-test-environment/action.yml index debc0198c..ccac467fe 100644 --- a/.github/actions/setup-test-environment/action.yml +++ b/.github/actions/setup-test-environment/action.yml @@ -111,7 +111,7 @@ runs: echo "✅ Architecture validation passed: $EXPECTED_ARCH" - name: Download and load Docker images (local build) - if: inputs.use-external-images == 'false' && inputs.released-chart-version == '' + if: inputs.use-external-images == 'false' shell: bash run: | echo "Loading platform-specific Docker images from artifacts for ${{ inputs.architecture }}..." @@ -156,7 +156,7 @@ runs: echo "✓ All required Docker images for $ARCH architecture are available" - name: Verify external Docker images (external images) - if: inputs.use-external-images == 'true' && inputs.released-chart-version == '' + if: inputs.use-external-images == 'true' shell: bash run: | echo "Using external Docker images with tag: ${{ inputs.image-tag }}" @@ -263,7 +263,7 @@ runs: echo "Resolved Gateway image: $GW_IMAGE" - name: Load Docker images into kind cluster (local build) - if: inputs.use-external-images == 'false' && inputs.released-chart-version == '' + if: inputs.use-external-images == 'false' shell: bash run: | echo "Loading local Docker images into kind cluster..." @@ -302,7 +302,7 @@ runs: echo "✓ All Docker images loaded into kind cluster successfully" - name: Pre-pull external images for kind cluster (external images) - if: inputs.use-external-images == 'true' && inputs.released-chart-version == '' + if: inputs.use-external-images == 'true' shell: bash run: | echo "Pre-pulling external Docker images for kind cluster..." @@ -726,45 +726,27 @@ runs: run: | echo "Deploying DocumentDB cluster on ${{ inputs.architecture }} architecture..." echo "Configuration: ${{ inputs.test-scenario-name }} (${{ inputs.node-count }} nodes, ${{ inputs.instances-per-node }} instances per node)" - - # Decide whether to pin documentdb/gateway images on the CR. - # - # In released-chart mode (released-chart-version != '') with no explicit - # image overrides, we omit the image block so the released operator uses - # its own tested default (public) documentdb/gateway images. That is what - # lets a fork stand up a cluster without building/loading database images. - # In all other modes (the artifact/local/external build pipelines) we keep - # the resolved, loaded-into-kind images so behavior is unchanged. - INCLUDE_IMAGE_BLOCK="true" - if [[ -n "${{ inputs.released-chart-version }}" \ - && -z "${{ inputs.documentdb-image }}" \ - && -z "${{ inputs.gateway-image }}" ]]; then - INCLUDE_IMAGE_BLOCK="false" - echo "Released-chart mode with no image overrides: operator default images will be used." - fi - + # Create DocumentDB resource - { - echo "apiVersion: documentdb.io/preview" - echo "kind: DocumentDB" - echo "metadata:" - echo " name: ${{ inputs.db-cluster-name }}" - echo " namespace: ${{ inputs.db-namespace }}" - echo "spec:" - echo " nodeCount: ${{ inputs.node-count }}" - echo " instancesPerNode: ${{ inputs.instances-per-node }}" - if [[ "${INCLUDE_IMAGE_BLOCK}" == "true" ]]; then - echo " image:" - echo " documentDB: ${DOCUMENTDB_IMAGE_RESOLVED}" - echo " gateway: ${GATEWAY_IMAGE_RESOLVED}" - fi - echo " resource:" - echo " storage:" - echo " pvcSize: 5Gi" - echo " storageClass: csi-hostpath-sc" - echo " exposeViaService:" - echo " serviceType: ClusterIP" - } | kubectl apply -f - + cat <-- - KIND_CLUSTER: documentdb-longhaul-smoke-amd64-smoke - MAX_DURATION: ${{ github.event.inputs.max_duration || '3m' }} jobs: + # --------------------------------------------------------------------------- + # Build operator + sidecar from this PR's source; reuse public documentdb / + # gateway images (registry mode) so the database images are NOT rebuilt. + # Produces the platform-images tar + helm chart artifacts the setup action + # loads into kind. + # --------------------------------------------------------------------------- + build: + name: Build Images and Charts + uses: ./.github/workflows/test-build-and-package.yml + with: + version: "0.2.0" + secrets: inherit + smoke: - runs-on: ubuntu-latest + name: Long-Haul Smoke + needs: build + if: always() && needs.build.result == 'success' + runs-on: ubuntu-22.04 timeout-minutes: 40 + env: + IMAGE_TAG: ${{ needs.build.outputs.image_tag }} + EXT_IMAGE_TAG: ${{ needs.build.outputs.ext_image_tag }} + CHART_VERSION: ${{ needs.build.outputs.chart_version || '0.1.0' }} + DRIVER_IMAGE: longhaul-test:smoke + # Must match the cluster name the composite action derives: + # documentdb--- + KIND_CLUSTER: documentdb-longhaul-amd64-smoke + MAX_DURATION: ${{ github.event.inputs.max_duration || '3m' }} steps: - name: Checkout uses: actions/checkout@v4 @@ -102,12 +130,27 @@ jobs: -t "${DRIVER_IMAGE}" \ . + # Only the artifacts the setup action consumes: the consolidated + # platform-specific image tarball (operator/sidecar/documentdb/gateway, + # loaded into kind) and the amd64 helm chart. + - name: Download platform images artifact + uses: actions/download-artifact@v4 + with: + name: build-platform-images + path: ./artifacts/build-platform-images + + - name: Download helm chart artifact + uses: actions/download-artifact@v4 + with: + name: build-helm-chart-amd64 + path: ./artifacts/build-helm-chart-amd64 + - name: Set up DocumentDB test environment (kind + operator + cluster) uses: ./.github/actions/setup-test-environment with: test-type: "longhaul" architecture: "amd64" - runner: "ubuntu-latest" + runner: "ubuntu-22.04" test-scenario-name: "smoke" node-count: "1" instances-per-node: "1" @@ -118,12 +161,10 @@ jobs: db-username: ${{ env.DB_USERNAME }} db-password: ${{ env.DB_PASSWORD }} db-port: ${{ env.GATEWAY_PORT }} - # image-tag / chart-version are required inputs but unused in - # released-chart mode; pass placeholders. - image-tag: "released" - chart-version: "released" - use-external-images: "true" - released-chart-version: "latest" + image-tag: ${{ env.IMAGE_TAG }} + documentdb-image-tag: ${{ env.EXT_IMAGE_TAG }} + chart-version: ${{ env.CHART_VERSION }} + use-external-images: "false" github-token: ${{ secrets.GITHUB_TOKEN }} repository-owner: ${{ github.repository_owner }} From 050ab451994af30184857227f7e48f7dd60ff5c1 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Tue, 7 Jul 2026 12:48:47 -0400 Subject: [PATCH 12/15] fix(longhaul): use tlsInsecure in smoke driver URI The smoke run reached the gateway but the driver's TLS handshake failed with 'x509: certificate signed by unknown authority'. NewFromURI relies solely on URI params for TLS, and the Go mongo driver v2 honors tlsInsecure=true (not tlsAllowInvalidCertificates) to skip verification of the self-signed gateway cert. Switch the URI to tlsInsecure=true and drop the redundant replicaSet param, matching the proven form in test/longhaul/README.md. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/longhaul-smoke.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/longhaul-smoke.yml b/.github/workflows/longhaul-smoke.yml index 0a2eb85f4..2f50ead1c 100644 --- a/.github/workflows/longhaul-smoke.yml +++ b/.github/workflows/longhaul-smoke.yml @@ -175,7 +175,10 @@ jobs: run: | # In-cluster DNS for the ClusterIP gateway service the operator creates. HOST="documentdb-service-${DB_NAME}.${DB_NS}.svc" - URI="mongodb://${DB_USERNAME}:${DB_PASSWORD}@${HOST}:${GATEWAY_PORT}/?directConnection=true&authMechanism=SCRAM-SHA-256&tls=true&tlsAllowInvalidCertificates=true&replicaSet=rs0" + # tlsInsecure=true (NOT tlsAllowInvalidCertificates) is the option the + # Go mongo driver v2 honors to skip verification of the self-signed + # gateway cert — this matches test/longhaul/README.md. + URI="mongodb://${DB_USERNAME}:${DB_PASSWORD}@${HOST}:${GATEWAY_PORT}/?directConnection=true&authMechanism=SCRAM-SHA-256&tls=true&tlsInsecure=true" kubectl create secret generic longhaul-documentdb-credentials \ -n "${DB_NS}" \ --from-literal=uri="${URI}" \ From dd49e8367adabd578356138e33feb070a2ac7391 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Wed, 8 Jul 2026 19:37:08 -0400 Subject: [PATCH 13/15] =?UTF-8?q?fix(longhaul):=20address=20PR=20review=20?= =?UTF-8?q?=E2=80=94=20cluster-access=20gate,=20helm=20deps,=20image=20pul?= =?UTF-8?q?l=20policy,=20doc=20accuracy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review comments on #413: - deploy: drop `|| true` from the "Verify cluster access" step so it fails fast on an unreachable/expired kubeconfig instead of silently proceeding to apply/rollout (alaye-ms). - monitor: run `helm dependency update` before `helm upgrade`; the chart's cloudnative-pg dependency is gitignored (charts/, Chart.lock), so the upgrade would fail on a fresh CI checkout. - deployment: set imagePullPolicy to Always so mutable tags (e.g. :main) are not silently served from a stale node cache. - deployment/rbac/monitor/image-build: correct leftover "Job"/"PVC" design language — the driver runs as a Deployment (no PVC); it resumes writer sequences from the DocumentDB workload collection and checkpoints to the longhaul-report ConfigMap. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/longhaul-deploy.yml | 7 +++++-- .github/workflows/longhaul-image-build.yml | 2 +- .github/workflows/longhaul-monitor.yaml | 12 ++++++++++-- test/longhaul/deploy/deployment.yaml | 13 ++++++++----- test/longhaul/deploy/rbac.yaml | 2 +- 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/longhaul-deploy.yml b/.github/workflows/longhaul-deploy.yml index 261fc051a..4491a7696 100644 --- a/.github/workflows/longhaul-deploy.yml +++ b/.github/workflows/longhaul-deploy.yml @@ -80,8 +80,11 @@ jobs: - name: Verify cluster access run: | set -euo pipefail - kubectl version --client=false --output=yaml | head -20 || true - kubectl -n "$NAMESPACE" get all || true + # No `|| true`: this step must fail fast if the cluster is + # unreachable (bad/expired kubeconfig), instead of silently + # continuing into the apply/rollout steps. + kubectl version --client=false --output=yaml + kubectl -n "$NAMESPACE" get all - name: Compute image ref id: img diff --git a/.github/workflows/longhaul-image-build.yml b/.github/workflows/longhaul-image-build.yml index 8e7e3e74b..ce9436945 100644 --- a/.github/workflows/longhaul-image-build.yml +++ b/.github/workflows/longhaul-image-build.yml @@ -113,5 +113,5 @@ jobs: echo "| Immutable | \`${{ steps.refs.outputs.image }}:sha-${{ steps.refs.outputs.short_sha }}\` |" echo "| Rolling | \`${{ steps.refs.outputs.image }}:main\` |" echo "" - echo "Deploy by setting the Job spec image to the immutable tag." + echo "Deploy by setting the driver Deployment image to the immutable tag." } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/longhaul-monitor.yaml b/.github/workflows/longhaul-monitor.yaml index 74ecf28e0..925abf97d 100644 --- a/.github/workflows/longhaul-monitor.yaml +++ b/.github/workflows/longhaul-monitor.yaml @@ -4,7 +4,7 @@ # Long Haul Test Monitor # # Polls the AKS cluster every hour to: -# 1. Check health: Job status, report staleness, test result +# 1. Check health: Deployment status, report staleness, test result # 2. Auto-upgrade operator: If healthy, upgrade the operator to the latest # version published to GHCR (via Helm). # 3. Publish desired DocumentDB version: Write the latest published DocumentDB @@ -13,7 +13,8 @@ # performs the upgrade in-band as a load-aware operation, so continuous # writers/verifiers can catch any data-integrity issues caused by it. # -# Fails if: Job failed, ConfigMap reports FAIL, report is stale (>2h), or Job not found. +# Fails if: the driver Deployment is unhealthy, the report ConfigMap +# reports FAIL, the report is stale (>2h), or the report ConfigMap is not found. # name: "Long Haul Monitor" @@ -231,6 +232,13 @@ jobs: echo "📦 Applying CRDs from chart..." kubectl apply -f operator/documentdb-helm-chart/crds/ + # The chart's cloudnative-pg dependency is NOT vendored in git + # (charts/ and Chart.lock are gitignored), so resolve + download + # it before upgrading, otherwise `helm upgrade` fails with a + # missing-dependency error on a fresh checkout. + echo "📦 Resolving chart dependencies..." + helm dependency update operator/documentdb-helm-chart + helm upgrade documentdb-operator operator/documentdb-helm-chart \ -n "${{ env.OPERATOR_NAMESPACE }}" \ --set image.tag="$EXPECTED" \ diff --git a/test/longhaul/deploy/deployment.yaml b/test/longhaul/deploy/deployment.yaml index d593dc3b8..49d5c09fe 100644 --- a/test/longhaul/deploy/deployment.yaml +++ b/test/longhaul/deploy/deployment.yaml @@ -7,11 +7,13 @@ # A Job's pod spec is immutable, so rolling a new image requires # delete-and-recreate. Using Deployment(replicas=1) lets the deploy # workflow do `kubectl set image deployment/longhaul-test ...` for -# one-line image rolls. The driver itself is restart-safe: it persists -# journal/checkpoint state to PVC and resumes from it. +# one-line image rolls. The driver itself is restart-safe: it resumes +# each writer's sequence from the DocumentDB workload collection and +# reads/writes progress to the longhaul-report ConfigMap (no PVC). # # Strategy: Recreate (single-replica stateful driver — never run two -# pods concurrently against the same PVC). +# driver pods concurrently against the same DocumentDB cluster / +# workload collection). # # Failure semantics: on critical failure (data loss, policy violation) # the driver exits non-zero. The Deployment auto-restarts the pod, which @@ -53,7 +55,8 @@ data: LONGHAUL_REPORT_INTERVAL: "1h" # Whether to drop the workload collection on startup. MUST stay "false" # on the long-haul cluster — restarts of the driver pod (rolling image - # updates, node drains) need to resume durability state from PVC. + # updates, node drains) need to resume durability state from the existing + # DocumentDB workload collection and longhaul-report ConfigMap checkpoints. # Override to "true" only for local/dev iterations. LONGHAUL_RESET_DATA: "false" # Replica range for scale operations. @@ -89,7 +92,7 @@ spec: containers: - name: driver image: ghcr.io/__OWNER__/documentdb-kubernetes-operator/longhaul-test:__IMAGE_TAG__ - imagePullPolicy: IfNotPresent + imagePullPolicy: Always envFrom: - configMapRef: name: longhaul-test-config diff --git a/test/longhaul/deploy/rbac.yaml b/test/longhaul/deploy/rbac.yaml index d1a10cf02..f5ed5abcd 100644 --- a/test/longhaul/deploy/rbac.yaml +++ b/test/longhaul/deploy/rbac.yaml @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -# ServiceAccount for the long-haul test Job. +# ServiceAccount for the long-haul test driver Deployment. apiVersion: v1 kind: ServiceAccount metadata: From c4aa39d982ff66012804c1e9a369cb1656bdb7f0 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Wed, 8 Jul 2026 19:47:11 -0400 Subject: [PATCH 14/15] ci(longhaul): make kubeconform lint resilient to schema-download rate limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kubeconform step downloads standard Kubernetes JSON schemas from raw.githubusercontent.com, which intermittently returns HTTP 429 to shared CI runner IPs. Both the initial run and a re-run failed purely on schema downloads (Invalid: 0 in every attempt — no actual manifest violations). Make the gate robust: - Cache the downloaded schemas across runs via actions/cache so steady-state lint performs zero schema downloads. - Pass `-cache` and wrap kubeconform in a retry-with-backoff loop so a cold cache hitting a transient 429 recovers instead of failing; the cache lets each retry re-fetch only the schemas still missing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/lint-longhaul-workflows.yml | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint-longhaul-workflows.yml b/.github/workflows/lint-longhaul-workflows.yml index 18f494b60..507f56f4c 100644 --- a/.github/workflows/lint-longhaul-workflows.yml +++ b/.github/workflows/lint-longhaul-workflows.yml @@ -68,19 +68,49 @@ jobs: .github/workflows/longhaul-monitor.yaml \ test/longhaul/deploy/ + - name: Cache kubeconform schemas + # Persist downloaded JSON schemas across runs so we don't re-hit + # raw.githubusercontent.com (which rate-limits with HTTP 429) on + # every run. Once populated, steady-state lint needs zero schema + # downloads; the retry loop below only covers a cold cache. + uses: actions/cache@v4 + with: + path: /tmp/kubeconform-cache + key: kubeconform-schemas-1.30.0-v1 + - name: kubeconform manifests run: | set -euo pipefail curl -sSL https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz \ | tar -xz -C /usr/local/bin kubeconform + mkdir -p /tmp/kubeconform-cache # Skip CRD-typed resources (DocumentDB) since their schemas live # in operator/documentdb-helm-chart/crds — pass via -schema-location. - kubeconform -strict -summary \ - -kubernetes-version 1.30.0 \ - -schema-location default \ - -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \ - -skip 'DocumentDB' \ - test/longhaul/deploy/ + # + # Schemas are fetched from raw.githubusercontent.com, which + # intermittently rate-limits shared CI runner IPs (HTTP 403 / + # "giving up after 3 attempts"). Cache downloaded schemas and retry + # with backoff so a transient rate limit doesn't fail the gate; the + # cache means each retry only re-fetches the schemas still missing. + run_kubeconform() { + kubeconform -strict -summary \ + -cache /tmp/kubeconform-cache \ + -kubernetes-version 1.30.0 \ + -schema-location default \ + -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \ + -skip 'DocumentDB' \ + test/longhaul/deploy/ + } + n=0 + until run_kubeconform; do + n=$((n + 1)) + if [ "$n" -ge 4 ]; then + echo "::error::kubeconform failed after $n attempts" + exit 1 + fi + echo "kubeconform attempt $n failed (likely schema-download rate limit); retrying in $((n * 30))s..." + sleep "$((n * 30))" + done - name: helm template smoke run: | From 40588d05620002d1a2b2e3d4c14fc1f7d9cbaa22 Mon Sep 17 00:00:00 2001 From: Wenting Wu Date: Wed, 8 Jul 2026 20:38:30 -0400 Subject: [PATCH 15/15] ci(longhaul): override imagePullPolicy to IfNotPresent in kind smoke gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shipped driver manifest now uses imagePullPolicy: Always (so the registry-based long-haul deploy always refreshes mutable tags like :main). The kind smoke gate side-loads the locally built longhaul-test:smoke image into the cluster, where it exists on no registry — Always forces a pull that fails with ErrImagePull and the rollout times out. Override the policy to IfNotPresent during the smoke deploy so the side-loaded image is used. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Wenting Wu --- .github/workflows/longhaul-smoke.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/longhaul-smoke.yml b/.github/workflows/longhaul-smoke.yml index 2f50ead1c..04a6be612 100644 --- a/.github/workflows/longhaul-smoke.yml +++ b/.github/workflows/longhaul-smoke.yml @@ -191,7 +191,12 @@ jobs: # Apply the real ConfigMap + Deployment, substituting only the # templated image ref with the locally built image loaded into kind. + # The shipped manifest uses imagePullPolicy: Always (correct for the + # registry-based long-haul deploy of mutable tags), but here the + # image is side-loaded into kind and exists on no registry, so + # override to IfNotPresent to use the local image instead of a pull. sed -e "s|ghcr.io/__OWNER__/documentdb-kubernetes-operator/longhaul-test:__IMAGE_TAG__|${DRIVER_IMAGE}|g" \ + -e "s|imagePullPolicy: Always|imagePullPolicy: IfNotPresent|g" \ test/longhaul/deploy/deployment.yaml | kubectl apply -f - # Bounded, deterministic smoke override — patch ONLY runtime knobs on