diff --git a/.github/workflows/lint-longhaul-workflows.yml b/.github/workflows/lint-longhaul-workflows.yml new file mode 100644 index 00000000..507f56f4 --- /dev/null +++ b/.github/workflows/lint-longhaul-workflows.yml @@ -0,0 +1,177 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Pre-merge gating for the long-haul CI/CD bundle. +# +# The long-haul {build, deploy, monitor} workflows only run post-merge +# (push to main / scheduled / workflow_run), so a typo in any of them is +# normally not caught until they fire against the live AKS cluster. This +# workflow runs on PRs that touch any of those files (plus the long-haul +# driver sources, deploy manifests, and Dockerfile) and exercises them +# statically: +# +# - actionlint : GitHub Actions syntax, expression bugs, missing +# permissions, embedded shellcheck on `run:` blocks. +# - yamllint : strict YAML formatting. +# - kubeconform : Kubernetes manifest schema validation against the +# v1.30 schema set (matches our compatibility matrix). +# - helm template : ensure the operator chart still renders so the +# auto-upgrade job in monitor.yaml will succeed. +# - go vet/test : the driver has real unit tests (config, journal, +# monitor, operations, report); compiling in the image +# build is not enough, so run them here. +# - docker build : build the long-haul image with push:false to catch +# context / go.mod-replace problems before they break +# the scheduled image build. +name: Lint Long-Haul Workflows + +on: + pull_request: + paths: + - ".github/workflows/longhaul-*.y*ml" + - ".github/workflows/lint-longhaul-workflows.yml" + - "test/longhaul/**" + - "operator/documentdb-helm-chart/**" + +permissions: + contents: read + +concurrency: + group: lint-longhaul-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: actionlint + # reviewdog/action-actionlint scans .github/workflows/ by default + # and embeds shellcheck so `run:` blocks are linted too. We rely + # on the default discovery — there is no point linting only the + # long-haul subset since neighbouring workflows share a runner + # image and any syntax problem would block CI anyway. + uses: reviewdog/action-actionlint@v1 + with: + reporter: github-pr-check + fail_level: error + + - name: yamllint + run: | + set -euo pipefail + pip install --quiet yamllint + yamllint -d '{extends: default, rules: {line-length: {max: 200}, comments-indentation: disable, document-start: disable, truthy: {check-keys: false}}}' \ + .github/workflows/longhaul-image-build.yml \ + .github/workflows/longhaul-deploy.yml \ + .github/workflows/longhaul-monitor.yaml \ + test/longhaul/deploy/ + + - name: Cache kubeconform schemas + # Persist downloaded JSON schemas across runs so we don't re-hit + # raw.githubusercontent.com (which rate-limits with HTTP 429) on + # every run. Once populated, steady-state lint needs zero schema + # downloads; the retry loop below only covers a cold cache. + uses: actions/cache@v4 + with: + path: /tmp/kubeconform-cache + key: kubeconform-schemas-1.30.0-v1 + + - name: kubeconform manifests + run: | + set -euo pipefail + curl -sSL https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz \ + | tar -xz -C /usr/local/bin kubeconform + mkdir -p /tmp/kubeconform-cache + # Skip CRD-typed resources (DocumentDB) since their schemas live + # in operator/documentdb-helm-chart/crds — pass via -schema-location. + # + # Schemas are fetched from raw.githubusercontent.com, which + # intermittently rate-limits shared CI runner IPs (HTTP 403 / + # "giving up after 3 attempts"). Cache downloaded schemas and retry + # with backoff so a transient rate limit doesn't fail the gate; the + # cache means each retry only re-fetches the schemas still missing. + run_kubeconform() { + kubeconform -strict -summary \ + -cache /tmp/kubeconform-cache \ + -kubernetes-version 1.30.0 \ + -schema-location default \ + -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \ + -skip 'DocumentDB' \ + test/longhaul/deploy/ + } + n=0 + until run_kubeconform; do + n=$((n + 1)) + if [ "$n" -ge 4 ]; then + echo "::error::kubeconform failed after $n attempts" + exit 1 + fi + echo "kubeconform attempt $n failed (likely schema-download rate limit); retrying in $((n * 30))s..." + sleep "$((n * 30))" + done + + - name: helm template smoke + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + # Fetch chart dependencies (e.g. cloudnative-pg) so the chart can + # be templated — mirrors the release workflows. + helm dependency update operator/documentdb-helm-chart + # If the chart fails to template, the monitor's auto-upgrade + # job will fail at runtime against the live cluster. + helm template documentdb-operator operator/documentdb-helm-chart \ + --namespace documentdb-operator \ + --api-versions cert-manager.io/v1/Certificate \ + > /tmp/rendered.yaml + # Sanity: rendered output must include the operator Deployment we + # target by name in monitor.yaml. + grep -q 'name: documentdb-operator$' /tmp/rendered.yaml \ + || (echo "::error::Rendered chart missing 'documentdb-operator' Deployment — monitor.yaml's name-based selector will fail" && exit 1) + + go-test: + # The driver has real unit tests (config, journal, monitor, operations, + # report). The image build only compiles them — run them here so a logic + # regression in the reporter/thresholds is caught at PR time. + runs-on: ubuntu-22.04 + timeout-minutes: 15 + defaults: + run: + working-directory: test/longhaul + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: test/longhaul/go.mod + cache-dependency-path: test/longhaul/go.sum + + - name: go vet + run: go vet ./... + + - name: go test + run: go test ./... -count=1 + + image-build-smoke: + # Mirrors the production longhaul-image-build.yml exactly (context, file, + # platforms) but with push:false. Catches context/replace-directive bugs + # at PR time rather than on the main-branch scheduled run. + runs-on: ubuntu-22.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build long-haul image (no push) + uses: docker/build-push-action@v6 + with: + context: . + file: test/longhaul/Dockerfile + push: false + platforms: linux/amd64 + cache-from: type=gha,scope=longhaul-pr + cache-to: type=gha,scope=longhaul-pr,mode=max diff --git a/.github/workflows/longhaul-deploy.yml b/.github/workflows/longhaul-deploy.yml new file mode 100644 index 00000000..4491a769 --- /dev/null +++ b/.github/workflows/longhaul-deploy.yml @@ -0,0 +1,184 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: LONGHAUL - Deploy Test Driver to AKS + +# Rolls a longhaul-test image onto the long-haul AKS cluster. +# +# Auth: a kubeconfig for the long-haul AKS cluster, provisioned out of band +# and stored as the LONGHAUL_KUBECONFIG repo secret. The kubeconfig MUST be +# privileged enough to: +# * apply Deployments + ConfigMaps in documentdb-test-ns (this workflow) +# * roll the operator via Helm in documentdb-operator + apply CRDs +# cluster-wide (longhaul-monitor.yaml's upgrade job) +# In practice that means cluster-admin or a sufficiently broad ClusterRole. +# +# Note: test/longhaul/deploy/rbac.yaml is the in-cluster RBAC granted to +# the long-haul *driver pod* (its ServiceAccount), not to this workflow. +# The two are independent: the pod's SA only needs namespace-scoped pod / +# documentdb / configmap access; this workflow needs more. +# +# Triggers: +# * workflow_run — auto-deploys after a successful build on main, pinning +# to the immutable :sha- tag built by that run. +# * workflow_dispatch — manual roll/rollback to a specific tag (e.g. an +# earlier :sha-, or :main for a quick smoke). + +on: + workflow_run: + workflows: ["LONGHAUL - Build Test Driver Image"] + types: [completed] + branches: [main] + + workflow_dispatch: + inputs: + image_tag: + description: 'Image tag to deploy (e.g. sha-5c30d7b or main).' + required: true + default: 'main' + +permissions: + contents: read + +env: + NAMESPACE: documentdb-test-ns + DEPLOYMENT: longhaul-test + CONTAINER: driver + +concurrency: + group: longhaul-deploy + cancel-in-progress: false + +jobs: + deploy: + name: Deploy to AKS + runs-on: ubuntu-22.04 + timeout-minutes: 15 + # When triggered by workflow_run, only proceed if the upstream build succeeded. + # workflow_dispatch always proceeds. + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Write kubeconfig + # Pass the secret via env, not ${{ }} expression. Expression-style + # interpolation pastes the secret value directly into the shell + # script before execution, which can lead to subtle shell-injection + # if the secret contains characters like $ or backticks. Using env: + # keeps the value as a literal string in the runner's environment. + env: + KUBECONFIG_CONTENT: ${{ secrets.LONGHAUL_KUBECONFIG }} + run: | + set -euo pipefail + install -m 700 -d "$RUNNER_TEMP/.kube" + KCFG="$RUNNER_TEMP/.kube/config" + printf '%s' "$KUBECONFIG_CONTENT" > "$KCFG" + chmod 600 "$KCFG" + echo "KUBECONFIG=$KCFG" >> "$GITHUB_ENV" + + - name: Verify cluster access + run: | + set -euo pipefail + # No `|| true`: this step must fail fast if the cluster is + # unreachable (bad/expired kubeconfig), instead of silently + # continuing into the apply/rollout steps. + kubectl version --client=false --output=yaml + kubectl -n "$NAMESPACE" get all + + - name: Compute image ref + id: img + env: + # workflow_dispatch path: maintainer-supplied tag. + # workflow_run path: derive immutable :sha- from the upstream + # build's head SHA, so we deploy exactly what was just built. + DISPATCH_TAG: ${{ github.event.inputs.image_tag }} + UPSTREAM_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + set -euo pipefail + OWNER_LC="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" + REPO_NAME="$(basename "${{ github.repository }}")" + + if [ "${{ github.event_name }}" = "workflow_run" ]; then + SHORT_SHA="$(echo "${UPSTREAM_SHA}" | cut -c1-7)" + TAG="sha-${SHORT_SHA}" + MUTABLE=false + else + TAG="${DISPATCH_TAG}" + # Treat anything that isn't an immutable sha-* tag as mutable + # (e.g. :main) so we force a rollout restart to re-pull. + case "$TAG" in + sha-*) MUTABLE=false ;; + *) MUTABLE=true ;; + esac + fi + + IMAGE="ghcr.io/${OWNER_LC}/${REPO_NAME}/longhaul-test:${TAG}" + { + echo "tag=${TAG}" + echo "image=${IMAGE}" + echo "mutable=${MUTABLE}" + } >> "$GITHUB_OUTPUT" + echo "Resolved image: ${IMAGE} (mutable=${MUTABLE})" + + - name: Apply Deployment manifest + # The Deployment manifest is the only thing the deploy workflow + # owns. Bootstrap resources (namespace, DocumentDB CR, credentials + # secret, driver ServiceAccount/Role/ClusterRole) live in + # setup.yaml + rbac.yaml and are applied once by a cluster admin. + # + # `kubectl apply` with the substituted image tag is sufficient to + # trigger a rollout when the tag changes — no extra `set image` + # call needed. + run: | + set -euo pipefail + OWNER_LC="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" + TAG='${{ steps.img.outputs.tag }}' + + sed -e "s|__OWNER__|${OWNER_LC}|g" \ + -e "s|__IMAGE_TAG__|${TAG}|g" \ + test/longhaul/deploy/deployment.yaml | kubectl apply -f - + + - name: Force re-pull for mutable tags + # `kubectl apply` only triggers a rollout when the pod-spec changes. + # For mutable tags like :main, the spec stays identical even when + # the underlying image moves — so we need an explicit restart to + # re-pull. Immutable :sha- tags don't need this. + if: steps.img.outputs.mutable == 'true' + run: | + set -euo pipefail + kubectl -n "$NAMESPACE" rollout restart "deployment/${DEPLOYMENT}" + + - name: Wait for rollout + run: | + set -euo pipefail + kubectl -n "$NAMESPACE" rollout status \ + "deployment/${DEPLOYMENT}" --timeout=5m + + - name: Pod status + if: always() + run: | + set -euo pipefail + kubectl -n "$NAMESPACE" get deployment "${DEPLOYMENT}" -o wide || true + kubectl -n "$NAMESPACE" get pods -l app.kubernetes.io/name=longhaul-test -o wide || true + POD="$(kubectl -n "$NAMESPACE" get pod -l app.kubernetes.io/name=longhaul-test \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + if [ -n "$POD" ]; then + echo "::group::Recent driver logs ($POD)" + kubectl -n "$NAMESPACE" logs "$POD" --tail=80 || true + echo "::endgroup::" + fi + + - name: Summary + if: always() + run: | + { + echo "## Long-haul deploy" + echo "" + echo "| Field | Value |" + echo "|-------|-------|" + echo "| Trigger | \`${{ github.event_name }}\` |" + echo "| Image | \`${{ steps.img.outputs.image }}\` |" + echo "| Namespace | \`$NAMESPACE\` |" + echo "| Deployment | \`$DEPLOYMENT\` |" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/longhaul-image-build.yml b/.github/workflows/longhaul-image-build.yml new file mode 100644 index 00000000..ce943694 --- /dev/null +++ b/.github/workflows/longhaul-image-build.yml @@ -0,0 +1,117 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: LONGHAUL - Build Test Driver Image + +# Builds the long-haul test driver image (test/longhaul/Dockerfile) and pushes +# it to GHCR at: +# ghcr.io//documentdb-kubernetes-operator/longhaul-test: +# +# Auth: built-in GITHUB_TOKEN (no extra secrets needed). Same pattern as the +# operator/sidecar and database image build workflows in this repo. +# +# Tags pushed every run: +# :sha- immutable, deployable +# :main rolling pointer to the latest main build (convenience only; +# pin to :sha- for actual deployments) +# +# This workflow does NOT deploy. Long-haul deploy is a separate workflow +# (workflow_dispatch) that uses these tags. + +on: + push: + branches: + - main + paths: + - 'test/longhaul/**' + - '.github/workflows/longhaul-image-build.yml' + + workflow_dispatch: + inputs: + extra_tag: + description: 'Optional additional tag (e.g. dev-myfeature). Leave blank for none.' + required: false + default: '' + +permissions: + contents: read + packages: write # push to GHCR + +env: + IMAGE_NAME: longhaul-test + # Build context MUST be the repo root, not test/longhaul/, because + # test/longhaul/go.mod has `replace` directives pointing to + # ../shared and ../../operator/src (see Dockerfile header at line 5-8). + CONTEXT: . + DOCKERFILE: test/longhaul/Dockerfile + +concurrency: + group: longhaul-image-build + cancel-in-progress: false + +jobs: + build-and-push: + name: Build and push longhaul image + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Compute image refs + id: refs + run: | + set -euo pipefail + OWNER_LC="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" + REPO_NAME="$(basename "${{ github.repository }}")" + IMG="ghcr.io/${OWNER_LC}/${REPO_NAME}/${IMAGE_NAME}" + SHORT_SHA="$(echo "${GITHUB_SHA}" | cut -c1-7)" + + TAGS="${IMG}:sha-${SHORT_SHA},${IMG}:main" + if [ -n "${{ github.event.inputs.extra_tag }}" ]; then + TAGS="${TAGS},${IMG}:${{ github.event.inputs.extra_tag }}" + fi + + { + echo "image=${IMG}" + echo "short_sha=${SHORT_SHA}" + echo "tags=${TAGS}" + } >> "$GITHUB_OUTPUT" + echo "Will push: ${TAGS}" + + - name: Login to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ${{ env.CONTEXT }} + file: ${{ env.DOCKERFILE }} + platforms: linux/amd64 + push: true + tags: ${{ steps.refs.outputs.tags }} + provenance: false + # GHA cache instead of a :buildcache image tag — keeps GHCR clean + # and is automatically GC'd by GitHub. + cache-from: type=gha,scope=longhaul-test + cache-to: type=gha,scope=longhaul-test,mode=max + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.created=${{ github.event.repository.updated_at }} + + - name: Summary + run: | + { + echo "## Long-haul image pushed" + echo "" + echo "| Tag | Reference |" + echo "|-----|-----------|" + echo "| Immutable | \`${{ steps.refs.outputs.image }}:sha-${{ steps.refs.outputs.short_sha }}\` |" + echo "| Rolling | \`${{ steps.refs.outputs.image }}:main\` |" + echo "" + echo "Deploy by setting the driver Deployment image to the immutable tag." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/longhaul-monitor.yaml b/.github/workflows/longhaul-monitor.yaml new file mode 100644 index 00000000..925abf97 --- /dev/null +++ b/.github/workflows/longhaul-monitor.yaml @@ -0,0 +1,303 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Long Haul Test Monitor +# +# Polls the AKS cluster every hour to: +# 1. Check health: Deployment status, report staleness, test result +# 2. Auto-upgrade operator: If healthy, upgrade the operator to the latest +# version published to GHCR (via Helm). +# 3. Publish desired DocumentDB version: Write the latest published DocumentDB +# tag to the `longhaul-versions` ConfigMap. This workflow does NOT upgrade +# DocumentDB itself — the long-haul test driver reads the ConfigMap and +# performs the upgrade in-band as a load-aware operation, so continuous +# writers/verifiers can catch any data-integrity issues caused by it. +# +# Fails if: the driver Deployment is unhealthy, the report ConfigMap +# reports FAIL, the report is stale (>2h), or the report ConfigMap is not found. +# +name: "Long Haul Monitor" + +on: + schedule: + - cron: "0 * * * *" # Every hour (matches LONGHAUL_REPORT_INTERVAL) + workflow_dispatch: {} # Allow manual trigger + +permissions: + contents: read + packages: read # Required: gh api /orgs/.../packages/container/.../versions + +concurrency: + group: longhaul-monitor + cancel-in-progress: false + +env: + NAMESPACE: documentdb-test-ns + DEPLOYMENT_NAME: longhaul-test + OPERATOR_NAMESPACE: documentdb-operator + OPERATOR_DEPLOYMENT: documentdb-operator + +jobs: + check-longhaul: + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - name: Set up kubeconfig + env: + KUBECONFIG_CONTENT: ${{ secrets.LONGHAUL_KUBECONFIG }} + run: | + set -euo pipefail + install -m 700 -d "$RUNNER_TEMP/.kube" + KCFG="$RUNNER_TEMP/.kube/config" + # Secret is raw kubeconfig YAML (not base64). Same format the + # deploy workflow uses; keep these in sync. + printf '%s' "$KUBECONFIG_CONTENT" > "$KCFG" + chmod 600 "$KCFG" + echo "KUBECONFIG=$KCFG" >> "$GITHUB_ENV" + + - name: Check Deployment exists and is healthy + id: deploy-check + run: | + set -euo pipefail + # Returns empty string if the Deployment doesn't exist. + READY=$(kubectl get deployment "${{ env.DEPLOYMENT_NAME }}" -n "${{ env.NAMESPACE }}" \ + -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true) + DESIRED=$(kubectl get deployment "${{ env.DEPLOYMENT_NAME }}" -n "${{ env.NAMESPACE }}" \ + -o jsonpath='{.spec.replicas}' 2>/dev/null || true) + # Detect "rollout is failing" via the Progressing condition. + PROGRESSING_REASON=$(kubectl get deployment "${{ env.DEPLOYMENT_NAME }}" -n "${{ env.NAMESPACE }}" \ + -o jsonpath='{.status.conditions[?(@.type=="Progressing")].reason}' 2>/dev/null || true) + + if [ -z "$DESIRED" ]; then + echo "exists=false" >> "$GITHUB_OUTPUT" + else + echo "exists=true" >> "$GITHUB_OUTPUT" + fi + { + echo "ready=${READY:-0}" + echo "desired=${DESIRED:-0}" + echo "progressing_reason=${PROGRESSING_REASON}" + } >> "$GITHUB_OUTPUT" + + - name: Check ConfigMap report + id: report-check + run: | + RESULT=$(kubectl get configmap longhaul-report -n ${{ env.NAMESPACE }} -o jsonpath='{.data.result}' 2>/dev/null || echo "UNKNOWN") + echo "result=$RESULT" >> "$GITHUB_OUTPUT" + + LAST_UPDATED=$(kubectl get configmap longhaul-report -n ${{ env.NAMESPACE }} -o jsonpath='{.data.last-updated}' 2>/dev/null || echo "") + echo "last_updated=$LAST_UPDATED" >> "$GITHUB_OUTPUT" + + REPORT=$(kubectl get configmap longhaul-report -n ${{ env.NAMESPACE }} -o jsonpath='{.data.latest-report}' 2>/dev/null || echo "No report available") + { + echo "report<> "$GITHUB_OUTPUT" + + - name: Evaluate status + run: | + set -euo pipefail + echo "=== Long Haul Test Status ===" + echo "Deployment exists: ${{ steps.deploy-check.outputs.exists }}" + echo "Ready / desired: ${{ steps.deploy-check.outputs.ready }} / ${{ steps.deploy-check.outputs.desired }}" + echo "Progressing reason: ${{ steps.deploy-check.outputs.progressing_reason }}" + echo "Report result: ${{ steps.report-check.outputs.result }}" + echo "Last updated: ${{ steps.report-check.outputs.last_updated }}" + echo "" + echo "=== Latest Report ===" + echo "${{ steps.report-check.outputs.report }}" + echo "" + + # Fail if Deployment doesn't exist — test should always be running. + # If you need to fix/redeploy the long-haul test, disable this workflow first: + # gh workflow disable "Long Haul Monitor" + # Re-enable after redeployment: + # gh workflow enable "Long Haul Monitor" + if [ "${{ steps.deploy-check.outputs.exists }}" != "true" ]; then + echo "::error::No long-haul Deployment found — test is not running. Disable this workflow while fixing." + exit 1 + fi + + # Fail if rollout has stalled (e.g. ImagePullBackOff, CrashLoopBackOff + # past the progress deadline). + if [ "${{ steps.deploy-check.outputs.progressing_reason }}" = "ProgressDeadlineExceeded" ]; then + echo "::error::Long haul Deployment rollout stalled (ProgressDeadlineExceeded)" + exit 1 + fi + + # Fail if no replicas are ready. + READY="${{ steps.deploy-check.outputs.ready }}" + if [ "${READY:-0}" -lt 1 ]; then + echo "::error::Long haul Deployment has 0 ready replicas" + exit 1 + fi + + # Fail if report is stale (>2h old) — indicates the test may be hung + LAST="${{ steps.report-check.outputs.last_updated }}" + if [ -n "$LAST" ] && [ "$LAST" != "UNKNOWN" ]; then + LAST_EPOCH=$(date -d "$LAST" +%s 2>/dev/null || echo "0") + NOW_EPOCH=$(date +%s) + AGE_SECONDS=$((NOW_EPOCH - LAST_EPOCH)) + if [ "$AGE_SECONDS" -gt 7200 ]; then + echo "::error::Long haul report is stale (last updated ${AGE_SECONDS}s ago, threshold 7200s). Test may be hung." + exit 1 + fi + fi + + # Fail if ConfigMap reports failure + if [ "${{ steps.report-check.outputs.result }}" = "FAIL" ]; then + echo "::error::Long haul test reported FAIL in ConfigMap" + exit 1 + fi + + echo "✅ Long haul test is healthy" + + - name: Dump driver logs on failure + # Tail recent driver logs whenever the evaluate step (or any prior + # step) fails, so the run summary contains enough context to + # triage without a kubectl session. + if: failure() + run: | + set +e + POD="$(kubectl -n "${{ env.NAMESPACE }}" get pod \ + -l app.kubernetes.io/name=longhaul-test \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)" + if [ -n "$POD" ]; then + echo "::group::Driver logs ($POD, last 200 lines)" + kubectl -n "${{ env.NAMESPACE }}" logs "$POD" --tail=200 || true + echo "::endgroup::" + echo "::group::Pod describe" + kubectl -n "${{ env.NAMESPACE }}" describe pod "$POD" || true + echo "::endgroup::" + else + echo "No long-haul driver pod found in ${{ env.NAMESPACE }}." + fi + + upgrade-if-needed: + runs-on: ubuntu-22.04 + timeout-minutes: 15 + needs: check-longhaul # Only upgrade if cluster is healthy + steps: + - uses: actions/checkout@v4 + + - name: Set up kubeconfig + env: + KUBECONFIG_CONTENT: ${{ secrets.LONGHAUL_KUBECONFIG }} + run: | + set -euo pipefail + install -m 700 -d "$RUNNER_TEMP/.kube" + KCFG="$RUNNER_TEMP/.kube/config" + printf '%s' "$KUBECONFIG_CONTENT" > "$KCFG" + chmod 600 "$KCFG" + echo "KUBECONFIG=$KCFG" >> "$GITHUB_ENV" + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Check and upgrade operator + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + # Get latest published operator tag from GHCR (strict semver only) + EXPECTED=$(gh api /orgs/documentdb/packages/container/documentdb-operator/versions \ + --jq '[.[].metadata.container.tags[] | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))] | sort_by(. | split(".") | map(tonumber)) | last' 2>/dev/null || echo "") + + if [ -z "$EXPECTED" ]; then + # Fallback to Chart.yaml if GHCR query fails + EXPECTED=$(grep 'appVersion:' operator/documentdb-helm-chart/Chart.yaml | awk '{print $2}' | tr -d '"') + echo "ℹ️ GHCR query failed, using Chart.yaml: $EXPECTED" + fi + + # Running version from cluster. Target the operator Deployment by + # NAME, not by app.kubernetes.io/name label — the chart applies + # the same label to three Deployments (operator, sidecar-injector + # in cnpg-system, wal-replica in cnpg-system); even though the + # ns filter excludes the cnpg-system ones today, naming is safer. + RUNNING=$(kubectl get deployment "${{ env.OPERATOR_DEPLOYMENT }}" \ + -n "${{ env.OPERATOR_NAMESPACE }}" \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null | sed 's/.*://') + + echo "Operator: expected=$EXPECTED running=$RUNNING" + + if [ -z "$RUNNING" ]; then + echo "::warning::Could not determine running operator version" + elif [ "$RUNNING" != "$EXPECTED" ]; then + echo "⬆️ Upgrading operator: $RUNNING → $EXPECTED" + + # Helm does NOT upgrade CRDs automatically. Apply them first so + # the new operator code does not reference fields the cluster's + # CRD schema lacks (e.g., schemaVersion). + echo "📦 Applying CRDs from chart..." + kubectl apply -f operator/documentdb-helm-chart/crds/ + + # The chart's cloudnative-pg dependency is NOT vendored in git + # (charts/ and Chart.lock are gitignored), so resolve + download + # it before upgrading, otherwise `helm upgrade` fails with a + # missing-dependency error on a fresh checkout. + echo "📦 Resolving chart dependencies..." + helm dependency update operator/documentdb-helm-chart + + helm upgrade documentdb-operator operator/documentdb-helm-chart \ + -n "${{ env.OPERATOR_NAMESPACE }}" \ + --set image.tag="$EXPECTED" \ + --wait --timeout 5m + + # Verify rollout completed on the operator Deployment specifically. + kubectl rollout status deployment "${{ env.OPERATOR_DEPLOYMENT }}" \ + -n "${{ env.OPERATOR_NAMESPACE }}" --timeout=3m + ACTUAL=$(kubectl get deployment "${{ env.OPERATOR_DEPLOYMENT }}" \ + -n "${{ env.OPERATOR_NAMESPACE }}" \ + -o jsonpath='{.spec.template.spec.containers[0].image}' | sed 's/.*://') + if [ "$ACTUAL" = "$EXPECTED" ]; then + echo "✅ Operator upgraded and verified: $ACTUAL" + else + echo "::error::Operator upgrade verification failed: expected=$EXPECTED actual=$ACTUAL" + exit 1 + fi + else + echo "✅ Operator is up to date ($RUNNING)" + fi + + - name: Publish desired DocumentDB version + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + # Determine the latest published DocumentDB tag and publish it to a + # ConfigMap. The long-haul test reads this and performs the actual + # upgrade as a load-aware operation (so continuous verifiers can + # detect any data-integrity issues caused by the upgrade). + EXPECTED=$(gh api '/orgs/documentdb/packages/container/documentdb-kubernetes-operator%2Fdocumentdb/versions' \ + --jq '[.[].metadata.container.tags[] | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))] | sort_by(. | split(".") | map(tonumber)) | last' 2>/dev/null || echo "") + + if [ -z "$EXPECTED" ]; then + EXPECTED=$(grep 'documentDbVersion:' operator/documentdb-helm-chart/values.yaml | awk '{print $2}' | tr -d '"') + echo "ℹ️ GHCR query failed, using values.yaml: $EXPECTED" + fi + + if [ -z "$EXPECTED" ]; then + echo "::warning::Could not determine desired DocumentDB version; skipping publish" + exit 0 + fi + + echo "📣 Publishing desired DocumentDB version: $EXPECTED" + # Use server-side apply so we can co-own this ConfigMap with the + # driver (the driver may add keys like last-applied-version or + # upgrade-attempt counters). Client-side apply with a full + # generated body would clobber any field we don't set here. + # --force-conflicts lets longhaul-monitor take ownership of the + # fields it manages (desired-documentdb-version, last-updated) even + # if the ConfigMap was originally created by a different field + # manager (e.g. an earlier `kubectl create`/client-side apply). + kubectl create configmap longhaul-versions \ + -n "${{ env.NAMESPACE }}" \ + --from-literal=desired-documentdb-version="$EXPECTED" \ + --from-literal=last-updated="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --dry-run=client -o yaml \ + | kubectl apply --server-side --force-conflicts --field-manager=longhaul-monitor -f - + + RUNNING=$(kubectl get documentdb documentdb-cluster -n "${{ env.NAMESPACE }}" \ + -o jsonpath='{.status.documentDBImage}' 2>/dev/null | sed 's/.*://') + echo "DocumentDB: desired=$EXPECTED running=$RUNNING (test will upgrade if drifted)" diff --git a/.github/workflows/longhaul-smoke.yml b/.github/workflows/longhaul-smoke.yml new file mode 100644 index 00000000..04a6be61 --- /dev/null +++ b/.github/workflows/longhaul-smoke.yml @@ -0,0 +1,303 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Long-Haul Smoke Gate +# +# Purpose +# A PR gate that stands up a *real* DocumentDB on an ephemeral kind cluster +# and runs the actual long-haul driver against it for a short, bounded +# window. Where lint-longhaul-workflows.yml statically validates the +# YAML/manifests, this workflow proves the driver + deploy manifests actually +# function end-to-end: the image builds, the container starts, it connects to +# DocumentDB over the gateway, writers + verifier run, and the driver reports +# PASS with a clean exit code. +# +# Images under test +# The operator + sidecar are built from THIS PR's source (via the shared +# test-build-and-package.yml reusable workflow) so a code change that breaks +# the operator is caught. The documentdb + gateway (database) images are NOT +# rebuilt: test-build-and-package.yml probes the public registry for the +# pinned documentDbVersion and, when present, runs in "registry mode" and +# simply re-tags the public images into the build artifact. The long-haul +# driver image is the only extra image this workflow builds. +# +# Environment bring-up is delegated to the shared composite action +# .github/actions/setup-test-environment +# (use-external-images: false — load operator/sidecar/documentdb/gateway from +# the build artifacts into kind), exactly like test-e2e.yml. No operator-install +# logic is duplicated here. +# +# Why reuse test/longhaul/deploy/{deployment,rbac}.yaml as-is +# The whole point of the gate is to exercise the SAME manifests the long-haul +# cluster ships, so a regression in the ConfigMap/Deployment (bad env key, +# too-low memory limit, wrong secret ref) fails the PR. We only override a +# handful of ConfigMap knobs (via a JSON merge patch) to make the run bounded +# and deterministic in CI — we never fork the manifest. +# +# Why not a Job +# deployment.yaml is a Deployment (restartPolicy: Always) so a bounded run +# that exits 0 would auto-restart. We handle that by scaling the Deployment +# to 0 the moment the driver's container first terminates, freezing the +# verdict, then asserting on BOTH the container exit code AND the +# longhaul-report ConfigMap's result field. +# +# CI budget +# Scale/upgrade disruption ops are disabled for the smoke run +# (LONGHAUL_MIN_INSTANCES == LONGHAUL_MAX_INSTANCES) so the gate is a fast, +# deterministic data-durability check (writers + verifier) that fits a +# GitHub-hosted runner and finishes in a few minutes. + +name: Long-Haul Smoke Gate + +on: + pull_request: + branches: [main] + paths: + - "test/longhaul/**" + - "test/shared/**" + - ".github/workflows/longhaul-smoke.yml" + - ".github/workflows/test-build-and-package.yml" + - ".github/actions/**" + workflow_dispatch: + inputs: + max_duration: + description: "Bounded driver run length (Go duration, e.g. 3m)" + required: false + default: "3m" + +permissions: + contents: read + actions: read + packages: read + +concurrency: + group: longhaul-smoke-${{ github.ref }} + cancel-in-progress: true + +env: + # Namespaces / names are aligned with the hardcoded values baked into + # test/longhaul/deploy/*.yaml so those manifests apply unmodified. + DB_NS: documentdb-test-ns + DB_NAME: documentdb-cluster + OPERATOR_NS: documentdb-operator + CERT_MANAGER_NS: cert-manager + DB_USERNAME: docdbuser + # Smoke-only credential for an ephemeral, throwaway kind cluster that is + # destroyed at the end of the job. Not a secret; never reused elsewhere. + DB_PASSWORD: SmokeTest123! + GATEWAY_PORT: "10260" + +jobs: + # --------------------------------------------------------------------------- + # Build operator + sidecar from this PR's source; reuse public documentdb / + # gateway images (registry mode) so the database images are NOT rebuilt. + # Produces the platform-images tar + helm chart artifacts the setup action + # loads into kind. + # --------------------------------------------------------------------------- + build: + name: Build Images and Charts + uses: ./.github/workflows/test-build-and-package.yml + with: + version: "0.2.0" + secrets: inherit + + smoke: + name: Long-Haul Smoke + needs: build + if: always() && needs.build.result == 'success' + runs-on: ubuntu-22.04 + timeout-minutes: 40 + env: + IMAGE_TAG: ${{ needs.build.outputs.image_tag }} + EXT_IMAGE_TAG: ${{ needs.build.outputs.ext_image_tag }} + CHART_VERSION: ${{ needs.build.outputs.chart_version || '0.1.0' }} + DRIVER_IMAGE: longhaul-test:smoke + # Must match the cluster name the composite action derives: + # documentdb--- + KIND_CLUSTER: documentdb-longhaul-amd64-smoke + MAX_DURATION: ${{ github.event.inputs.max_duration || '3m' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build long-haul driver image + run: | + # Context MUST be the repo root: the Dockerfile copies ../shared and + # ../../operator/src (see test/longhaul/Dockerfile header). Built now, + # before the cluster exists, and loaded into kind after setup. + docker build \ + -f test/longhaul/Dockerfile \ + -t "${DRIVER_IMAGE}" \ + . + + # Only the artifacts the setup action consumes: the consolidated + # platform-specific image tarball (operator/sidecar/documentdb/gateway, + # loaded into kind) and the amd64 helm chart. + - name: Download platform images artifact + uses: actions/download-artifact@v4 + with: + name: build-platform-images + path: ./artifacts/build-platform-images + + - name: Download helm chart artifact + uses: actions/download-artifact@v4 + with: + name: build-helm-chart-amd64 + path: ./artifacts/build-helm-chart-amd64 + + - name: Set up DocumentDB test environment (kind + operator + cluster) + uses: ./.github/actions/setup-test-environment + with: + test-type: "longhaul" + architecture: "amd64" + runner: "ubuntu-22.04" + test-scenario-name: "smoke" + node-count: "1" + instances-per-node: "1" + cert-manager-namespace: ${{ env.CERT_MANAGER_NS }} + operator-namespace: ${{ env.OPERATOR_NS }} + db-namespace: ${{ env.DB_NS }} + db-cluster-name: ${{ env.DB_NAME }} + db-username: ${{ env.DB_USERNAME }} + db-password: ${{ env.DB_PASSWORD }} + db-port: ${{ env.GATEWAY_PORT }} + image-tag: ${{ env.IMAGE_TAG }} + documentdb-image-tag: ${{ env.EXT_IMAGE_TAG }} + chart-version: ${{ env.CHART_VERSION }} + use-external-images: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} + repository-owner: ${{ github.repository_owner }} + + - name: Load driver image into kind + run: kind load docker-image "${DRIVER_IMAGE}" --name "${KIND_CLUSTER}" + + - name: Create driver connection secret (longhaul-documentdb-credentials) + run: | + # In-cluster DNS for the ClusterIP gateway service the operator creates. + HOST="documentdb-service-${DB_NAME}.${DB_NS}.svc" + # tlsInsecure=true (NOT tlsAllowInvalidCertificates) is the option the + # Go mongo driver v2 honors to skip verification of the self-signed + # gateway cert — this matches test/longhaul/README.md. + URI="mongodb://${DB_USERNAME}:${DB_PASSWORD}@${HOST}:${GATEWAY_PORT}/?directConnection=true&authMechanism=SCRAM-SHA-256&tls=true&tlsInsecure=true" + kubectl create secret generic longhaul-documentdb-credentials \ + -n "${DB_NS}" \ + --from-literal=uri="${URI}" \ + --dry-run=client -o yaml | kubectl apply -f - + + - name: Deploy long-haul driver (real manifests, bounded override) + run: | + # RBAC applies unmodified (namespace matches DB_NS). + kubectl apply -f test/longhaul/deploy/rbac.yaml + + # Apply the real ConfigMap + Deployment, substituting only the + # templated image ref with the locally built image loaded into kind. + # The shipped manifest uses imagePullPolicy: Always (correct for the + # registry-based long-haul deploy of mutable tags), but here the + # image is side-loaded into kind and exists on no registry, so + # override to IfNotPresent to use the local image instead of a pull. + sed -e "s|ghcr.io/__OWNER__/documentdb-kubernetes-operator/longhaul-test:__IMAGE_TAG__|${DRIVER_IMAGE}|g" \ + -e "s|imagePullPolicy: Always|imagePullPolicy: IfNotPresent|g" \ + test/longhaul/deploy/deployment.yaml | kubectl apply -f - + + # Bounded, deterministic smoke override — patch ONLY runtime knobs on + # the shipped ConfigMap; the manifest structure is unchanged. + # - MAX_DURATION: finite run + # - RESET_DATA: fresh collection each CI run + # - MIN==MAX instances: disable disruptive scale ops (fast + stable) + # - short cadences so the verifier gets several cycles in the window + PATCH=$(jq -nc \ + --arg dur "${MAX_DURATION}" \ + '{data: { + LONGHAUL_MAX_DURATION: $dur, + LONGHAUL_RESET_DATA: "true", + LONGHAUL_NUM_WRITERS: "2", + LONGHAUL_OP_COOLDOWN: "30s", + LONGHAUL_STEADY_STATE_WAIT: "10s", + LONGHAUL_RECOVERY_TIMEOUT: "2m", + LONGHAUL_REPORT_INTERVAL: "30s", + LONGHAUL_MIN_INSTANCES: "1", + LONGHAUL_MAX_INSTANCES: "1" + }}') + kubectl patch configmap longhaul-test-config -n "${DB_NS}" --type merge -p "${PATCH}" + + # Restart so the pod picks up the patched ConfigMap (envFrom is only + # read at container start). + kubectl rollout restart deployment/longhaul-test -n "${DB_NS}" + kubectl rollout status deployment/longhaul-test -n "${DB_NS}" --timeout=120s + + - name: Wait for driver to complete one bounded run + id: wait + run: | + set -euo pipefail + deadline=$(( $(date +%s) + 900 )) # 15 min hard cap + exit_code="" + while [[ $(date +%s) -lt ${deadline} ]]; do + pod=$(kubectl get pods -n "${DB_NS}" \ + -l app.kubernetes.io/name=longhaul-test \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") + if [[ -z "${pod}" ]]; then + sleep 5; continue + fi + + # Look for a terminated driver container (current or previous + # restart) — that is the terminal verdict of a bounded run. + term_cur=$(kubectl get pod "${pod}" -n "${DB_NS}" \ + -o jsonpath='{.status.containerStatuses[0].state.terminated.exitCode}' 2>/dev/null || echo "") + term_prev=$(kubectl get pod "${pod}" -n "${DB_NS}" \ + -o jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}' 2>/dev/null || echo "") + + if [[ -n "${term_cur}" ]]; then + exit_code="${term_cur}" + elif [[ -n "${term_prev}" ]]; then + exit_code="${term_prev}" + fi + + if [[ -n "${exit_code}" ]]; then + echo "Driver container terminated with exit code ${exit_code}." + # Freeze state before the Deployment restarts the pod again. + kubectl scale deployment/longhaul-test -n "${DB_NS}" --replicas=0 || true + break + fi + echo " driver still running..." + sleep 10 + done + + if [[ -z "${exit_code}" ]]; then + echo "::error::Driver did not terminate within the smoke window." + exit 1 + fi + echo "exit_code=${exit_code}" >> "${GITHUB_OUTPUT}" + + - name: Assert PASS verdict + run: | + exit_code="${{ steps.wait.outputs.exit_code }}" + result=$(kubectl get configmap longhaul-report -n "${DB_NS}" \ + -o jsonpath='{.data.result}' 2>/dev/null || echo "MISSING") + echo "Driver exit code : ${exit_code}" + echo "Report result : ${result}" + + if [[ "${exit_code}" != "0" ]]; then + echo "::error::Driver exited non-zero (${exit_code})." + exit 1 + fi + if [[ "${result}" != "PASS" ]]; then + echo "::error::longhaul-report result is '${result}', expected PASS." + exit 1 + fi + echo "✅ Long-haul smoke gate passed (exit 0, report PASS)." + + - name: Diagnostics on failure + if: failure() + run: | + echo "===== DocumentDB pods =====" + kubectl get pods -n "${DB_NS}" -o wide || true + echo "===== Driver logs (current) =====" + kubectl logs -n "${DB_NS}" -l app.kubernetes.io/name=longhaul-test --tail=200 || true + echo "===== Driver logs (previous) =====" + kubectl logs -n "${DB_NS}" -l app.kubernetes.io/name=longhaul-test --previous --tail=200 || true + echo "===== longhaul-report ConfigMap =====" + kubectl get configmap longhaul-report -n "${DB_NS}" -o yaml || true + echo "===== DocumentDB describe =====" + kubectl describe documentdb "${DB_NAME}" -n "${DB_NS}" || true + echo "===== Operator logs =====" + kubectl logs -n "${OPERATOR_NS}" deployment/documentdb-operator --tail=100 || true diff --git a/test/longhaul/deploy/deployment.yaml b/test/longhaul/deploy/deployment.yaml new file mode 100644 index 00000000..49d5c09f --- /dev/null +++ b/test/longhaul/deploy/deployment.yaml @@ -0,0 +1,118 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Long Haul Test - Kubernetes Deployment +# +# Why a Deployment (not a Job)? +# A Job's pod spec is immutable, so rolling a new image requires +# delete-and-recreate. Using Deployment(replicas=1) lets the deploy +# workflow do `kubectl set image deployment/longhaul-test ...` for +# one-line image rolls. The driver itself is restart-safe: it resumes +# each writer's sequence from the DocumentDB workload collection and +# reads/writes progress to the longhaul-report ConfigMap (no PVC). +# +# Strategy: Recreate (single-replica stateful driver — never run two +# driver pods concurrently against the same DocumentDB cluster / +# workload collection). +# +# Failure semantics: on critical failure (data loss, policy violation) +# the driver exits non-zero. The Deployment auto-restarts the pod, which +# gives MTBF data; the alert workflow polls the report ConfigMap and +# pages on incident-count thresholds. +# +# Image refs are templated; the longhaul-deploy workflow substitutes: +# __OWNER__ -> lowercased ${{ github.repository_owner }} +# __IMAGE_TAG__ -> input.image_tag (e.g. sha-5c30d7b or main) +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhaul-test-config + namespace: documentdb-test-ns + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +data: + # Test duration. "0s" = run-until-failure (the design default for the + # long-haul cluster). Override to a finite value (e.g. "30m", "3h") only + # for short ad-hoc / smoke runs. + LONGHAUL_MAX_DURATION: "0s" + # Target DocumentDB cluster. + LONGHAUL_NAMESPACE: "documentdb-test-ns" + LONGHAUL_CLUSTER_NAME: "documentdb-cluster" + # Writer/verifier counts. + LONGHAUL_NUM_WRITERS: "5" + # Operation scheduling. + LONGHAUL_OP_COOLDOWN: "10m" + LONGHAUL_RECOVERY_TIMEOUT: "5m" + # How long the cluster must be observed healthy before the next + # operation fires. Matches the in-code default (60s) and the design doc + # (Section "Steady-state gate"). Keep explicit so operators can lower + # it for chaos runs or raise it during incident triage without a + # rebuild. + LONGHAUL_STEADY_STATE_WAIT: "60s" + # Checkpoint report cadence. Matches the monitor workflow cron (hourly) + # — bump both together if changed. + LONGHAUL_REPORT_INTERVAL: "1h" + # Whether to drop the workload collection on startup. MUST stay "false" + # on the long-haul cluster — restarts of the driver pod (rolling image + # updates, node drains) need to resume durability state from the existing + # DocumentDB workload collection and longhaul-report ConfigMap checkpoints. + # Override to "true" only for local/dev iterations. + LONGHAUL_RESET_DATA: "false" + # Replica range for scale operations. + # Bounds for spec.instancesPerNode (CRD allows 1-3). Default Min=1, Max=3 + # exercises full scale-up/down cycle. Set Min=Max to disable scale ops. + LONGHAUL_MIN_INSTANCES: "1" + LONGHAUL_MAX_INSTANCES: "3" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: longhaul-test + namespace: documentdb-test-ns + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing + template: + metadata: + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing + spec: + serviceAccountName: longhaul-test + restartPolicy: Always + containers: + - name: driver + image: ghcr.io/__OWNER__/documentdb-kubernetes-operator/longhaul-test:__IMAGE_TAG__ + imagePullPolicy: Always + envFrom: + - configMapRef: + name: longhaul-test-config + env: + - name: LONGHAUL_DOCUMENTDB_URI + valueFrom: + secretKeyRef: + name: longhaul-documentdb-credentials + key: uri + # Soft heap ceiling below the cgroup limit so Go's GC turns + # aggressive before the hard limit, preventing OOMKills under a + # transient allocation spike. The verify path streams its cursor + # (O(1) memory), so steady-state usage is a few MB; this is a + # belt-and-suspenders guard, not a working-set requirement. + - name: GOMEMLIMIT + value: "200MiB" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi diff --git a/test/longhaul/deploy/rbac.yaml b/test/longhaul/deploy/rbac.yaml new file mode 100644 index 00000000..f5ed5abc --- /dev/null +++ b/test/longhaul/deploy/rbac.yaml @@ -0,0 +1,81 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# ServiceAccount for the long-haul test driver Deployment. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhaul-test + namespace: documentdb-test-ns + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +--- +# Role granting permissions within the target namespace. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: longhaul-test + namespace: documentdb-test-ns + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +rules: + # Read pod status for health monitoring. + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + # Read and patch DocumentDB CRs for health check and scale operations. + - apiGroups: ["documentdb.io"] + resources: ["dbs"] + verbs: ["get", "list", "patch"] + # Create/update ConfigMaps for periodic report persistence. + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: longhaul-test + namespace: documentdb-test-ns + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: longhaul-test +subjects: + - kind: ServiceAccount + name: longhaul-test + namespace: documentdb-test-ns +--- +# ClusterRole for metrics-server access (metrics API is cluster-scoped). +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhaul-test-metrics + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +rules: + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhaul-test-metrics + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhaul-test-metrics +subjects: + - kind: ServiceAccount + name: longhaul-test + namespace: documentdb-test-ns diff --git a/test/longhaul/workload/verifier.go b/test/longhaul/workload/verifier.go index 34787673..10addf7d 100644 --- a/test/longhaul/workload/verifier.go +++ b/test/longhaul/workload/verifier.go @@ -115,55 +115,80 @@ type auditResult struct { findings []finding } -// auditDocs is the pure decision core of verifyWriter. Given the docs the -// verifier read (sorted by seq ascending) plus the writer's expected starting -// seq and current tip, it returns the new expected seq, the gap/checksum/tail -// counters, and a list of findings for the caller to log. +// auditor is the incremental decision core of verifyWriter. Feed it the docs +// for a writer in ascending seq order via step(); it accumulates gap/checksum +// counters and findings while holding only the running expectedSeq — never the +// docs themselves. finish(maxSeq) applies trailing tail-loss detection and +// returns the aggregate. Keeping state O(1) (not O(docs)) is what lets the +// production scan stream a cursor instead of buffering a whole cycle's window, +// which after a disruption pause can be tens of thousands of docs. // // Invariants checked: // - For each doc, if doc.Seq > expectedSeq, the slots in [expectedSeq, doc.Seq) // are missing (internal gap). // - For each doc, checksum is recomputed and compared. -// - After processing all docs, if expectedSeq <= maxSeq the trailing slots +// - After all docs, if expectedSeq <= maxSeq the trailing slots // [expectedSeq, maxSeq] are missing (tail loss). -// - On exit, newExpectedSeq is always maxSeq+1 when maxSeq >= initial +// - On finish, newExpectedSeq is always maxSeq+1 when maxSeq >= initial // expectedSeq, so the next cycle accounts for everything past maxSeq. -func auditDocs(writerID string, docs []WriteDocument, expectedSeq, maxSeq int64) auditResult { - var r auditResult - for _, doc := range docs { - if doc.Seq > expectedSeq { - gaps := doc.Seq - expectedSeq - r.internalGaps += gaps - r.findings = append(r.findings, finding{ - kind: findingGap, writerID: writerID, - seq: expectedSeq, endSeq: doc.Seq, count: gaps, - }) - } - expectedSeq = doc.Seq + 1 - - want := computeChecksum(doc.WriterID, doc.Seq, doc.Payload) - if doc.Checksum != want { - r.checksumErrors++ - r.findings = append(r.findings, finding{ - kind: findingChecksum, writerID: writerID, - seq: doc.Seq, count: 1, - stored: doc.Checksum, computed: want, - }) - } +type auditor struct { + writerID string + expectedSeq int64 + r auditResult +} + +func newAuditor(writerID string, expectedSeq int64) *auditor { + return &auditor{writerID: writerID, expectedSeq: expectedSeq} +} + +// step folds a single doc into the running audit state. +func (a *auditor) step(doc WriteDocument) { + if doc.Seq > a.expectedSeq { + gaps := doc.Seq - a.expectedSeq + a.r.internalGaps += gaps + a.r.findings = append(a.r.findings, finding{ + kind: findingGap, writerID: a.writerID, + seq: a.expectedSeq, endSeq: doc.Seq, count: gaps, + }) } + a.expectedSeq = doc.Seq + 1 - if expectedSeq <= maxSeq { - tail := maxSeq - expectedSeq + 1 - r.tailLoss = tail - r.findings = append(r.findings, finding{ - kind: findingTail, writerID: writerID, - seq: expectedSeq, endSeq: maxSeq, count: tail, + want := computeChecksum(doc.WriterID, doc.Seq, doc.Payload) + if doc.Checksum != want { + a.r.checksumErrors++ + a.r.findings = append(a.r.findings, finding{ + kind: findingChecksum, writerID: a.writerID, + seq: doc.Seq, count: 1, + stored: doc.Checksum, computed: want, }) - expectedSeq = maxSeq + 1 } +} - r.newExpectedSeq = expectedSeq - return r +// finish applies tail-loss detection for [expectedSeq, maxSeq] and returns the +// aggregate result. +func (a *auditor) finish(maxSeq int64) auditResult { + if a.expectedSeq <= maxSeq { + tail := maxSeq - a.expectedSeq + 1 + a.r.tailLoss = tail + a.r.findings = append(a.r.findings, finding{ + kind: findingTail, writerID: a.writerID, + seq: a.expectedSeq, endSeq: maxSeq, count: tail, + }) + a.expectedSeq = maxSeq + 1 + } + a.r.newExpectedSeq = a.expectedSeq + return a.r +} + +// auditDocs is the pure, table-testable entry point: it replays a slice of docs +// through an auditor. Production code uses the streaming path in scanWriter, +// which feeds cursor-decoded docs to an auditor one at a time. +func auditDocs(writerID string, docs []WriteDocument, expectedSeq, maxSeq int64) auditResult { + a := newAuditor(writerID, expectedSeq) + for _, doc := range docs { + a.step(doc) + } + return a.finish(maxSeq) } func (v *Verifier) verifyWriter(ctx context.Context, w *Writer) { @@ -185,13 +210,12 @@ func (v *Verifier) verifyWriter(ctx context.Context, w *Writer) { return } - docs, err := v.fetchDocs(ctx, writerID, expectedSeq, maxSeq) + r, err := v.scanWriter(ctx, writerID, expectedSeq, maxSeq) if err != nil { v.journal.Warn("verifier", fmt.Sprintf("query failed for writer %s: %v", writerID, err)) return } - r := auditDocs(writerID, docs, expectedSeq, maxSeq) v.metrics.VerifyGapsDetected.Add(r.internalGaps + r.tailLoss) v.metrics.ChecksumErrors.Add(r.checksumErrors) for _, f := range r.findings { @@ -201,10 +225,12 @@ func (v *Verifier) verifyWriter(ctx context.Context, w *Writer) { v.nextSeq[writerID] = r.newExpectedSeq } -// fetchDocs reads all docs for writerID with seq in [expectedSeq, maxSeq], -// sorted by seq ascending. Decode errors are logged but skipped (the rest of -// the scan continues; a skipped doc looks like a gap to auditDocs). -func (v *Verifier) fetchDocs(ctx context.Context, writerID string, expectedSeq, maxSeq int64) ([]WriteDocument, error) { +// scanWriter streams all docs for writerID with seq in [expectedSeq, maxSeq] +// (sorted by seq ascending) through an auditor, holding only one decoded doc at +// a time. This keeps verify memory O(1) per cycle regardless of how many docs +// accumulated since the last tick — which can be tens of thousands after a +// disruption pause — instead of materialising the whole window in a slice. +func (v *Verifier) scanWriter(ctx context.Context, writerID string, expectedSeq, maxSeq int64) (auditResult, error) { opts := options.Find().SetSort(bson.D{{Key: "seq", Value: 1}}) filter := bson.D{ {Key: "writer_id", Value: writerID}, @@ -215,20 +241,27 @@ func (v *Verifier) fetchDocs(ctx context.Context, writerID string, expectedSeq, } cursor, err := v.collection.Find(ctx, filter, opts) if err != nil { - return nil, err + return auditResult{}, err } defer cursor.Close(ctx) - var out []WriteDocument + a := newAuditor(writerID, expectedSeq) for cursor.Next(ctx) { var doc WriteDocument if err := cursor.Decode(&doc); err != nil { + // Decode errors are logged but skipped (the rest of the scan + // continues; a skipped doc looks like a gap to the auditor). v.journal.Warn("verifier", fmt.Sprintf("decode error for writer %s: %v", writerID, err)) continue } - out = append(out, doc) + a.step(doc) + } + // Surface iteration errors rather than silently treating a truncated read as + // tail loss, which would be a false positive. + if err := cursor.Err(); err != nil { + return auditResult{}, err } - return out, nil + return a.finish(maxSeq), nil } func (v *Verifier) logFinding(f finding) {