ci(longhaul): add build/deploy/monitor workflows + lint & functional smoke gates

.github/workflows/lint-longhaul-workflows.yml

@@ -0,0 +1,177 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Pre-merge gating for the long-haul CI/CD bundle.
+#
+# The long-haul {build, deploy, monitor} workflows only run post-merge
+# (push to main / scheduled / workflow_run), so a typo in any of them is
+# normally not caught until they fire against the live AKS cluster. This
+# workflow runs on PRs that touch any of those files (plus the long-haul
+# driver sources, deploy manifests, and Dockerfile) and exercises them
+# statically:
+#
+#   - actionlint    : GitHub Actions syntax, expression bugs, missing
+#                     permissions, embedded shellcheck on `run:` blocks.
+#   - yamllint      : strict YAML formatting.
+#   - kubeconform   : Kubernetes manifest schema validation against the
+#                     v1.30 schema set (matches our compatibility matrix).
+#   - helm template : ensure the operator chart still renders so the
+#                     auto-upgrade job in monitor.yaml will succeed.
+#   - go vet/test   : the driver has real unit tests (config, journal,
+#                     monitor, operations, report); compiling in the image
+#                     build is not enough, so run them here.
+#   - docker build  : build the long-haul image with push:false to catch
+#                     context / go.mod-replace problems before they break
+#                     the scheduled image build.
+name: Lint Long-Haul Workflows
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/longhaul-*.y*ml"
+      - ".github/workflows/lint-longhaul-workflows.yml"
+      - "test/longhaul/**"
+      - "operator/documentdb-helm-chart/**"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: lint-longhaul-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: actionlint
+        # reviewdog/action-actionlint scans .github/workflows/ by default
+        # and embeds shellcheck so `run:` blocks are linted too. We rely
+        # on the default discovery — there is no point linting only the
+        # long-haul subset since neighbouring workflows share a runner
+        # image and any syntax problem would block CI anyway.
+        uses: reviewdog/action-actionlint@v1
+        with:
+          reporter: github-pr-check
+          fail_level: error
+
+      - name: yamllint
+        run: |
+          set -euo pipefail
+          pip install --quiet yamllint
+          yamllint -d '{extends: default, rules: {line-length: {max: 200}, comments-indentation: disable, document-start: disable, truthy: {check-keys: false}}}' \
+            .github/workflows/longhaul-image-build.yml \
+            .github/workflows/longhaul-deploy.yml \
+            .github/workflows/longhaul-monitor.yaml \
+            test/longhaul/deploy/
+
+      - name: Cache kubeconform schemas
+        # Persist downloaded JSON schemas across runs so we don't re-hit
+        # raw.githubusercontent.com (which rate-limits with HTTP 429) on
+        # every run. Once populated, steady-state lint needs zero schema
+        # downloads; the retry loop below only covers a cold cache.
+        uses: actions/cache@v4
+        with:
+          path: /tmp/kubeconform-cache
+          key: kubeconform-schemas-1.30.0-v1
+
+      - name: kubeconform manifests
+        run: |
+          set -euo pipefail
+          curl -sSL https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz \
+            | tar -xz -C /usr/local/bin kubeconform
+          mkdir -p /tmp/kubeconform-cache
+          # Skip CRD-typed resources (DocumentDB) since their schemas live
+          # in operator/documentdb-helm-chart/crds — pass via -schema-location.
+          #
+          # Schemas are fetched from raw.githubusercontent.com, which
+          # intermittently rate-limits shared CI runner IPs (HTTP 403 /
+          # "giving up after 3 attempts"). Cache downloaded schemas and retry
+          # with backoff so a transient rate limit doesn't fail the gate; the
+          # cache means each retry only re-fetches the schemas still missing.
+          run_kubeconform() {
+            kubeconform -strict -summary \
+              -cache /tmp/kubeconform-cache \
+              -kubernetes-version 1.30.0 \
+              -schema-location default \
+              -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \
+              -skip 'DocumentDB' \
+              test/longhaul/deploy/
+          }
+          n=0
+          until run_kubeconform; do
+            n=$((n + 1))
+            if [ "$n" -ge 4 ]; then
+              echo "::error::kubeconform failed after $n attempts"
+              exit 1
+            fi
+            echo "kubeconform attempt $n failed (likely schema-download rate limit); retrying in $((n * 30))s..."
+            sleep "$((n * 30))"
+          done
+
+      - name: helm template smoke
+        run: |
+          set -euo pipefail
+          curl -sSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+          # Fetch chart dependencies (e.g. cloudnative-pg) so the chart can
+          # be templated — mirrors the release workflows.
+          helm dependency update operator/documentdb-helm-chart
+          # If the chart fails to template, the monitor's auto-upgrade
+          # job will fail at runtime against the live cluster.
+          helm template documentdb-operator operator/documentdb-helm-chart \
+            --namespace documentdb-operator \
+            --api-versions cert-manager.io/v1/Certificate \
+            > /tmp/rendered.yaml
+          # Sanity: rendered output must include the operator Deployment we
+          # target by name in monitor.yaml.
+          grep -q 'name: documentdb-operator$' /tmp/rendered.yaml \
+            || (echo "::error::Rendered chart missing 'documentdb-operator' Deployment — monitor.yaml's name-based selector will fail" && exit 1)
+
+  go-test:
+    # The driver has real unit tests (config, journal, monitor, operations,
+    # report). The image build only compiles them — run them here so a logic
+    # regression in the reporter/thresholds is caught at PR time.
+    runs-on: ubuntu-22.04
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: test/longhaul
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: test/longhaul/go.mod
+          cache-dependency-path: test/longhaul/go.sum
+
+      - name: go vet
+        run: go vet ./...
+
+      - name: go test
+        run: go test ./... -count=1
+
+  image-build-smoke:
+    # Mirrors the production longhaul-image-build.yml exactly (context, file,
+    # platforms) but with push:false. Catches context/replace-directive bugs
+    # at PR time rather than on the main-branch scheduled run.
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build long-haul image (no push)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: test/longhaul/Dockerfile
+          push: false
+          platforms: linux/amd64
+          cache-from: type=gha,scope=longhaul-pr
+          cache-to: type=gha,scope=longhaul-pr,mode=max

.github/workflows/longhaul-deploy.yml

@@ -0,0 +1,184 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+name: LONGHAUL - Deploy Test Driver to AKS
+
+# Rolls a longhaul-test image onto the long-haul AKS cluster.
+#
+# Auth: a kubeconfig for the long-haul AKS cluster, provisioned out of band
+# and stored as the LONGHAUL_KUBECONFIG repo secret. The kubeconfig MUST be
+# privileged enough to:
+#   * apply Deployments + ConfigMaps in documentdb-test-ns (this workflow)
+#   * roll the operator via Helm in documentdb-operator + apply CRDs
+#     cluster-wide (longhaul-monitor.yaml's upgrade job)
+# In practice that means cluster-admin or a sufficiently broad ClusterRole.
+#
+# Note: test/longhaul/deploy/rbac.yaml is the in-cluster RBAC granted to
+# the long-haul *driver pod* (its ServiceAccount), not to this workflow.
+# The two are independent: the pod's SA only needs namespace-scoped pod /
+# documentdb / configmap access; this workflow needs more.
+#
+# Triggers:
+#   * workflow_run — auto-deploys after a successful build on main, pinning
+#     to the immutable :sha-<short> tag built by that run.
+#   * workflow_dispatch — manual roll/rollback to a specific tag (e.g. an
+#     earlier :sha-<short>, or :main for a quick smoke).
+
+on:
+  workflow_run:
+    workflows: ["LONGHAUL - Build Test Driver Image"]
+    types: [completed]
+    branches: [main]
+
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: 'Image tag to deploy (e.g. sha-5c30d7b or main).'
+        required: true
+        default: 'main'
+
+permissions:
+  contents: read
+
+env:
+  NAMESPACE: documentdb-test-ns
+  DEPLOYMENT: longhaul-test
+  CONTAINER: driver
+
+concurrency:
+  group: longhaul-deploy
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    name: Deploy to AKS
+    runs-on: ubuntu-22.04
+    timeout-minutes: 15
+    # When triggered by workflow_run, only proceed if the upstream build succeeded.
+    # workflow_dispatch always proceeds.
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Write kubeconfig
+        # Pass the secret via env, not ${{ }} expression. Expression-style
+        # interpolation pastes the secret value directly into the shell
+        # script before execution, which can lead to subtle shell-injection
+        # if the secret contains characters like $ or backticks. Using env:
+        # keeps the value as a literal string in the runner's environment.
+        env:
+          KUBECONFIG_CONTENT: ${{ secrets.LONGHAUL_KUBECONFIG }}
+        run: |
+          set -euo pipefail
+          install -m 700 -d "$RUNNER_TEMP/.kube"
+          KCFG="$RUNNER_TEMP/.kube/config"
+          printf '%s' "$KUBECONFIG_CONTENT" > "$KCFG"
+          chmod 600 "$KCFG"
+          echo "KUBECONFIG=$KCFG" >> "$GITHUB_ENV"
+
+      - name: Verify cluster access
+        run: |
+          set -euo pipefail
+          # No `|| true`: this step must fail fast if the cluster is
+          # unreachable (bad/expired kubeconfig), instead of silently
+          # continuing into the apply/rollout steps.
+          kubectl version --client=false --output=yaml
+          kubectl -n "$NAMESPACE" get all
+
+      - name: Compute image ref
+        id: img
+        env:
+          # workflow_dispatch path: maintainer-supplied tag.
+          # workflow_run path: derive immutable :sha-<short> from the upstream
+          # build's head SHA, so we deploy exactly what was just built.
+          DISPATCH_TAG: ${{ github.event.inputs.image_tag }}
+          UPSTREAM_SHA: ${{ github.event.workflow_run.head_sha }}
+        run: |
+          set -euo pipefail
+          OWNER_LC="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')"
+          REPO_NAME="$(basename "${{ github.repository }}")"
+
+          if [ "${{ github.event_name }}" = "workflow_run" ]; then
+            SHORT_SHA="$(echo "${UPSTREAM_SHA}" | cut -c1-7)"
+            TAG="sha-${SHORT_SHA}"
+            MUTABLE=false
+          else
+            TAG="${DISPATCH_TAG}"
+            # Treat anything that isn't an immutable sha-* tag as mutable
+            # (e.g. :main) so we force a rollout restart to re-pull.
+            case "$TAG" in
+              sha-*) MUTABLE=false ;;
+              *)     MUTABLE=true  ;;
+            esac
+          fi
+
+          IMAGE="ghcr.io/${OWNER_LC}/${REPO_NAME}/longhaul-test:${TAG}"
+          {
+            echo "tag=${TAG}"
+            echo "image=${IMAGE}"
+            echo "mutable=${MUTABLE}"
+          } >> "$GITHUB_OUTPUT"
+          echo "Resolved image: ${IMAGE} (mutable=${MUTABLE})"
+
+      - name: Apply Deployment manifest
+        # The Deployment manifest is the only thing the deploy workflow
+        # owns. Bootstrap resources (namespace, DocumentDB CR, credentials
+        # secret, driver ServiceAccount/Role/ClusterRole) live in
+        # setup.yaml + rbac.yaml and are applied once by a cluster admin.
+        #
+        # `kubectl apply` with the substituted image tag is sufficient to
+        # trigger a rollout when the tag changes — no extra `set image`
+        # call needed.
+        run: |
+          set -euo pipefail
+          OWNER_LC="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')"
+          TAG='${{ steps.img.outputs.tag }}'
+
+          sed -e "s|__OWNER__|${OWNER_LC}|g" \
+              -e "s|__IMAGE_TAG__|${TAG}|g" \
+              test/longhaul/deploy/deployment.yaml | kubectl apply -f -
+
+      - name: Force re-pull for mutable tags
+        # `kubectl apply` only triggers a rollout when the pod-spec changes.
+        # For mutable tags like :main, the spec stays identical even when
+        # the underlying image moves — so we need an explicit restart to
+        # re-pull. Immutable :sha-<short> tags don't need this.
+        if: steps.img.outputs.mutable == 'true'
+        run: |
+          set -euo pipefail
+          kubectl -n "$NAMESPACE" rollout restart "deployment/${DEPLOYMENT}"
+
+      - name: Wait for rollout
+        run: |
+          set -euo pipefail
+          kubectl -n "$NAMESPACE" rollout status \
+            "deployment/${DEPLOYMENT}" --timeout=5m
+
+      - name: Pod status
+        if: always()
+        run: |
+          set -euo pipefail
+          kubectl -n "$NAMESPACE" get deployment "${DEPLOYMENT}" -o wide || true
+          kubectl -n "$NAMESPACE" get pods -l app.kubernetes.io/name=longhaul-test -o wide || true
+          POD="$(kubectl -n "$NAMESPACE" get pod -l app.kubernetes.io/name=longhaul-test \
+                  -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
+          if [ -n "$POD" ]; then
+            echo "::group::Recent driver logs ($POD)"
+            kubectl -n "$NAMESPACE" logs "$POD" --tail=80 || true
+            echo "::endgroup::"
+          fi
+
+      - name: Summary
+        if: always()
+        run: |
+          {
+            echo "## Long-haul deploy"
+            echo ""
+            echo "| Field | Value |"
+            echo "|-------|-------|"
+            echo "| Trigger | \`${{ github.event_name }}\` |"
+            echo "| Image | \`${{ steps.img.outputs.image }}\` |"
+            echo "| Namespace | \`$NAMESPACE\` |"
+            echo "| Deployment | \`$DEPLOYMENT\` |"
+          } >> "$GITHUB_STEP_SUMMARY"