[SEA-NodeJS] Pin the kernel by SHA (KERNEL_REV) + kernel-e2e CI #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Kernel E2E Tests | |
| # Runs the SEA backend e2e suite (tests/e2e/sea/**) against a real | |
| # Databricks warehouse with a freshly-built napi-rs kernel binding. | |
| # | |
| # The kernel is a private repo with no published binary artifact. We pin | |
| # a kernel SHA in the `KERNEL_REV` file at the repo root, check the kernel | |
| # out via a GitHub App token, and run `npm run build:native` to compile | |
| # the napi binding into native/sea/ in the same checkout the tests run | |
| # against. Bumping `KERNEL_REV` is the ONLY way to pick up a new kernel | |
| # version — this keeps the driver <-> kernel pair bisectable, so a driver | |
| # change and the kernel revision it depends on always land together. | |
| # | |
| # Why this exists: the committed native/sea/index.d.ts + index.js are the | |
| # TypeScript declarations and the napi-rs platform router; the actual | |
| # `.node` binary is gitignored (large, per-platform) and is NOT in the | |
| # repo. The standard `main.yml` e2e job has no binary, so its SEA suite | |
| # skips (it gates on DATABRICKS_PECOTESTING_* secrets it doesn't set). | |
| # This workflow is what actually exercises the SEA path end-to-end against | |
| # a known kernel revision. | |
| # | |
| # Gate semantics: | |
| # - Plain PR events post a synthetic-success check so the required | |
| # "Kernel E2E" check doesn't block PRs that don't touch the SEA path. | |
| # Real tests run in the merge queue. | |
| # - `kernel-e2e` label triggers a preview run on the PR; the label is | |
| # auto-removed on `synchronize` for the same security reason. | |
| # - merge_group fires the real gate — runs when SEA-relevant files | |
| # changed, auto-passes otherwise. | |
| # | |
| # Required external setup (one-time, by a repo admin): | |
| # 1. `kernel-e2e` label exists in this repo. | |
| # 2. `INTEGRATION_TEST_APP_ID` / `INTEGRATION_TEST_PRIVATE_KEY` secrets | |
| # exist and the GitHub App's repo allowlist includes | |
| # `databricks/databricks-sql-kernel`. | |
| # 3. `KERNEL_REV` at the repo root contains a 40-char kernel commit SHA. | |
| # 4. `azure-prod` environment exposes DATABRICKS_HOST / | |
| # TEST_PECO_WAREHOUSE_HTTP_PATH / DATABRICKS_TOKEN. | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, labeled] | |
| merge_group: | |
| permissions: | |
| contents: read | |
| id-token: write | |
| concurrency: | |
| group: kernel-e2e-${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| # ─────────────────────────────────────────────────────────────── | |
| # Security: auto-remove `kernel-e2e` label on new commits so a | |
| # labelled preview run can't be re-triggered with unreviewed code. | |
| # ─────────────────────────────────────────────────────────────── | |
| strip-label: | |
| if: github.event_name == 'pull_request' && github.event.action == 'synchronize' | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Remove kernel-e2e label | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 | |
| with: | |
| github-token: ${{ github.token }} | |
| script: | | |
| try { | |
| await github.rest.issues.removeLabel({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.payload.pull_request.number, | |
| name: 'kernel-e2e', | |
| }); | |
| } catch (error) { | |
| if (error.status !== 404) throw error; | |
| } | |
| # ─────────────────────────────────────────────────────────────── | |
| # Synthetic success on every non-label PR event so the required | |
| # "Kernel E2E" check doesn't permablock PRs that don't touch SEA | |
| # code. Real run happens in the merge queue (or via explicit label). | |
| # ─────────────────────────────────────────────────────────────── | |
| skip-kernel-e2e-pr: | |
| if: github.event_name == 'pull_request' && github.event.action != 'labeled' | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| permissions: | |
| checks: write | |
| steps: | |
| - name: Post synthetic-success check | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 | |
| with: | |
| github-token: ${{ github.token }} | |
| script: | | |
| await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'Kernel E2E', | |
| head_sha: context.payload.pull_request.head.sha, | |
| status: 'completed', | |
| conclusion: 'success', | |
| completed_at: new Date().toISOString(), | |
| output: { | |
| title: 'Skipped on PR — runs in merge queue', | |
| summary: 'Kernel E2E is skipped on PRs and runs as a required gate in the merge queue. Add the `kernel-e2e` label to preview on this PR.' | |
| } | |
| }); | |
| # ─────────────────────────────────────────────────────────────── | |
| # Detect whether SEA-relevant files changed. Used by both the | |
| # labelled-PR path and the merge-queue path to decide between | |
| # "really run the suite" and "auto-pass the check". | |
| # ─────────────────────────────────────────────────────────────── | |
| detect-changes: | |
| if: | | |
| github.event_name == 'merge_group' || | |
| (github.event_name == 'pull_request' && | |
| github.event.action == 'labeled' && | |
| contains(github.event.pull_request.labels.*.name, 'kernel-e2e')) | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| outputs: | |
| run_tests: ${{ steps.changed.outputs.run_tests }} | |
| head_sha: ${{ steps.refs.outputs.head_sha }} | |
| steps: | |
| - name: Resolve head SHA | |
| id: refs | |
| env: | |
| MERGE_QUEUE_REF: ${{ github.event.merge_group.head_ref }} | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 | |
| with: | |
| script: | | |
| if (context.eventName === 'pull_request') { | |
| core.setOutput('head_sha', context.payload.pull_request.head.sha); | |
| return; | |
| } | |
| core.setOutput('head_sha', context.payload.merge_group.head_sha); | |
| - name: Check out repo at head SHA | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| ref: ${{ steps.refs.outputs.head_sha }} | |
| fetch-depth: 0 | |
| - name: Detect SEA-relevant changes | |
| id: changed | |
| env: | |
| HEAD_SHA: ${{ steps.refs.outputs.head_sha }} | |
| BASE_SHA: ${{ github.event_name == 'merge_group' && github.event.merge_group.base_sha || github.event.pull_request.base.sha }} | |
| run: | | |
| CHANGED=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA") | |
| echo "Changed files:" | |
| echo "$CHANGED" | |
| # Run when the SEA driver layer, the napi binding contract, SEA | |
| # e2e tests, this workflow, the kernel revision pin, or core deps | |
| # move. | |
| if echo "$CHANGED" | grep -qE "^(lib/sea/|native/sea/|tests/e2e/sea/|tests/unit/sea/|\.github/workflows/kernel-e2e\.yml|KERNEL_REV|package\.json|package-lock\.json)"; then | |
| echo "run_tests=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run_tests=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| # ─────────────────────────────────────────────────────────────── | |
| # Real test job. Builds the napi binding from the pinned kernel SHA | |
| # and runs the SEA e2e suite against the dogfood warehouse. | |
| # ─────────────────────────────────────────────────────────────── | |
| run-kernel-e2e: | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.run_tests == 'true' | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| environment: azure-prod | |
| permissions: | |
| contents: read | |
| checks: write | |
| id-token: write | |
| env: | |
| # SEA e2e tests gate on the DATABRICKS_PECOTESTING_* vars; map the | |
| # warehouse secrets onto them so the suite actually runs (it skips | |
| # when they are absent). | |
| DATABRICKS_PECOTESTING_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} | |
| DATABRICKS_PECOTESTING_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} | |
| DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_TOKEN }} | |
| steps: | |
| - name: Check out driver | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| ref: ${{ needs.detect-changes.outputs.head_sha }} | |
| - name: Read pinned kernel SHA | |
| id: kernel-rev | |
| run: | | |
| if [[ ! -f KERNEL_REV ]]; then | |
| echo "::error::KERNEL_REV file missing" | |
| exit 1 | |
| fi | |
| REV=$(tr -d '[:space:]' < KERNEL_REV) | |
| if [[ ! "$REV" =~ ^[0-9a-f]{40}$ ]]; then | |
| echo "::error::KERNEL_REV must be a 40-char commit SHA, got: $REV" | |
| exit 1 | |
| fi | |
| echo "rev=$REV" >> "$GITHUB_OUTPUT" | |
| echo "Pinned kernel SHA: $REV" | |
| - name: Generate GitHub App token (kernel repo read access) | |
| id: app-token | |
| uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 | |
| with: | |
| app-id: ${{ secrets.INTEGRATION_TEST_APP_ID }} | |
| private-key: ${{ secrets.INTEGRATION_TEST_PRIVATE_KEY }} | |
| owner: databricks | |
| repositories: databricks-sql-kernel | |
| - name: Check out kernel at pinned SHA | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| repository: databricks/databricks-sql-kernel | |
| ref: ${{ steps.kernel-rev.outputs.rev }} | |
| token: ${{ steps.app-token.outputs.token }} | |
| path: databricks-sql-kernel | |
| - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 | |
| with: | |
| node-version: 20 | |
| - name: Set up Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2 | |
| with: | |
| cache: false | |
| - name: Cache cargo build artifacts (keyed on kernel SHA) | |
| uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 | |
| with: | |
| workspaces: databricks-sql-kernel | |
| key: kernel-${{ steps.kernel-rev.outputs.rev }} | |
| - name: Set up JFrog (npm registry proxy) | |
| uses: ./.github/actions/setup-jfrog | |
| - name: Configure Cargo for JFrog proxy | |
| shell: bash | |
| # databricks-protected-runner-group blocks direct egress to | |
| # index.crates.io, so cargo must route through JFrog's | |
| # db-cargo-remote proxy. Reuses the JFrog token setup-jfrog | |
| # exported into the environment. | |
| run: | | |
| set -euo pipefail | |
| mkdir -p ~/.cargo | |
| cat > ~/.cargo/config.toml << 'EOF' | |
| [source.crates-io] | |
| replace-with = "jfrog" | |
| [source.jfrog] | |
| registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" | |
| [registries.jfrog] | |
| index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" | |
| credential-provider = ["cargo:token"] | |
| EOF | |
| cat > ~/.cargo/credentials.toml << EOF | |
| [registries.jfrog] | |
| token = "Bearer ${JFROG_ACCESS_TOKEN}" | |
| EOF | |
| echo "CARGO_REGISTRIES_JFROG_TOKEN=Bearer ${JFROG_ACCESS_TOKEN}" >> "$GITHUB_ENV" | |
| - name: Install driver deps | |
| run: npm ci | |
| - name: Build napi binding from pinned kernel | |
| # build:native cd's into ${DATABRICKS_SQL_KERNEL_REPO}/napi, runs the | |
| # napi-rs build, and copies index.* into native/sea/. Pointing it at | |
| # the SHA-pinned kernel checkout is what makes the binary match | |
| # KERNEL_REV exactly. | |
| env: | |
| DATABRICKS_SQL_KERNEL_REPO: ${{ github.workspace }}/databricks-sql-kernel | |
| run: npm run build:native | |
| - name: Assert committed binding matches KERNEL_REV | |
| # The committed native/sea/index.d.ts + index.js are the consumer-facing | |
| # type contract + platform router; they MUST correspond to the pinned | |
| # kernel. build:native just regenerated them from the KERNEL_REV | |
| # checkout, so any diff means the committed contract drifted from the | |
| # pin — fail loudly and tell the author to commit the regenerated files. | |
| # (The .node binaries are gitignored, so git diff only sees the contract.) | |
| run: | | |
| if ! git diff --exit-code -- native/sea/index.d.ts native/sea/index.js; then | |
| echo "::error::native/sea/index.d.ts / index.js are out of sync with KERNEL_REV ($(tr -d '[:space:]' < KERNEL_REV)). Run 'npm run build:native' against that kernel SHA and commit native/sea/index.*." | |
| exit 1 | |
| fi | |
| echo "Committed binding matches KERNEL_REV." | |
| - name: Smoke-check binding loads | |
| run: node -e "const b=require('./native/sea'); if(typeof b.version!=='function'){throw new Error('napi binding failed to load')} console.log('kernel binding ok:', b.version())" | |
| - name: Run SEA e2e tests | |
| # Invoke mocha directly rather than via `npm run e2e -- <glob>`: routing a | |
| # glob through the npm-script's inner shell mangles `**` and silently | |
| # resolves to ZERO files (a false pass). mocha expands the quoted glob | |
| # itself, reliably matching every tests/e2e/sea file. | |
| run: NODE_OPTIONS="--max-old-space-size=4096" npx mocha --config tests/e2e/.mocharc.js "tests/e2e/sea/**/*.test.ts" | |
| - name: Post Kernel E2E check (success) | |
| if: success() | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 | |
| with: | |
| github-token: ${{ github.token }} | |
| script: | | |
| await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'Kernel E2E', | |
| head_sha: '${{ needs.detect-changes.outputs.head_sha }}', | |
| status: 'completed', | |
| conclusion: 'success', | |
| completed_at: new Date().toISOString(), | |
| output: { | |
| title: 'Kernel E2E passed', | |
| summary: 'tests/e2e/sea ran green against the pinned kernel SHA.' | |
| } | |
| }); | |
| - name: Post Kernel E2E check (failure) | |
| if: failure() | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 | |
| with: | |
| github-token: ${{ github.token }} | |
| script: | | |
| await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'Kernel E2E', | |
| head_sha: '${{ needs.detect-changes.outputs.head_sha }}', | |
| status: 'completed', | |
| conclusion: 'failure', | |
| completed_at: new Date().toISOString(), | |
| output: { | |
| title: 'Kernel E2E failed', | |
| summary: 'See workflow logs for details.' | |
| } | |
| }); | |
| # ─────────────────────────────────────────────────────────────── | |
| # Auto-pass the Kernel E2E check in the merge queue when no SEA- | |
| # relevant files changed. | |
| # ─────────────────────────────────────────────────────────────── | |
| auto-pass-merge-queue: | |
| needs: detect-changes | |
| if: github.event_name == 'merge_group' && needs.detect-changes.outputs.run_tests != 'true' | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| permissions: | |
| checks: write | |
| steps: | |
| - name: Auto-pass | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 | |
| with: | |
| github-token: ${{ github.token }} | |
| script: | | |
| await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'Kernel E2E', | |
| head_sha: '${{ github.event.merge_group.head_sha }}', | |
| status: 'completed', | |
| conclusion: 'success', | |
| completed_at: new Date().toISOString(), | |
| output: { | |
| title: 'Skipped — no SEA-relevant changes', | |
| summary: 'No files under lib/sea/, native/sea/, tests/e2e/sea/, tests/unit/sea/, KERNEL_REV, package.json, or package-lock.json changed.' | |
| } | |
| }); |