From 5104b05371a3bfb27cb4d6a2c88791ece91171c5 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 11:34:00 -0500 Subject: [PATCH 01/25] Add Kitmaker Portal publish POC via Charon Ferry Adds a reusable kitmaker_portal.yaml workflow with 'smoke-test' (poll an existing status ID) and 'release' (POST + poll) modes, called from a manual workflow_dispatch job in pr.yaml and a tag-gated job in build.yaml after wheels are attached to a GitHub Release. Kitmaker Portal request/response field names are best-effort pending confirmation from the Charon/Kitmaker teams (KITMAKER-4800). Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yaml | 49 +++++++ .github/workflows/kitmaker_portal.yaml | 188 +++++++++++++++++++++++++ .github/workflows/pr.yaml | 23 +++ 3 files changed, 260 insertions(+) create mode 100644 .github/workflows/kitmaker_portal.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 45b7b461a3..ab54434adc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -287,6 +287,55 @@ jobs: package-name: cuopt_sh_client package-type: python publish-wheel-search-key: cuopt_wheel_python_cuopt-sh-client + release-github: + # Attaches the built wheels to a GitHub Release on the version tag, so Kitmaker + # can pull them from a public Release URL (Kitmaker itself never reaches inside NVIDIA). + if: startsWith(github.ref, 'refs/tags/v') + needs: + - wheel-publish-libcuopt + - wheel-publish-cuopt + - wheel-publish-cuopt-server + - wheel-publish-cuopt-sh-client + permissions: + contents: write + runs-on: linux-amd64-cpu4 + outputs: + release-tag: ${{ steps.release.outputs.tag }} + steps: + - name: Download built wheels + uses: actions/download-artifact@v4 + with: + pattern: "wheel_*" + path: dist + merge-multiple: true + - name: Create GitHub Release and upload wheels + id: release + env: + GH_TOKEN: ${{ github.token }} + run: | + tag="${GITHUB_REF#refs/tags/}" + mapfile -t wheels < <(find dist -name '*.whl') + gh release create "$tag" \ + --repo "${{ github.repository }}" \ + --title "$tag" \ + --generate-notes \ + "${wheels[@]}" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + kitmaker-portal-release: + # Calls the Kitmaker Portal release API (via Charon Ferry) to publish the wheels + # from the GitHub Release above to pypi.org / pypi.nvidia.com. + if: startsWith(github.ref, 'refs/tags/v') + needs: release-github + permissions: + id-token: write + contents: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: ./.github/workflows/kitmaker_portal.yaml + with: + mode: release + release_tag: ${{ needs.release-github.outputs.release-tag }} + tests: # Docker image build / tests aren't necessary for the 'test.yaml' workflow, # so 'test.yaml' can be triggered without waiting for those. diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml new file mode 100644 index 0000000000..5b67df1a1a --- /dev/null +++ b/.github/workflows/kitmaker_portal.yaml @@ -0,0 +1,188 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Kitmaker Portal (via Charon Ferry) + +on: + workflow_call: + inputs: + mode: + description: | + mode: 'smoke-test' polls an existing Kitmaker Portal status ID (no side effects). + 'release' requests a new release for 'release_tag' and polls it to completion. + required: true + type: string + status_id: + description: "status_id: existing Kitmaker Portal status UUID. Required for mode == 'smoke-test'." + type: string + release_tag: + description: "release_tag: git tag whose GitHub Release should be published to Kitmaker. Required for mode == 'release'." + type: string + outputs: + status_id: + description: Kitmaker Portal status UUID created by mode == 'release'. + value: ${{ jobs.release.outputs.status_id }} + +defaults: + run: + shell: bash + +jobs: + smoke-test: + if: inputs.mode == 'smoke-test' + runs-on: [self-hosted, linux] + permissions: + id-token: write + contents: read + steps: + - name: Start staging Ferry tunnel + run: | + command -v tbot + + cat > "$RUNNER_TEMP/tbot.yaml" <<'YAML' + version: v2 + proxy_server: nv-stg-ps.teleport.sh:443 + onboarding: + join_method: github + token: charon-gha-runners + storage: + type: memory + outputs: [] + services: + - type: application-tunnel + app_name: charon + listen: tcp://127.0.0.1:8888 + YAML + + tbot start -c "$RUNNER_TEMP/tbot.yaml" \ + >"$RUNNER_TEMP/tbot.log" 2>&1 & + + timeout 30 bash -c \ + 'until curl -sf http://127.0.0.1:8888/ready >/dev/null; do sleep 1; done' + + - name: Mint Ferry identity token + id: ferry-token + uses: actions/github-script@v7 + with: + script: | + const token = await core.getIDToken('charon.nvidia.com'); + core.setSecret(token); + core.setOutput('token', token); + + - name: Poll Kitmaker Portal status + env: + FERRY_TOKEN: ${{ steps.ferry-token.outputs.token }} + # Store the complete value, such as "Bearer …", in this secret. + PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} + STATUS_ID: ${{ inputs.status_id }} + run: | + if [[ -z "$STATUS_ID" ]]; then + echo "::error::mode == 'smoke-test' requires 'status_id'" + exit 1 + fi + + curl --fail-with-body --show-error \ + -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ + -H "Authorization: $PORTAL_AUTHORIZATION" \ + "http://127.0.0.1:8888/kitmaker-portal/api/v0/status/$STATUS_ID" + + release: + # NOTE: request/response field names below (release_url, uuid, state) are best-effort + # from the Slack thread with Andrew Dickson / Jesus Alvarez (KITMAKER-4800) and must be + # confirmed against the actual Kitmaker Portal API contract before this can run for real. + if: inputs.mode == 'release' + runs-on: [self-hosted, linux] + permissions: + id-token: write + contents: read + outputs: + status_id: ${{ steps.kitmaker-release.outputs.status_id }} + env: + KITMAKER_PROJECT_ID: ${{ vars.KITMAKER_PROJECT_ID }} + steps: + - name: Require a release tag + run: | + if [[ -z "${{ inputs.release_tag }}" ]]; then + echo "::error::mode == 'release' requires 'release_tag'" + exit 1 + fi + + - name: Start staging Ferry tunnel + run: | + command -v tbot + + cat > "$RUNNER_TEMP/tbot.yaml" <<'YAML' + version: v2 + proxy_server: nv-stg-ps.teleport.sh:443 + onboarding: + join_method: github + token: charon-gha-runners + storage: + type: memory + outputs: [] + services: + - type: application-tunnel + app_name: charon + listen: tcp://127.0.0.1:8888 + YAML + + tbot start -c "$RUNNER_TEMP/tbot.yaml" \ + >"$RUNNER_TEMP/tbot.log" 2>&1 & + + timeout 30 bash -c \ + 'until curl -sf http://127.0.0.1:8888/ready >/dev/null; do sleep 1; done' + + - name: Mint Ferry identity token + id: ferry-token + uses: actions/github-script@v7 + with: + script: | + const token = await core.getIDToken('charon.nvidia.com'); + core.setSecret(token); + core.setOutput('token', token); + + - name: Request Kitmaker Portal release + id: kitmaker-release + env: + FERRY_TOKEN: ${{ steps.ferry-token.outputs.token }} + # Store the complete value, such as "Bearer …", in this secret. + PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} + RELEASE_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.release_tag }} + run: | + response=$(curl --fail-with-body --show-error \ + -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ + -H "Authorization: $PORTAL_AUTHORIZATION" \ + -H "Content-Type: application/json" \ + -d "{\"release_url\": \"${RELEASE_URL}\"}" \ + "http://127.0.0.1:8888/kitmaker-portal/api/v0/projects/${KITMAKER_PROJECT_ID}/releases") + + echo "$response" + status_id=$(echo "$response" | jq -r '.uuid') + echo "status_id=$status_id" >> "$GITHUB_OUTPUT" + + - name: Poll Kitmaker Portal status + env: + FERRY_TOKEN: ${{ steps.ferry-token.outputs.token }} + PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} + STATUS_ID: ${{ steps.kitmaker-release.outputs.status_id }} + run: | + for _ in $(seq 1 60); do + response=$(curl --fail-with-body --show-error \ + -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ + -H "Authorization: $PORTAL_AUTHORIZATION" \ + "http://127.0.0.1:8888/kitmaker-portal/api/v0/status/$STATUS_ID") + state=$(echo "$response" | jq -r '.state') + echo "Kitmaker Portal release status: $state" + case "$state" in + succeeded|completed|success) + exit 0 + ;; + failed|error) + echo "$response" + exit 1 + ;; + esac + sleep 30 + done + echo "Timed out waiting for Kitmaker Portal release to complete" + exit 1 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3169519c0a..536c28609d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -7,6 +7,15 @@ on: push: branches: - "pull-request/[0-9]+" + workflow_dispatch: + inputs: + kitmaker_status_id: + description: | + kitmaker_status_id: existing Kitmaker Portal status UUID to poll, e.g. one + returned by a prior 'kitmaker-portal-release' run in build.yaml. + Used only to smoke-test the Charon Ferry -> kitmaker-portal path in staging. + required: true + type: string concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -585,6 +594,20 @@ jobs: with: build_type: pull-request script: ci/test_self_hosted_service.sh + kitmaker-portal-smoke-test: + # Manual-only POC call to validate the Charon Ferry -> kitmaker-portal.nvidia.com + # (staging) path before wiring the real publish call in build.yaml. Not part of + # pr-builder's required checks since it isn't triggered by normal PR pushes. + if: github.event_name == 'workflow_dispatch' + permissions: + id-token: write + contents: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: ./.github/workflows/kitmaker_portal.yaml + with: + mode: smoke-test + status_id: ${{ inputs.kitmaker_status_id }} + pr-test-summary: name: "PR test summary (non-blocking)" needs: From b479770461a35753e5fe021f68890e2001e0dac2 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 11:44:10 -0500 Subject: [PATCH 02/25] Match Kitmaker Portal API against real docs; split release by package Reconciles the request/response schema against the actual Kitmaker Portal API docs (kitmaker.gitlab-master-pages.nvidia.com), which differ from our earlier guesses: - POST body is {project_name, payload: [{pic, job_type, url, upload}]} with one entry per wheel/sdist asset URL, not a single release URL. - upload must be explicitly true, or Kitmaker only dry-run validates. - Success response field is release_uuid, not uuid. - Status field is 'status' (pending/in_progress/completed/failed), not 'state'. Since a Kitmaker project must be named identically to its wheel component, split the single kitmaker-portal-release job in build.yaml into one job per package (libcuopt/cuopt/cuopt_server/cuopt_sh_client), mirroring the existing wheel-publish-* jobs. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yaml | 53 ++++++++++++++++- .github/workflows/kitmaker_portal.yaml | 80 ++++++++++++++++++++------ 2 files changed, 113 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ab54434adc..d8bbf2ada5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -322,9 +322,11 @@ jobs: "${wheels[@]}" echo "tag=$tag" >> "$GITHUB_OUTPUT" - kitmaker-portal-release: - # Calls the Kitmaker Portal release API (via Charon Ferry) to publish the wheels - # from the GitHub Release above to pypi.org / pypi.nvidia.com. + # Each job below calls the Kitmaker Portal release API (via Charon Ferry) to publish + # one package's wheels from the GitHub Release above to pypi.org / pypi.nvidia.com. + # Kitmaker projects are 1:1 with wheel/component name, so one call per package, + # mirroring the wheel-publish-* jobs above. + kitmaker-portal-release-libcuopt: if: startsWith(github.ref, 'refs/tags/v') needs: release-github permissions: @@ -335,6 +337,51 @@ jobs: with: mode: release release_tag: ${{ needs.release-github.outputs.release-tag }} + project_name: libcuopt + project_id: ${{ vars.KITMAKER_PROJECT_ID_LIBCUOPT }} + wheel-asset-pattern: "^libcuopt-" + kitmaker-portal-release-cuopt: + if: startsWith(github.ref, 'refs/tags/v') + needs: release-github + permissions: + id-token: write + contents: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: ./.github/workflows/kitmaker_portal.yaml + with: + mode: release + release_tag: ${{ needs.release-github.outputs.release-tag }} + project_name: cuopt + project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT }} + wheel-asset-pattern: "^cuopt-" + kitmaker-portal-release-cuopt-server: + if: startsWith(github.ref, 'refs/tags/v') + needs: release-github + permissions: + id-token: write + contents: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: ./.github/workflows/kitmaker_portal.yaml + with: + mode: release + release_tag: ${{ needs.release-github.outputs.release-tag }} + project_name: cuopt_server + project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SERVER }} + wheel-asset-pattern: "^cuopt_server-" + kitmaker-portal-release-cuopt-sh-client: + if: startsWith(github.ref, 'refs/tags/v') + needs: release-github + permissions: + id-token: write + contents: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: ./.github/workflows/kitmaker_portal.yaml + with: + mode: release + release_tag: ${{ needs.release-github.outputs.release-tag }} + project_name: cuopt_sh_client + project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SH_CLIENT }} + wheel-asset-pattern: "^cuopt_sh_client-" tests: # Docker image build / tests aren't necessary for the 'test.yaml' workflow, diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 5b67df1a1a..08cd5aa984 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -16,7 +16,22 @@ on: description: "status_id: existing Kitmaker Portal status UUID. Required for mode == 'smoke-test'." type: string release_tag: - description: "release_tag: git tag whose GitHub Release should be published to Kitmaker. Required for mode == 'release'." + description: "release_tag: git tag whose GitHub Release assets should be published to Kitmaker. Required for mode == 'release'." + type: string + project_name: + description: "project_name: Kitmaker project name. Must exactly match the wheel/component name. Required for mode == 'release'." + type: string + project_id: + description: "project_id: numeric Kitmaker project ID (from GET /api/v0/projects). Required for mode == 'release'." + type: string + wheel-asset-pattern: + description: | + wheel-asset-pattern: extended regex (matched with jq's 'test()') selecting this + project's wheel/sdist asset names out of the GitHub Release, e.g. '^libcuopt-'. + Required for mode == 'release'. + type: string + pic: + description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to vars.KITMAKER_RELEASE_PIC." type: string outputs: status_id: @@ -87,9 +102,8 @@ jobs: "http://127.0.0.1:8888/kitmaker-portal/api/v0/status/$STATUS_ID" release: - # NOTE: request/response field names below (release_url, uuid, state) are best-effort - # from the Slack thread with Andrew Dickson / Jesus Alvarez (KITMAKER-4800) and must be - # confirmed against the actual Kitmaker Portal API contract before this can run for real. + # Schema (POST /api/v0/projects/{id}/releases, GET /api/v0/status/{uuid}) follows + # https://kitmaker.gitlab-master-pages.nvidia.com/kitmaker-docs/users/portal-api/wheel-release-api.html if: inputs.mode == 'release' runs-on: [self-hosted, linux] permissions: @@ -98,15 +112,38 @@ jobs: outputs: status_id: ${{ steps.kitmaker-release.outputs.status_id }} env: - KITMAKER_PROJECT_ID: ${{ vars.KITMAKER_PROJECT_ID }} + GH_TOKEN: ${{ github.token }} + PROJECT_NAME: ${{ inputs.project_name }} + PROJECT_ID: ${{ inputs.project_id }} + ASSET_PATTERN: ${{ inputs.wheel-asset-pattern }} + PIC: ${{ inputs.pic || vars.KITMAKER_RELEASE_PIC }} steps: - - name: Require a release tag + - name: Validate inputs + run: | + [[ -n "${{ inputs.release_tag }}" ]] || { echo "::error::mode == 'release' requires 'release_tag'"; exit 1; } + [[ -n "$PROJECT_NAME" ]] || { echo "::error::mode == 'release' requires 'project_name'"; exit 1; } + [[ -n "$PROJECT_ID" ]] || { echo "::error::mode == 'release' requires 'project_id'"; exit 1; } + [[ -n "$ASSET_PATTERN" ]] || { echo "::error::mode == 'release' requires 'wheel-asset-pattern'"; exit 1; } + [[ -n "$PIC" ]] || { echo "::error::'pic' input or vars.KITMAKER_RELEASE_PIC must be set"; exit 1; } + + - name: Collect wheel asset URLs from the GitHub Release + id: assets + env: + RELEASE_TAG: ${{ inputs.release_tag }} run: | - if [[ -z "${{ inputs.release_tag }}" ]]; then - echo "::error::mode == 'release' requires 'release_tag'" + urls_json=$(gh release view "$RELEASE_TAG" --repo "${{ github.repository }}" --json assets \ + --jq "[.assets[] | select(.name | test(\"$ASSET_PATTERN\")) | .url]") + + if [[ "$(echo "$urls_json" | jq 'length')" -eq 0 ]]; then + echo "::error::No assets on release '$RELEASE_TAG' matched pattern '$ASSET_PATTERN'" exit 1 fi + payload=$(echo "$urls_json" | jq -c --arg pic "$PIC" ' + map({pic: $pic, job_type: "wheel-release-job", url: ., upload: true}) + ') + echo "payload=$payload" >> "$GITHUB_OUTPUT" + - name: Start staging Ferry tunnel run: | command -v tbot @@ -147,17 +184,20 @@ jobs: FERRY_TOKEN: ${{ steps.ferry-token.outputs.token }} # Store the complete value, such as "Bearer …", in this secret. PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} - RELEASE_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.release_tag }} + PAYLOAD: ${{ steps.assets.outputs.payload }} run: | + body=$(jq -n --arg project_name "$PROJECT_NAME" --argjson payload "$PAYLOAD" \ + '{project_name: $project_name, payload: $payload}') + response=$(curl --fail-with-body --show-error \ -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ -H "Authorization: $PORTAL_AUTHORIZATION" \ -H "Content-Type: application/json" \ - -d "{\"release_url\": \"${RELEASE_URL}\"}" \ - "http://127.0.0.1:8888/kitmaker-portal/api/v0/projects/${KITMAKER_PROJECT_ID}/releases") + -d "$body" \ + "http://127.0.0.1:8888/kitmaker-portal/api/v0/projects/${PROJECT_ID}/releases") echo "$response" - status_id=$(echo "$response" | jq -r '.uuid') + status_id=$(echo "$response" | jq -r '.release_uuid') echo "status_id=$status_id" >> "$GITHUB_OUTPUT" - name: Poll Kitmaker Portal status @@ -171,13 +211,19 @@ jobs: -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ -H "Authorization: $PORTAL_AUTHORIZATION" \ "http://127.0.0.1:8888/kitmaker-portal/api/v0/status/$STATUS_ID") - state=$(echo "$response" | jq -r '.state') - echo "Kitmaker Portal release status: $state" - case "$state" in - succeeded|completed|success) + status=$(echo "$response" | jq -r '.status') + echo "Kitmaker Portal release status: $status" + case "$status" in + completed) exit 0 ;; - failed|error) + failed) + echo "$response" + exit 1 + ;; + pending|in_progress) ;; + *) + echo "::error::Unexpected status '$status'" echo "$response" exit 1 ;; From ba31fdce6a91f0dba5b3b8d1b06fd80a0c89e4ba Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 13:17:06 -0500 Subject: [PATCH 03/25] Default Kitmaker release requests to dry-run (upload: false) The 'upload' field on the Kitmaker release API defaults to false server-side (validate only, no real publish) but our workflow was hardcoding upload: true unconditionally, meaning any manual test of 'release' mode would have triggered a real PyPI publish with no safe dry-run path. Adds an 'upload' input to kitmaker_portal.yaml (default false), and only build.yaml's real tag-triggered jobs pass upload: true explicitly. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yaml | 4 ++++ .github/workflows/kitmaker_portal.yaml | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d8bbf2ada5..8a1f751abd 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -340,6 +340,7 @@ jobs: project_name: libcuopt project_id: ${{ vars.KITMAKER_PROJECT_ID_LIBCUOPT }} wheel-asset-pattern: "^libcuopt-" + upload: true kitmaker-portal-release-cuopt: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -354,6 +355,7 @@ jobs: project_name: cuopt project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT }} wheel-asset-pattern: "^cuopt-" + upload: true kitmaker-portal-release-cuopt-server: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -368,6 +370,7 @@ jobs: project_name: cuopt_server project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SERVER }} wheel-asset-pattern: "^cuopt_server-" + upload: true kitmaker-portal-release-cuopt-sh-client: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -382,6 +385,7 @@ jobs: project_name: cuopt_sh_client project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SH_CLIENT }} wheel-asset-pattern: "^cuopt_sh_client-" + upload: true tests: # Docker image build / tests aren't necessary for the 'test.yaml' workflow, diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 08cd5aa984..01a398b955 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -33,6 +33,13 @@ on: pic: description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to vars.KITMAKER_RELEASE_PIC." type: string + upload: + description: | + upload: forwarded to the Kitmaker release API. 'false' (default) runs validation + checks only, with no real publish. Must be explicitly set to 'true' to actually + publish wheels. Only used for mode == 'release'. + type: boolean + default: false outputs: status_id: description: Kitmaker Portal status UUID created by mode == 'release'. @@ -130,6 +137,7 @@ jobs: id: assets env: RELEASE_TAG: ${{ inputs.release_tag }} + UPLOAD: ${{ inputs.upload }} run: | urls_json=$(gh release view "$RELEASE_TAG" --repo "${{ github.repository }}" --json assets \ --jq "[.assets[] | select(.name | test(\"$ASSET_PATTERN\")) | .url]") @@ -139,8 +147,10 @@ jobs: exit 1 fi - payload=$(echo "$urls_json" | jq -c --arg pic "$PIC" ' - map({pic: $pic, job_type: "wheel-release-job", url: ., upload: true}) + echo "upload=$UPLOAD (false = validate only, no real publish; true = publishes for real)" + + payload=$(echo "$urls_json" | jq -c --arg pic "$PIC" --argjson upload "$UPLOAD" ' + map({pic: $pic, job_type: "wheel-release-job", url: ., upload: $upload}) ') echo "payload=$payload" >> "$GITHUB_OUTPUT" From 8c9540c74145ec90869b5f5acc700aa9b91ea286 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 13:25:24 -0500 Subject: [PATCH 04/25] Add workflow_dispatch to kitmaker_portal.yaml for direct manual testing Lets 'release' mode be exercised directly (e.g. against a disposable Kitmaker test project) without needing a real tag push through build.yaml. Also renames wheel-asset-pattern -> wheel_asset_pattern: GitHub Actions expression dot-notation is ambiguous with subtraction for hyphenated property names, and the other custom inputs already use underscores. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yaml | 8 ++--- .github/workflows/kitmaker_portal.yaml | 44 +++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8a1f751abd..0d62868335 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -339,7 +339,7 @@ jobs: release_tag: ${{ needs.release-github.outputs.release-tag }} project_name: libcuopt project_id: ${{ vars.KITMAKER_PROJECT_ID_LIBCUOPT }} - wheel-asset-pattern: "^libcuopt-" + wheel_asset_pattern: "^libcuopt-" upload: true kitmaker-portal-release-cuopt: if: startsWith(github.ref, 'refs/tags/v') @@ -354,7 +354,7 @@ jobs: release_tag: ${{ needs.release-github.outputs.release-tag }} project_name: cuopt project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT }} - wheel-asset-pattern: "^cuopt-" + wheel_asset_pattern: "^cuopt-" upload: true kitmaker-portal-release-cuopt-server: if: startsWith(github.ref, 'refs/tags/v') @@ -369,7 +369,7 @@ jobs: release_tag: ${{ needs.release-github.outputs.release-tag }} project_name: cuopt_server project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SERVER }} - wheel-asset-pattern: "^cuopt_server-" + wheel_asset_pattern: "^cuopt_server-" upload: true kitmaker-portal-release-cuopt-sh-client: if: startsWith(github.ref, 'refs/tags/v') @@ -384,7 +384,7 @@ jobs: release_tag: ${{ needs.release-github.outputs.release-tag }} project_name: cuopt_sh_client project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SH_CLIENT }} - wheel-asset-pattern: "^cuopt_sh_client-" + wheel_asset_pattern: "^cuopt_sh_client-" upload: true tests: diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 01a398b955..1791f00e95 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -24,9 +24,9 @@ on: project_id: description: "project_id: numeric Kitmaker project ID (from GET /api/v0/projects). Required for mode == 'release'." type: string - wheel-asset-pattern: + wheel_asset_pattern: description: | - wheel-asset-pattern: extended regex (matched with jq's 'test()') selecting this + wheel_asset_pattern: extended regex (matched with jq's 'test()') selecting this project's wheel/sdist asset names out of the GitHub Release, e.g. '^libcuopt-'. Required for mode == 'release'. type: string @@ -44,6 +44,42 @@ on: status_id: description: Kitmaker Portal status UUID created by mode == 'release'. value: ${{ jobs.release.outputs.status_id }} + workflow_dispatch: + inputs: + mode: + description: "mode: 'smoke-test' polls an existing status ID. 'release' requests a new release and polls it." + required: true + type: choice + options: + - smoke-test + - release + status_id: + description: "status_id: existing Kitmaker Portal status UUID. Required for mode == 'smoke-test'." + type: string + release_tag: + description: "release_tag: git tag whose GitHub Release assets should be published to Kitmaker. Required for mode == 'release'." + type: string + project_name: + description: "project_name: Kitmaker project name. Must exactly match the registered project. Required for mode == 'release'." + type: string + project_id: + description: "project_id: numeric Kitmaker project ID (from GET /api/v0/projects). Required for mode == 'release'." + type: string + wheel_asset_pattern: + description: | + wheel_asset_pattern: extended regex (matched with jq's 'test()') selecting this + project's wheel/sdist asset names out of the GitHub Release, e.g. '^libcuopt-'. + Required for mode == 'release'. + type: string + pic: + description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to vars.KITMAKER_RELEASE_PIC." + type: string + upload: + description: | + upload: forwarded to the Kitmaker release API. 'false' (default) runs validation + checks only, with no real publish. Only used for mode == 'release'. + type: boolean + default: false defaults: run: @@ -122,7 +158,7 @@ jobs: GH_TOKEN: ${{ github.token }} PROJECT_NAME: ${{ inputs.project_name }} PROJECT_ID: ${{ inputs.project_id }} - ASSET_PATTERN: ${{ inputs.wheel-asset-pattern }} + ASSET_PATTERN: ${{ inputs.wheel_asset_pattern }} PIC: ${{ inputs.pic || vars.KITMAKER_RELEASE_PIC }} steps: - name: Validate inputs @@ -130,7 +166,7 @@ jobs: [[ -n "${{ inputs.release_tag }}" ]] || { echo "::error::mode == 'release' requires 'release_tag'"; exit 1; } [[ -n "$PROJECT_NAME" ]] || { echo "::error::mode == 'release' requires 'project_name'"; exit 1; } [[ -n "$PROJECT_ID" ]] || { echo "::error::mode == 'release' requires 'project_id'"; exit 1; } - [[ -n "$ASSET_PATTERN" ]] || { echo "::error::mode == 'release' requires 'wheel-asset-pattern'"; exit 1; } + [[ -n "$ASSET_PATTERN" ]] || { echo "::error::mode == 'release' requires 'wheel_asset_pattern'"; exit 1; } [[ -n "$PIC" ]] || { echo "::error::'pic' input or vars.KITMAKER_RELEASE_PIC must be set"; exit 1; } - name: Collect wheel asset URLs from the GitHub Release From 44b5ec26971bf59bb18889d66ae6aa74898e7c1e Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 13:56:25 -0500 Subject: [PATCH 05/25] Test kitmaker_portal.yaml release mode via pr.yaml, not workflow_dispatch workflow_dispatch can't target a workflow file that only exists on a branch (GitHub requires it on the default branch first). pr.yaml is already registered on main, so extend its dispatch inputs to drive either mode of kitmaker_portal.yaml via workflow_call, which has no such restriction. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/pr.yaml | 53 +++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 536c28609d..f89e3394c8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -9,13 +9,41 @@ on: - "pull-request/[0-9]+" workflow_dispatch: inputs: - kitmaker_status_id: + kitmaker_mode: description: | - kitmaker_status_id: existing Kitmaker Portal status UUID to poll, e.g. one - returned by a prior 'kitmaker-portal-release' run in build.yaml. - Used only to smoke-test the Charon Ferry -> kitmaker-portal path in staging. + kitmaker_mode: POC entry point for testing kitmaker_portal.yaml without merging it + to main first ('workflow_dispatch' can't target a brand-new file on another branch, + but this already-registered workflow can 'uses:' one via workflow_call). + 'smoke-test' polls 'kitmaker_status_id'. 'release' requests a release using the + kitmaker_* inputs below. required: true + type: choice + default: smoke-test + options: + - smoke-test + - release + kitmaker_status_id: + description: "kitmaker_status_id: existing Kitmaker Portal status UUID. Required for kitmaker_mode == 'smoke-test'." + type: string + kitmaker_release_tag: + description: "kitmaker_release_tag: git tag whose GitHub Release assets should be published. Required for kitmaker_mode == 'release'." + type: string + kitmaker_project_name: + description: "kitmaker_project_name: Kitmaker project name. Required for kitmaker_mode == 'release'." + type: string + kitmaker_project_id: + description: "kitmaker_project_id: numeric Kitmaker project ID. Required for kitmaker_mode == 'release'." + type: string + kitmaker_wheel_asset_pattern: + description: "kitmaker_wheel_asset_pattern: regex selecting wheel/sdist assets from the release. Required for kitmaker_mode == 'release'." + type: string + kitmaker_pic: + description: "kitmaker_pic: 'person in charge' email required by the Kitmaker release API." type: string + kitmaker_upload: + description: "kitmaker_upload: 'false' (default) validates only, no real publish. Only used for kitmaker_mode == 'release'." + type: boolean + default: false concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -594,10 +622,13 @@ jobs: with: build_type: pull-request script: ci/test_self_hosted_service.sh - kitmaker-portal-smoke-test: + kitmaker-portal-test: # Manual-only POC call to validate the Charon Ferry -> kitmaker-portal.nvidia.com - # (staging) path before wiring the real publish call in build.yaml. Not part of - # pr-builder's required checks since it isn't triggered by normal PR pushes. + # path before wiring the real publish call in build.yaml. Also used as a workaround + # to test kitmaker_portal.yaml before merging it to main: 'workflow_dispatch' can't + # target a workflow file that only exists on a branch, but this already-registered + # workflow can 'uses:' one via workflow_call. Not part of pr-builder's required + # checks since it isn't triggered by normal PR pushes. if: github.event_name == 'workflow_dispatch' permissions: id-token: write @@ -605,8 +636,14 @@ jobs: secrets: inherit # zizmor: ignore[secrets-inherit] uses: ./.github/workflows/kitmaker_portal.yaml with: - mode: smoke-test + mode: ${{ inputs.kitmaker_mode }} status_id: ${{ inputs.kitmaker_status_id }} + release_tag: ${{ inputs.kitmaker_release_tag }} + project_name: ${{ inputs.kitmaker_project_name }} + project_id: ${{ inputs.kitmaker_project_id }} + wheel_asset_pattern: ${{ inputs.kitmaker_wheel_asset_pattern }} + pic: ${{ inputs.kitmaker_pic }} + upload: ${{ inputs.kitmaker_upload }} pr-test-summary: name: "PR test summary (non-blocking)" From 2918bbccfe9c951f891ecae35310abefa118ab4d Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 14:11:53 -0500 Subject: [PATCH 06/25] Try RAPIDS' existing self-hosted runner pool for Charon Ferry jobs [self-hosted, linux] from Andrew's example doesn't match any runner label used elsewhere in this repo (linux-amd64-cpu4, linux-amd64-gpu-*, etc.), and the smoke-test/release jobs sat queued indefinitely with no pickup. Trying linux-amd64-cpu4, the pool already proven to work for other jobs in this repo, as a diagnostic step. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/kitmaker_portal.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 1791f00e95..b2450623cf 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -88,7 +88,7 @@ defaults: jobs: smoke-test: if: inputs.mode == 'smoke-test' - runs-on: [self-hosted, linux] + runs-on: linux-amd64-cpu4 permissions: id-token: write contents: read @@ -148,7 +148,7 @@ jobs: # Schema (POST /api/v0/projects/{id}/releases, GET /api/v0/status/{uuid}) follows # https://kitmaker.gitlab-master-pages.nvidia.com/kitmaker-docs/users/portal-api/wheel-release-api.html if: inputs.mode == 'release' - runs-on: [self-hosted, linux] + runs-on: linux-amd64-cpu4 permissions: id-token: write contents: read From 2004a33ac164215b9ad03d31d197fd7dce3520d5 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 14:16:30 -0500 Subject: [PATCH 07/25] Use ubuntu-latest + official teleport-actions for Ferry tunnel setup linux-amd64-cpu4 confirmed the real blocker: tbot isn't installed on that image ('command -v tbot' failed, exit code 1, aborting the step immediately under bash -e). Switches to ubuntu-latest (matches the original design intent: a standard GitHub-hosted runner reaching Kitmaker via Charon Ferry, per the initial ask to the Charon team) and replaces the hand-rolled tbot.yaml + background process + wait-loop with the official teleport-actions/setup + teleport-actions/application-tunnel actions, which install tbot and manage the tunnel lifecycle (including readiness waiting and log capture on failure) directly. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/kitmaker_portal.yaml | 74 +++++++++----------------- 1 file changed, 26 insertions(+), 48 deletions(-) diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index b2450623cf..b1f4c317bc 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -88,35 +88,24 @@ defaults: jobs: smoke-test: if: inputs.mode == 'smoke-test' - runs-on: linux-amd64-cpu4 + runs-on: ubuntu-latest permissions: id-token: write contents: read steps: - - name: Start staging Ferry tunnel - run: | - command -v tbot - - cat > "$RUNNER_TEMP/tbot.yaml" <<'YAML' - version: v2 - proxy_server: nv-stg-ps.teleport.sh:443 - onboarding: - join_method: github - token: charon-gha-runners - storage: - type: memory - outputs: [] - services: - - type: application-tunnel - app_name: charon - listen: tcp://127.0.0.1:8888 - YAML - - tbot start -c "$RUNNER_TEMP/tbot.yaml" \ - >"$RUNNER_TEMP/tbot.log" 2>&1 & + - name: Install Teleport + uses: teleport-actions/setup@v1 + with: + version: auto + proxy: nv-stg-ps.teleport.sh:443 - timeout 30 bash -c \ - 'until curl -sf http://127.0.0.1:8888/ready >/dev/null; do sleep 1; done' + - name: Start staging Ferry tunnel + uses: teleport-actions/application-tunnel@v1 + with: + proxy: nv-stg-ps.teleport.sh:443 + token: charon-gha-runners + app: charon + listen: tcp://127.0.0.1:8888 - name: Mint Ferry identity token id: ferry-token @@ -148,7 +137,7 @@ jobs: # Schema (POST /api/v0/projects/{id}/releases, GET /api/v0/status/{uuid}) follows # https://kitmaker.gitlab-master-pages.nvidia.com/kitmaker-docs/users/portal-api/wheel-release-api.html if: inputs.mode == 'release' - runs-on: linux-amd64-cpu4 + runs-on: ubuntu-latest permissions: id-token: write contents: read @@ -190,30 +179,19 @@ jobs: ') echo "payload=$payload" >> "$GITHUB_OUTPUT" - - name: Start staging Ferry tunnel - run: | - command -v tbot - - cat > "$RUNNER_TEMP/tbot.yaml" <<'YAML' - version: v2 - proxy_server: nv-stg-ps.teleport.sh:443 - onboarding: - join_method: github - token: charon-gha-runners - storage: - type: memory - outputs: [] - services: - - type: application-tunnel - app_name: charon - listen: tcp://127.0.0.1:8888 - YAML - - tbot start -c "$RUNNER_TEMP/tbot.yaml" \ - >"$RUNNER_TEMP/tbot.log" 2>&1 & + - name: Install Teleport + uses: teleport-actions/setup@v1 + with: + version: auto + proxy: nv-stg-ps.teleport.sh:443 - timeout 30 bash -c \ - 'until curl -sf http://127.0.0.1:8888/ready >/dev/null; do sleep 1; done' + - name: Start staging Ferry tunnel + uses: teleport-actions/application-tunnel@v1 + with: + proxy: nv-stg-ps.teleport.sh:443 + token: charon-gha-runners + app: charon + listen: tcp://127.0.0.1:8888 - name: Mint Ferry identity token id: ferry-token From b6c015d3c9819bfe2f9092ca30308b6c930b46da Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 14:18:40 -0500 Subject: [PATCH 08/25] Fix swallowed error response bodies in kitmaker_portal.yaml 'response=\$(curl --fail-with-body ...)' under bash -e aborts the script on a non-2xx before the subsequent 'echo "\$response"' line ever runs, so failures showed 'exit code 22' with no visible response body (as seen when the real 'Request Kitmaker Portal release' call 403'd). Captures HTTP status separately from the body and always echoes the body before deciding to fail, in both the release POST and the status poll. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/kitmaker_portal.yaml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index b1f4c317bc..8675dd2480 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -213,14 +213,23 @@ jobs: body=$(jq -n --arg project_name "$PROJECT_NAME" --argjson payload "$PAYLOAD" \ '{project_name: $project_name, payload: $payload}') - response=$(curl --fail-with-body --show-error \ + http_response=$(curl -sS -w '\n%{http_code}' \ -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ -H "Authorization: $PORTAL_AUTHORIZATION" \ -H "Content-Type: application/json" \ -d "$body" \ "http://127.0.0.1:8888/kitmaker-portal/api/v0/projects/${PROJECT_ID}/releases") + http_code=$(echo "$http_response" | tail -n1) + response=$(echo "$http_response" | sed '$d') + + echo "HTTP $http_code" echo "$response" + + if [[ "$http_code" != "202" ]]; then + exit 1 + fi + status_id=$(echo "$response" | jq -r '.release_uuid') echo "status_id=$status_id" >> "$GITHUB_OUTPUT" @@ -231,10 +240,19 @@ jobs: STATUS_ID: ${{ steps.kitmaker-release.outputs.status_id }} run: | for _ in $(seq 1 60); do - response=$(curl --fail-with-body --show-error \ + http_response=$(curl -sS -w '\n%{http_code}' \ -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ -H "Authorization: $PORTAL_AUTHORIZATION" \ "http://127.0.0.1:8888/kitmaker-portal/api/v0/status/$STATUS_ID") + http_code=$(echo "$http_response" | tail -n1) + response=$(echo "$http_response" | sed '$d') + + if [[ "$http_code" != "200" ]]; then + echo "HTTP $http_code" + echo "$response" + exit 1 + fi + status=$(echo "$response" | jq -r '.status') echo "Kitmaker Portal release status: $status" case "$status" in From 5051620e953523210bc480e54194a8b73a4affd0 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 15:22:14 -0500 Subject: [PATCH 09/25] Prepend 'Bearer ' to the Kitmaker Authorization header in the workflow The KITMAKER_PORTAL_AUTHORIZATION secret holds the bare token (kmp_...), matching what's copied directly from the Portal UI. Sending it as-is produced 401 {"detail":"Authentication required"} since Kitmaker expects an 'Authorization: Bearer ' scheme. Prepending 'Bearer ' in the workflow means the secret can just hold the raw token, which is less error-prone than relying on whoever sets/rotates it to remember to include the scheme themselves. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/kitmaker_portal.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 8675dd2480..96e84b1fb7 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -119,7 +119,7 @@ jobs: - name: Poll Kitmaker Portal status env: FERRY_TOKEN: ${{ steps.ferry-token.outputs.token }} - # Store the complete value, such as "Bearer …", in this secret. + # Store just the bare token (e.g. kmp_...); "Bearer " is prepended below. PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} STATUS_ID: ${{ inputs.status_id }} run: | @@ -130,7 +130,7 @@ jobs: curl --fail-with-body --show-error \ -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ - -H "Authorization: $PORTAL_AUTHORIZATION" \ + -H "Authorization: Bearer $PORTAL_AUTHORIZATION" \ "http://127.0.0.1:8888/kitmaker-portal/api/v0/status/$STATUS_ID" release: @@ -206,7 +206,7 @@ jobs: id: kitmaker-release env: FERRY_TOKEN: ${{ steps.ferry-token.outputs.token }} - # Store the complete value, such as "Bearer …", in this secret. + # Store just the bare token (e.g. kmp_...); "Bearer " is prepended below. PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} PAYLOAD: ${{ steps.assets.outputs.payload }} run: | @@ -215,7 +215,7 @@ jobs: http_response=$(curl -sS -w '\n%{http_code}' \ -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ - -H "Authorization: $PORTAL_AUTHORIZATION" \ + -H "Authorization: Bearer $PORTAL_AUTHORIZATION" \ -H "Content-Type: application/json" \ -d "$body" \ "http://127.0.0.1:8888/kitmaker-portal/api/v0/projects/${PROJECT_ID}/releases") @@ -242,7 +242,7 @@ jobs: for _ in $(seq 1 60); do http_response=$(curl -sS -w '\n%{http_code}' \ -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ - -H "Authorization: $PORTAL_AUTHORIZATION" \ + -H "Authorization: Bearer $PORTAL_AUTHORIZATION" \ "http://127.0.0.1:8888/kitmaker-portal/api/v0/status/$STATUS_ID") http_code=$(echo "$http_response" | tail -n1) response=$(echo "$http_response" | sed '$d') From afd53ebc1b2920773b152dd85ac05131d7837a1c Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 15:25:42 -0500 Subject: [PATCH 10/25] Default pic from a secret instead of a variable Kitmaker's release API rejects any pic value that doesn't match the target project's registered owner email, so testing against a real project requires a real email. A repo Variable is still visible to anyone who checks repo Settings; a Secret is auto-masked by GitHub Actions anywhere it would appear in log output, which is a better fit for not exposing it on a public OSS repo. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/kitmaker_portal.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 96e84b1fb7..7227b624c1 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -31,7 +31,7 @@ on: Required for mode == 'release'. type: string pic: - description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to vars.KITMAKER_RELEASE_PIC." + description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to secrets.KITMAKER_RELEASE_PIC." type: string upload: description: | @@ -72,7 +72,7 @@ on: Required for mode == 'release'. type: string pic: - description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to vars.KITMAKER_RELEASE_PIC." + description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to secrets.KITMAKER_RELEASE_PIC." type: string upload: description: | @@ -148,7 +148,7 @@ jobs: PROJECT_NAME: ${{ inputs.project_name }} PROJECT_ID: ${{ inputs.project_id }} ASSET_PATTERN: ${{ inputs.wheel_asset_pattern }} - PIC: ${{ inputs.pic || vars.KITMAKER_RELEASE_PIC }} + PIC: ${{ inputs.pic || secrets.KITMAKER_RELEASE_PIC }} steps: - name: Validate inputs run: | @@ -156,7 +156,7 @@ jobs: [[ -n "$PROJECT_NAME" ]] || { echo "::error::mode == 'release' requires 'project_name'"; exit 1; } [[ -n "$PROJECT_ID" ]] || { echo "::error::mode == 'release' requires 'project_id'"; exit 1; } [[ -n "$ASSET_PATTERN" ]] || { echo "::error::mode == 'release' requires 'wheel_asset_pattern'"; exit 1; } - [[ -n "$PIC" ]] || { echo "::error::'pic' input or vars.KITMAKER_RELEASE_PIC must be set"; exit 1; } + [[ -n "$PIC" ]] || { echo "::error::'pic' input or secrets.KITMAKER_RELEASE_PIC must be set"; exit 1; } - name: Collect wheel asset URLs from the GitHub Release id: assets From db8a3e3120fa522e45994805bdcd9be082795b10 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 15:25:55 -0500 Subject: [PATCH 11/25] Match actual secret name: KITMAKER_PORTAL_ACC_OWNER_EMAIL Co-Authored-By: Claude Sonnet 5 --- .github/workflows/kitmaker_portal.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 7227b624c1..cee9e3b8ae 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -31,7 +31,7 @@ on: Required for mode == 'release'. type: string pic: - description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to secrets.KITMAKER_RELEASE_PIC." + description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL." type: string upload: description: | @@ -72,7 +72,7 @@ on: Required for mode == 'release'. type: string pic: - description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to secrets.KITMAKER_RELEASE_PIC." + description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL." type: string upload: description: | @@ -148,7 +148,7 @@ jobs: PROJECT_NAME: ${{ inputs.project_name }} PROJECT_ID: ${{ inputs.project_id }} ASSET_PATTERN: ${{ inputs.wheel_asset_pattern }} - PIC: ${{ inputs.pic || secrets.KITMAKER_RELEASE_PIC }} + PIC: ${{ inputs.pic || secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL }} steps: - name: Validate inputs run: | @@ -156,7 +156,7 @@ jobs: [[ -n "$PROJECT_NAME" ]] || { echo "::error::mode == 'release' requires 'project_name'"; exit 1; } [[ -n "$PROJECT_ID" ]] || { echo "::error::mode == 'release' requires 'project_id'"; exit 1; } [[ -n "$ASSET_PATTERN" ]] || { echo "::error::mode == 'release' requires 'wheel_asset_pattern'"; exit 1; } - [[ -n "$PIC" ]] || { echo "::error::'pic' input or secrets.KITMAKER_RELEASE_PIC must be set"; exit 1; } + [[ -n "$PIC" ]] || { echo "::error::'pic' input or secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL must be set"; exit 1; } - name: Collect wheel asset URLs from the GitHub Release id: assets From 544a231fff192faccef00327f96bdfdfc3a3f4e7 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 15:27:16 -0500 Subject: [PATCH 12/25] Poll: treat unrecognized status values as still-in-progress The POST itself succeeded end to end (HTTP 202, real release_uuid) -- first successful live round-trip through Charon Ferry into Kitmaker. Polling then hard-failed on 'processing', a status value the docs don't mention (only pending|in_progress|completed|failed). Real API responses can drift from docs, so only treat completed/failed as terminal and keep polling on anything else instead. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/kitmaker_portal.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index cee9e3b8ae..9edd97d60d 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -263,12 +263,11 @@ jobs: echo "$response" exit 1 ;; - pending|in_progress) ;; - *) - echo "::error::Unexpected status '$status'" - echo "$response" - exit 1 - ;; + # Any other status (e.g. 'pending', 'in_progress', 'processing') is + # treated as still-in-progress: keep polling rather than hard-failing + # on a status value not covered by the documented pending|in_progress + # |completed|failed set (observed 'processing' in practice). + *) ;; esac sleep 30 done From 62ebc575fb9d9cc0ca49c0b500e2c845cd8377a7 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 15:29:32 -0500 Subject: [PATCH 13/25] Fix pic-masking gap in smoke-test; trim comments smoke-test never referenced secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL, so GitHub Actions never registered it for masking in that job -- a polled status response embedding a real pic email leaked in plaintext (caught and the run deleted). Referencing it (unused) in the job's env forces masking to apply there too. Also trims comments that either duplicated what the code already shows (Bearer prefix) or explained non-essential detail (schema doc link, verbose unknown-status rationale). Co-Authored-By: Claude Sonnet 5 --- .github/workflows/kitmaker_portal.yaml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 9edd97d60d..8e6de97bd5 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -92,6 +92,10 @@ jobs: permissions: id-token: write contents: read + env: + # Referenced here (unused) so GitHub Actions masks it in this job's logs too, + # since a polled status response can echo a release's pic email back. + PIC: ${{ secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL }} steps: - name: Install Teleport uses: teleport-actions/setup@v1 @@ -119,7 +123,6 @@ jobs: - name: Poll Kitmaker Portal status env: FERRY_TOKEN: ${{ steps.ferry-token.outputs.token }} - # Store just the bare token (e.g. kmp_...); "Bearer " is prepended below. PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} STATUS_ID: ${{ inputs.status_id }} run: | @@ -134,8 +137,6 @@ jobs: "http://127.0.0.1:8888/kitmaker-portal/api/v0/status/$STATUS_ID" release: - # Schema (POST /api/v0/projects/{id}/releases, GET /api/v0/status/{uuid}) follows - # https://kitmaker.gitlab-master-pages.nvidia.com/kitmaker-docs/users/portal-api/wheel-release-api.html if: inputs.mode == 'release' runs-on: ubuntu-latest permissions: @@ -206,7 +207,6 @@ jobs: id: kitmaker-release env: FERRY_TOKEN: ${{ steps.ferry-token.outputs.token }} - # Store just the bare token (e.g. kmp_...); "Bearer " is prepended below. PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} PAYLOAD: ${{ steps.assets.outputs.payload }} run: | @@ -263,11 +263,7 @@ jobs: echo "$response" exit 1 ;; - # Any other status (e.g. 'pending', 'in_progress', 'processing') is - # treated as still-in-progress: keep polling rather than hard-failing - # on a status value not covered by the documented pending|in_progress - # |completed|failed set (observed 'processing' in practice). - *) ;; + *) ;; # still in progress, keep polling esac sleep 30 done From e812b89e0092b7eb198eec40312fa78a127fdc3c Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 15:32:51 -0500 Subject: [PATCH 14/25] Split Kitmaker release jobs by package x CUDA major version Real Kitmaker projects are libcuopt-cu12/cu13, cuopt-cu12/cu13, cuopt-server-cu12/cu13, and cuopt-sh-client (no CUDA suffix, pure package) -- not one project per package as assumed earlier. Splits the 4 kitmaker-portal-release-* jobs into 7, matching project name and wheel_asset_pattern to each. CUDA suffix pattern (cu12/cu13 appended to the package name, e.g. libcuopt_cu12-*.whl) follows the 'append-cuda-suffix' convention already used elsewhere in this repo's wheels-build.yaml calls. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yaml | 75 ++++++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0d62868335..35b4a5b886 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -324,9 +324,9 @@ jobs: # Each job below calls the Kitmaker Portal release API (via Charon Ferry) to publish # one package's wheels from the GitHub Release above to pypi.org / pypi.nvidia.com. - # Kitmaker projects are 1:1 with wheel/component name, so one call per package, - # mirroring the wheel-publish-* jobs above. - kitmaker-portal-release-libcuopt: + # Kitmaker projects are 1:1 with wheel/component name x CUDA major version (except + # cuopt-sh-client, a pure package with no CUDA suffix), mirroring wheel-publish-* above. + kitmaker-portal-release-libcuopt-cu12: if: startsWith(github.ref, 'refs/tags/v') needs: release-github permissions: @@ -337,11 +337,11 @@ jobs: with: mode: release release_tag: ${{ needs.release-github.outputs.release-tag }} - project_name: libcuopt - project_id: ${{ vars.KITMAKER_PROJECT_ID_LIBCUOPT }} - wheel_asset_pattern: "^libcuopt-" + project_name: libcuopt-cu12 + project_id: ${{ vars.KITMAKER_PROJECT_ID_LIBCUOPT_CU12 }} + wheel_asset_pattern: "^libcuopt_cu12-" upload: true - kitmaker-portal-release-cuopt: + kitmaker-portal-release-libcuopt-cu13: if: startsWith(github.ref, 'refs/tags/v') needs: release-github permissions: @@ -352,11 +352,11 @@ jobs: with: mode: release release_tag: ${{ needs.release-github.outputs.release-tag }} - project_name: cuopt - project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT }} - wheel_asset_pattern: "^cuopt-" + project_name: libcuopt-cu13 + project_id: ${{ vars.KITMAKER_PROJECT_ID_LIBCUOPT_CU13 }} + wheel_asset_pattern: "^libcuopt_cu13-" upload: true - kitmaker-portal-release-cuopt-server: + kitmaker-portal-release-cuopt-cu12: if: startsWith(github.ref, 'refs/tags/v') needs: release-github permissions: @@ -367,9 +367,54 @@ jobs: with: mode: release release_tag: ${{ needs.release-github.outputs.release-tag }} - project_name: cuopt_server - project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SERVER }} - wheel_asset_pattern: "^cuopt_server-" + project_name: cuopt-cu12 + project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_CU12 }} + wheel_asset_pattern: "^cuopt_cu12-" + upload: true + kitmaker-portal-release-cuopt-cu13: + if: startsWith(github.ref, 'refs/tags/v') + needs: release-github + permissions: + id-token: write + contents: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: ./.github/workflows/kitmaker_portal.yaml + with: + mode: release + release_tag: ${{ needs.release-github.outputs.release-tag }} + project_name: cuopt-cu13 + project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_CU13 }} + wheel_asset_pattern: "^cuopt_cu13-" + upload: true + kitmaker-portal-release-cuopt-server-cu12: + if: startsWith(github.ref, 'refs/tags/v') + needs: release-github + permissions: + id-token: write + contents: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: ./.github/workflows/kitmaker_portal.yaml + with: + mode: release + release_tag: ${{ needs.release-github.outputs.release-tag }} + project_name: cuopt-server-cu12 + project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SERVER_CU12 }} + wheel_asset_pattern: "^cuopt_server_cu12-" + upload: true + kitmaker-portal-release-cuopt-server-cu13: + if: startsWith(github.ref, 'refs/tags/v') + needs: release-github + permissions: + id-token: write + contents: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: ./.github/workflows/kitmaker_portal.yaml + with: + mode: release + release_tag: ${{ needs.release-github.outputs.release-tag }} + project_name: cuopt-server-cu13 + project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SERVER_CU13 }} + wheel_asset_pattern: "^cuopt_server_cu13-" upload: true kitmaker-portal-release-cuopt-sh-client: if: startsWith(github.ref, 'refs/tags/v') @@ -382,7 +427,7 @@ jobs: with: mode: release release_tag: ${{ needs.release-github.outputs.release-tag }} - project_name: cuopt_sh_client + project_name: cuopt-sh-client project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SH_CLIENT }} wheel_asset_pattern: "^cuopt_sh_client-" upload: true From f2a6dcd7d988c9ecc7f62c331b7efe9618c450ff Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 15:43:22 -0500 Subject: [PATCH 15/25] Keep upload: false everywhere until the POC is validated All 7 kitmaker-portal-release-* jobs now dry-run only, no real publish, until we're confident in the wheel_asset_pattern values and overall flow against the real (non-test) Kitmaker projects. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 35b4a5b886..8c7d272bb9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -340,7 +340,7 @@ jobs: project_name: libcuopt-cu12 project_id: ${{ vars.KITMAKER_PROJECT_ID_LIBCUOPT_CU12 }} wheel_asset_pattern: "^libcuopt_cu12-" - upload: true + upload: false # TODO: flip to true once POC is validated kitmaker-portal-release-libcuopt-cu13: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -355,7 +355,7 @@ jobs: project_name: libcuopt-cu13 project_id: ${{ vars.KITMAKER_PROJECT_ID_LIBCUOPT_CU13 }} wheel_asset_pattern: "^libcuopt_cu13-" - upload: true + upload: false # TODO: flip to true once POC is validated kitmaker-portal-release-cuopt-cu12: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -370,7 +370,7 @@ jobs: project_name: cuopt-cu12 project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_CU12 }} wheel_asset_pattern: "^cuopt_cu12-" - upload: true + upload: false # TODO: flip to true once POC is validated kitmaker-portal-release-cuopt-cu13: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -385,7 +385,7 @@ jobs: project_name: cuopt-cu13 project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_CU13 }} wheel_asset_pattern: "^cuopt_cu13-" - upload: true + upload: false # TODO: flip to true once POC is validated kitmaker-portal-release-cuopt-server-cu12: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -400,7 +400,7 @@ jobs: project_name: cuopt-server-cu12 project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SERVER_CU12 }} wheel_asset_pattern: "^cuopt_server_cu12-" - upload: true + upload: false # TODO: flip to true once POC is validated kitmaker-portal-release-cuopt-server-cu13: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -415,7 +415,7 @@ jobs: project_name: cuopt-server-cu13 project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SERVER_CU13 }} wheel_asset_pattern: "^cuopt_server_cu13-" - upload: true + upload: false # TODO: flip to true once POC is validated kitmaker-portal-release-cuopt-sh-client: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -430,7 +430,7 @@ jobs: project_name: cuopt-sh-client project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SH_CLIENT }} wheel_asset_pattern: "^cuopt_sh_client-" - upload: true + upload: false # TODO: flip to true once POC is validated tests: # Docker image build / tests aren't necessary for the 'test.yaml' workflow, From 64ff00f44ba92f59476b09f2cf684aaf0a3e37a7 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 16:40:06 -0500 Subject: [PATCH 16/25] Add curl retries and Slack failure notification curl --retry 3 --retry-delay 5 on all Charon Ferry / Kitmaker Portal calls, so a single transient network blip doesn't fail the whole job. Adds a Slack notification (reusing CUOPT_SLACK_BOT_TOKEN/CHANNEL_ID/ MENTION_ID, same chat.postMessage pattern as ci/build_summary.sh) on release-mode failure, since nothing previously surfaced a failed Kitmaker publish beyond the Actions UI. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/kitmaker_portal.yaml | 29 +++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 8e6de97bd5..634bf7e186 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -131,7 +131,7 @@ jobs: exit 1 fi - curl --fail-with-body --show-error \ + curl --fail-with-body --show-error --retry 3 --retry-delay 5 \ -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ -H "Authorization: Bearer $PORTAL_AUTHORIZATION" \ "http://127.0.0.1:8888/kitmaker-portal/api/v0/status/$STATUS_ID" @@ -213,7 +213,7 @@ jobs: body=$(jq -n --arg project_name "$PROJECT_NAME" --argjson payload "$PAYLOAD" \ '{project_name: $project_name, payload: $payload}') - http_response=$(curl -sS -w '\n%{http_code}' \ + http_response=$(curl -sS --retry 3 --retry-delay 5 -w '\n%{http_code}' \ -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ -H "Authorization: Bearer $PORTAL_AUTHORIZATION" \ -H "Content-Type: application/json" \ @@ -240,7 +240,7 @@ jobs: STATUS_ID: ${{ steps.kitmaker-release.outputs.status_id }} run: | for _ in $(seq 1 60); do - http_response=$(curl -sS -w '\n%{http_code}' \ + http_response=$(curl -sS --retry 3 --retry-delay 5 -w '\n%{http_code}' \ -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ -H "Authorization: Bearer $PORTAL_AUTHORIZATION" \ "http://127.0.0.1:8888/kitmaker-portal/api/v0/status/$STATUS_ID") @@ -269,3 +269,26 @@ jobs: done echo "Timed out waiting for Kitmaker Portal release to complete" exit 1 + + - name: Notify Slack on failure + if: failure() + env: + SLACK_BOT_TOKEN: ${{ secrets.CUOPT_SLACK_BOT_TOKEN }} + SLACK_CHANNEL_ID: ${{ secrets.CUOPT_SLACK_CHANNEL_ID }} + SLACK_MENTION_ID: ${{ secrets.CUOPT_SLACK_MENTION_ID }} + PROJECT_NAME: ${{ inputs.project_name }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + if [[ -z "$SLACK_BOT_TOKEN" || -z "$SLACK_CHANNEL_ID" ]]; then + echo "SLACK_BOT_TOKEN or SLACK_CHANNEL_ID not set, skipping notification." + exit 0 + fi + + mention="${SLACK_MENTION_ID:+<@${SLACK_MENTION_ID}> }" + text="${mention}Kitmaker Portal release failed for project '${PROJECT_NAME}': ${RUN_URL}" + + curl -sS --max-time 30 -X POST \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + -H "Content-Type: application/json" \ + --data "$(jq -n --arg channel "$SLACK_CHANNEL_ID" --arg text "$text" '{channel: $channel, text: $text}')" \ + "https://slack.com/api/chat.postMessage" From 3b5cd0abbb64cad25fbef0ce699f50f42f5a91e2 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 16:42:42 -0500 Subject: [PATCH 17/25] Support user-group mentions in Kitmaker Slack failure notification Matches the CUOPT_SLACK_MENTION_ID convention already used in ci/utils/generate_slack_payloads.py: an 'S'-prefixed ID is a Slack subteam/user-group and needs , not <@ID> -- the latter silently fails to ping a group. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/kitmaker_portal.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 634bf7e186..c3d2bd97b5 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -284,7 +284,12 @@ jobs: exit 0 fi - mention="${SLACK_MENTION_ID:+<@${SLACK_MENTION_ID}> }" + mention="" + if [[ "$SLACK_MENTION_ID" == S* ]]; then + mention=" " + elif [[ -n "$SLACK_MENTION_ID" ]]; then + mention="<@${SLACK_MENTION_ID}> " + fi text="${mention}Kitmaker Portal release failed for project '${PROJECT_NAME}': ${RUN_URL}" curl -sS --max-time 30 -X POST \ From 54083a0e35b1c41c129f0e1c8cc19df12ea21bbb Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 21 Aug 2026 17:00:12 -0500 Subject: [PATCH 18/25] Remove redundant defaults/permissions blocks - 'defaults: run: shell: bash' was a no-op: ubuntu-latest already defaults run: steps to bash. - Both jobs declared the identical id-token/contents permissions; hoisted to a single workflow-level permissions block. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/kitmaker_portal.yaml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index c3d2bd97b5..338bb5390b 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -81,17 +81,14 @@ on: type: boolean default: false -defaults: - run: - shell: bash +permissions: + id-token: write + contents: read jobs: smoke-test: if: inputs.mode == 'smoke-test' runs-on: ubuntu-latest - permissions: - id-token: write - contents: read env: # Referenced here (unused) so GitHub Actions masks it in this job's logs too, # since a polled status response can echo a release's pic email back. @@ -139,9 +136,6 @@ jobs: release: if: inputs.mode == 'release' runs-on: ubuntu-latest - permissions: - id-token: write - contents: read outputs: status_id: ${{ steps.kitmaker-release.outputs.status_id }} env: From 59c9b4209045f63d3fb158410adf378f0c4a6bd4 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 24 Aug 2026 12:49:18 -0500 Subject: [PATCH 19/25] Fix release-github artifact download pattern Real artifact names from wheels-build.yaml are prefixed cuopt_wheel_* (e.g. cuopt_wheel_cpp_libcuopt_x86_64_cu12), not wheel_*, confirmed against a real build.yaml run's artifact list. The download-artifact pattern matched 0 of 14 artifacts, so release-github created a real GitHub Release with zero wheel assets attached, which then made every kitmaker-portal-release-* job fail at asset collection. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8c7d272bb9..5f5187e77a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -305,7 +305,7 @@ jobs: - name: Download built wheels uses: actions/download-artifact@v4 with: - pattern: "wheel_*" + pattern: "cuopt_wheel_*" path: dist merge-multiple: true - name: Create GitHub Release and upload wheels From 944108e619644efb66a0911d9b347559e400fae2 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 24 Aug 2026 13:32:22 -0500 Subject: [PATCH 20/25] Refresh Ferry OIDC token on every poll iteration 6 of 7 real E2E kitmaker-portal-release-* jobs succeeded; the 7th (libcuopt-cu13, the slowest to process) got HTTP 401 partway through polling after ~7 minutes of successful 'processing' responses. The GitHub OIDC token minted once at job start via core.getIDToken() expired before that wheel finished on Kitmaker's side. Re-mints FERRY_TOKEN on every poll iteration instead, using the same OIDC request endpoint (ACTIONS_ID_TOKEN_REQUEST_URL/_TOKEN, already present in the job's environment given id-token: write) that core.getIDToken() itself calls under the hood. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/kitmaker_portal.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 338bb5390b..9be1cb8d31 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -229,11 +229,17 @@ jobs: - name: Poll Kitmaker Portal status env: - FERRY_TOKEN: ${{ steps.ferry-token.outputs.token }} PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} STATUS_ID: ${{ steps.kitmaker-release.outputs.status_id }} run: | for _ in $(seq 1 60); do + # Re-mint on every iteration: the poll loop can run up to 30 minutes and the + # GitHub OIDC token minted once at job start can expire before a large wheel + # (e.g. libcuopt) finishes processing on Kitmaker's side. + FERRY_TOKEN=$(curl -sS --retry 3 --retry-delay 5 \ + -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=charon.nvidia.com" | jq -r '.value') + http_response=$(curl -sS --retry 3 --retry-delay 5 -w '\n%{http_code}' \ -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ -H "Authorization: Bearer $PORTAL_AUTHORIZATION" \ From b6698e2d210e90c5942bf232659ced06e84bc673 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 24 Aug 2026 14:37:16 -0500 Subject: [PATCH 21/25] Remove PR-testing scaffolding from pr.yaml; clean up stale comments pr.yaml's kitmaker_* workflow_dispatch inputs and kitmaker-portal-test job existed only to test kitmaker_portal.yaml via workflow_call before it existed on main (workflow_dispatch can't target a brand-new file on another branch). Once this merges, kitmaker_portal.yaml is directly dispatchable on its own, so the workaround is dead weight -- reverts pr.yaml to match main exactly. Also rewords the upload: false comments in build.yaml's kitmaker-portal-release-* jobs: the E2E flow is now fully validated against real Kitmaker projects, so 'TODO: flip once POC is validated' is stale. Kept upload: false as a deliberate staged-rollout default pending team sign-off, not a leftover TODO. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yaml | 14 ++++----- .github/workflows/pr.yaml | 60 ------------------------------------ 2 files changed, 7 insertions(+), 67 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5f5187e77a..9ab9648f26 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -340,7 +340,7 @@ jobs: project_name: libcuopt-cu12 project_id: ${{ vars.KITMAKER_PROJECT_ID_LIBCUOPT_CU12 }} wheel_asset_pattern: "^libcuopt_cu12-" - upload: false # TODO: flip to true once POC is validated + upload: false # Staged rollout: flip to true once the team signs off on going live kitmaker-portal-release-libcuopt-cu13: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -355,7 +355,7 @@ jobs: project_name: libcuopt-cu13 project_id: ${{ vars.KITMAKER_PROJECT_ID_LIBCUOPT_CU13 }} wheel_asset_pattern: "^libcuopt_cu13-" - upload: false # TODO: flip to true once POC is validated + upload: false # Staged rollout: flip to true once the team signs off on going live kitmaker-portal-release-cuopt-cu12: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -370,7 +370,7 @@ jobs: project_name: cuopt-cu12 project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_CU12 }} wheel_asset_pattern: "^cuopt_cu12-" - upload: false # TODO: flip to true once POC is validated + upload: false # Staged rollout: flip to true once the team signs off on going live kitmaker-portal-release-cuopt-cu13: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -385,7 +385,7 @@ jobs: project_name: cuopt-cu13 project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_CU13 }} wheel_asset_pattern: "^cuopt_cu13-" - upload: false # TODO: flip to true once POC is validated + upload: false # Staged rollout: flip to true once the team signs off on going live kitmaker-portal-release-cuopt-server-cu12: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -400,7 +400,7 @@ jobs: project_name: cuopt-server-cu12 project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SERVER_CU12 }} wheel_asset_pattern: "^cuopt_server_cu12-" - upload: false # TODO: flip to true once POC is validated + upload: false # Staged rollout: flip to true once the team signs off on going live kitmaker-portal-release-cuopt-server-cu13: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -415,7 +415,7 @@ jobs: project_name: cuopt-server-cu13 project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SERVER_CU13 }} wheel_asset_pattern: "^cuopt_server_cu13-" - upload: false # TODO: flip to true once POC is validated + upload: false # Staged rollout: flip to true once the team signs off on going live kitmaker-portal-release-cuopt-sh-client: if: startsWith(github.ref, 'refs/tags/v') needs: release-github @@ -430,7 +430,7 @@ jobs: project_name: cuopt-sh-client project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SH_CLIENT }} wheel_asset_pattern: "^cuopt_sh_client-" - upload: false # TODO: flip to true once POC is validated + upload: false # Staged rollout: flip to true once the team signs off on going live tests: # Docker image build / tests aren't necessary for the 'test.yaml' workflow, diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f89e3394c8..3169519c0a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -7,43 +7,6 @@ on: push: branches: - "pull-request/[0-9]+" - workflow_dispatch: - inputs: - kitmaker_mode: - description: | - kitmaker_mode: POC entry point for testing kitmaker_portal.yaml without merging it - to main first ('workflow_dispatch' can't target a brand-new file on another branch, - but this already-registered workflow can 'uses:' one via workflow_call). - 'smoke-test' polls 'kitmaker_status_id'. 'release' requests a release using the - kitmaker_* inputs below. - required: true - type: choice - default: smoke-test - options: - - smoke-test - - release - kitmaker_status_id: - description: "kitmaker_status_id: existing Kitmaker Portal status UUID. Required for kitmaker_mode == 'smoke-test'." - type: string - kitmaker_release_tag: - description: "kitmaker_release_tag: git tag whose GitHub Release assets should be published. Required for kitmaker_mode == 'release'." - type: string - kitmaker_project_name: - description: "kitmaker_project_name: Kitmaker project name. Required for kitmaker_mode == 'release'." - type: string - kitmaker_project_id: - description: "kitmaker_project_id: numeric Kitmaker project ID. Required for kitmaker_mode == 'release'." - type: string - kitmaker_wheel_asset_pattern: - description: "kitmaker_wheel_asset_pattern: regex selecting wheel/sdist assets from the release. Required for kitmaker_mode == 'release'." - type: string - kitmaker_pic: - description: "kitmaker_pic: 'person in charge' email required by the Kitmaker release API." - type: string - kitmaker_upload: - description: "kitmaker_upload: 'false' (default) validates only, no real publish. Only used for kitmaker_mode == 'release'." - type: boolean - default: false concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -622,29 +585,6 @@ jobs: with: build_type: pull-request script: ci/test_self_hosted_service.sh - kitmaker-portal-test: - # Manual-only POC call to validate the Charon Ferry -> kitmaker-portal.nvidia.com - # path before wiring the real publish call in build.yaml. Also used as a workaround - # to test kitmaker_portal.yaml before merging it to main: 'workflow_dispatch' can't - # target a workflow file that only exists on a branch, but this already-registered - # workflow can 'uses:' one via workflow_call. Not part of pr-builder's required - # checks since it isn't triggered by normal PR pushes. - if: github.event_name == 'workflow_dispatch' - permissions: - id-token: write - contents: read - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: ./.github/workflows/kitmaker_portal.yaml - with: - mode: ${{ inputs.kitmaker_mode }} - status_id: ${{ inputs.kitmaker_status_id }} - release_tag: ${{ inputs.kitmaker_release_tag }} - project_name: ${{ inputs.kitmaker_project_name }} - project_id: ${{ inputs.kitmaker_project_id }} - wheel_asset_pattern: ${{ inputs.kitmaker_wheel_asset_pattern }} - pic: ${{ inputs.kitmaker_pic }} - upload: ${{ inputs.kitmaker_upload }} - pr-test-summary: name: "PR test summary (non-blocking)" needs: From d030b684921f170b7d60e7fed6fb6d131aa3d8bc Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 24 Aug 2026 14:57:25 -0500 Subject: [PATCH 22/25] Remove unused smoke-test mode from kitmaker_portal.yaml smoke-test (poll an existing status ID) was never wired into any automated job -- only reachable via manual workflow_dispatch, and we decided against building a nightly health-check around it given Charon's refs/tags/v* ref-scoping doesn't cover nightly's main-branch dispatch. With only one mode left, drops the now-unnecessary 'mode' input entirely rather than leaving a single-branch switch. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yaml | 7 -- .github/workflows/kitmaker_portal.yaml | 95 ++++---------------------- 2 files changed, 13 insertions(+), 89 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9ab9648f26..b076dca1e4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -335,7 +335,6 @@ jobs: secrets: inherit # zizmor: ignore[secrets-inherit] uses: ./.github/workflows/kitmaker_portal.yaml with: - mode: release release_tag: ${{ needs.release-github.outputs.release-tag }} project_name: libcuopt-cu12 project_id: ${{ vars.KITMAKER_PROJECT_ID_LIBCUOPT_CU12 }} @@ -350,7 +349,6 @@ jobs: secrets: inherit # zizmor: ignore[secrets-inherit] uses: ./.github/workflows/kitmaker_portal.yaml with: - mode: release release_tag: ${{ needs.release-github.outputs.release-tag }} project_name: libcuopt-cu13 project_id: ${{ vars.KITMAKER_PROJECT_ID_LIBCUOPT_CU13 }} @@ -365,7 +363,6 @@ jobs: secrets: inherit # zizmor: ignore[secrets-inherit] uses: ./.github/workflows/kitmaker_portal.yaml with: - mode: release release_tag: ${{ needs.release-github.outputs.release-tag }} project_name: cuopt-cu12 project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_CU12 }} @@ -380,7 +377,6 @@ jobs: secrets: inherit # zizmor: ignore[secrets-inherit] uses: ./.github/workflows/kitmaker_portal.yaml with: - mode: release release_tag: ${{ needs.release-github.outputs.release-tag }} project_name: cuopt-cu13 project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_CU13 }} @@ -395,7 +391,6 @@ jobs: secrets: inherit # zizmor: ignore[secrets-inherit] uses: ./.github/workflows/kitmaker_portal.yaml with: - mode: release release_tag: ${{ needs.release-github.outputs.release-tag }} project_name: cuopt-server-cu12 project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SERVER_CU12 }} @@ -410,7 +405,6 @@ jobs: secrets: inherit # zizmor: ignore[secrets-inherit] uses: ./.github/workflows/kitmaker_portal.yaml with: - mode: release release_tag: ${{ needs.release-github.outputs.release-tag }} project_name: cuopt-server-cu13 project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SERVER_CU13 }} @@ -425,7 +419,6 @@ jobs: secrets: inherit # zizmor: ignore[secrets-inherit] uses: ./.github/workflows/kitmaker_portal.yaml with: - mode: release release_tag: ${{ needs.release-github.outputs.release-tag }} project_name: cuopt-sh-client project_id: ${{ vars.KITMAKER_PROJECT_ID_CUOPT_SH_CLIENT }} diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 9be1cb8d31..375a8cd651 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -6,29 +6,19 @@ name: Kitmaker Portal (via Charon Ferry) on: workflow_call: inputs: - mode: - description: | - mode: 'smoke-test' polls an existing Kitmaker Portal status ID (no side effects). - 'release' requests a new release for 'release_tag' and polls it to completion. - required: true - type: string - status_id: - description: "status_id: existing Kitmaker Portal status UUID. Required for mode == 'smoke-test'." - type: string release_tag: - description: "release_tag: git tag whose GitHub Release assets should be published to Kitmaker. Required for mode == 'release'." + description: "release_tag: git tag whose GitHub Release assets should be published to Kitmaker." type: string project_name: - description: "project_name: Kitmaker project name. Must exactly match the wheel/component name. Required for mode == 'release'." + description: "project_name: Kitmaker project name. Must exactly match the wheel/component name." type: string project_id: - description: "project_id: numeric Kitmaker project ID (from GET /api/v0/projects). Required for mode == 'release'." + description: "project_id: numeric Kitmaker project ID (from GET /api/v0/projects)." type: string wheel_asset_pattern: description: | wheel_asset_pattern: extended regex (matched with jq's 'test()') selecting this project's wheel/sdist asset names out of the GitHub Release, e.g. '^libcuopt-'. - Required for mode == 'release'. type: string pic: description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL." @@ -37,39 +27,28 @@ on: description: | upload: forwarded to the Kitmaker release API. 'false' (default) runs validation checks only, with no real publish. Must be explicitly set to 'true' to actually - publish wheels. Only used for mode == 'release'. + publish wheels. type: boolean default: false outputs: status_id: - description: Kitmaker Portal status UUID created by mode == 'release'. + description: Kitmaker Portal status UUID created by this release. value: ${{ jobs.release.outputs.status_id }} workflow_dispatch: inputs: - mode: - description: "mode: 'smoke-test' polls an existing status ID. 'release' requests a new release and polls it." - required: true - type: choice - options: - - smoke-test - - release - status_id: - description: "status_id: existing Kitmaker Portal status UUID. Required for mode == 'smoke-test'." - type: string release_tag: - description: "release_tag: git tag whose GitHub Release assets should be published to Kitmaker. Required for mode == 'release'." + description: "release_tag: git tag whose GitHub Release assets should be published to Kitmaker." type: string project_name: - description: "project_name: Kitmaker project name. Must exactly match the registered project. Required for mode == 'release'." + description: "project_name: Kitmaker project name. Must exactly match the registered project." type: string project_id: - description: "project_id: numeric Kitmaker project ID (from GET /api/v0/projects). Required for mode == 'release'." + description: "project_id: numeric Kitmaker project ID (from GET /api/v0/projects)." type: string wheel_asset_pattern: description: | wheel_asset_pattern: extended regex (matched with jq's 'test()') selecting this project's wheel/sdist asset names out of the GitHub Release, e.g. '^libcuopt-'. - Required for mode == 'release'. type: string pic: description: "pic: 'person in charge' email required by the Kitmaker release API. Defaults to secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL." @@ -77,7 +56,7 @@ on: upload: description: | upload: forwarded to the Kitmaker release API. 'false' (default) runs validation - checks only, with no real publish. Only used for mode == 'release'. + checks only, with no real publish. type: boolean default: false @@ -86,55 +65,7 @@ permissions: contents: read jobs: - smoke-test: - if: inputs.mode == 'smoke-test' - runs-on: ubuntu-latest - env: - # Referenced here (unused) so GitHub Actions masks it in this job's logs too, - # since a polled status response can echo a release's pic email back. - PIC: ${{ secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL }} - steps: - - name: Install Teleport - uses: teleport-actions/setup@v1 - with: - version: auto - proxy: nv-stg-ps.teleport.sh:443 - - - name: Start staging Ferry tunnel - uses: teleport-actions/application-tunnel@v1 - with: - proxy: nv-stg-ps.teleport.sh:443 - token: charon-gha-runners - app: charon - listen: tcp://127.0.0.1:8888 - - - name: Mint Ferry identity token - id: ferry-token - uses: actions/github-script@v7 - with: - script: | - const token = await core.getIDToken('charon.nvidia.com'); - core.setSecret(token); - core.setOutput('token', token); - - - name: Poll Kitmaker Portal status - env: - FERRY_TOKEN: ${{ steps.ferry-token.outputs.token }} - PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} - STATUS_ID: ${{ inputs.status_id }} - run: | - if [[ -z "$STATUS_ID" ]]; then - echo "::error::mode == 'smoke-test' requires 'status_id'" - exit 1 - fi - - curl --fail-with-body --show-error --retry 3 --retry-delay 5 \ - -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ - -H "Authorization: Bearer $PORTAL_AUTHORIZATION" \ - "http://127.0.0.1:8888/kitmaker-portal/api/v0/status/$STATUS_ID" - release: - if: inputs.mode == 'release' runs-on: ubuntu-latest outputs: status_id: ${{ steps.kitmaker-release.outputs.status_id }} @@ -147,10 +78,10 @@ jobs: steps: - name: Validate inputs run: | - [[ -n "${{ inputs.release_tag }}" ]] || { echo "::error::mode == 'release' requires 'release_tag'"; exit 1; } - [[ -n "$PROJECT_NAME" ]] || { echo "::error::mode == 'release' requires 'project_name'"; exit 1; } - [[ -n "$PROJECT_ID" ]] || { echo "::error::mode == 'release' requires 'project_id'"; exit 1; } - [[ -n "$ASSET_PATTERN" ]] || { echo "::error::mode == 'release' requires 'wheel_asset_pattern'"; exit 1; } + [[ -n "${{ inputs.release_tag }}" ]] || { echo "::error::'release_tag' is required"; exit 1; } + [[ -n "$PROJECT_NAME" ]] || { echo "::error::'project_name' is required"; exit 1; } + [[ -n "$PROJECT_ID" ]] || { echo "::error::'project_id' is required"; exit 1; } + [[ -n "$ASSET_PATTERN" ]] || { echo "::error::'wheel_asset_pattern' is required"; exit 1; } [[ -n "$PIC" ]] || { echo "::error::'pic' input or secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL must be set"; exit 1; } - name: Collect wheel asset URLs from the GitHub Release From 28a82a3c37e4f43edce4fbd40031d60f747380e0 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 24 Aug 2026 15:01:02 -0500 Subject: [PATCH 23/25] Address CodeRabbit review findings - Declare an explicit workflow_call.secrets contract on kitmaker_portal.yaml (KITMAKER_PORTAL_AUTHORIZATION required; KITMAKER_PORTAL_ACC_OWNER_EMAIL/CUOPT_SLACK_* optional) and pass only those explicitly from build.yaml's 7 callers, instead of 'secrets: inherit' forwarding every caller-visible secret. - Fix release_tag being interpolated raw into Bash source in the 'Validate inputs' step -- this was correctly fixed elsewhere already but got reintroduced when the smoke-test removal rewrite recreated this step. Now sourced via the job-level RELEASE_TAG env var like everywhere else. - Stop interpolating wheel_asset_pattern into the --jq program text (a quote or backslash in the pattern could break or manipulate the jq program); pass it as jq data via --arg instead. - Add --connect-timeout/--max-time to every Charon Ferry / Kitmaker Portal curl call, so a connection that's accepted but never responds can't hang the runner indefinitely. - Harden release_uuid/status extraction with jq -e + non-empty-string checks instead of plain jq -r, so a malformed/null response fails loudly instead of silently becoming the string "null". - Fail release-github immediately if no wheel files were found, instead of silently creating an asset-free GitHub Release that the next job would submit to Kitmaker. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yaml | 53 ++++++++++++++++++++++---- .github/workflows/kitmaker_portal.yaml | 42 +++++++++++++++----- 2 files changed, 79 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b076dca1e4..ef5a22e9dc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -315,6 +315,10 @@ jobs: run: | tag="${GITHUB_REF#refs/tags/}" mapfile -t wheels < <(find dist -name '*.whl') + if [[ ${#wheels[@]} -eq 0 ]]; then + echo "::error::No wheel files found in dist/ -- refusing to create an asset-free release" + exit 1 + fi gh release create "$tag" \ --repo "${{ github.repository }}" \ --title "$tag" \ @@ -332,7 +336,12 @@ jobs: permissions: id-token: write contents: read - secrets: inherit # zizmor: ignore[secrets-inherit] + secrets: + KITMAKER_PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} + KITMAKER_PORTAL_ACC_OWNER_EMAIL: ${{ secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL }} + CUOPT_SLACK_BOT_TOKEN: ${{ secrets.CUOPT_SLACK_BOT_TOKEN }} + CUOPT_SLACK_CHANNEL_ID: ${{ secrets.CUOPT_SLACK_CHANNEL_ID }} + CUOPT_SLACK_MENTION_ID: ${{ secrets.CUOPT_SLACK_MENTION_ID }} uses: ./.github/workflows/kitmaker_portal.yaml with: release_tag: ${{ needs.release-github.outputs.release-tag }} @@ -346,7 +355,12 @@ jobs: permissions: id-token: write contents: read - secrets: inherit # zizmor: ignore[secrets-inherit] + secrets: + KITMAKER_PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} + KITMAKER_PORTAL_ACC_OWNER_EMAIL: ${{ secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL }} + CUOPT_SLACK_BOT_TOKEN: ${{ secrets.CUOPT_SLACK_BOT_TOKEN }} + CUOPT_SLACK_CHANNEL_ID: ${{ secrets.CUOPT_SLACK_CHANNEL_ID }} + CUOPT_SLACK_MENTION_ID: ${{ secrets.CUOPT_SLACK_MENTION_ID }} uses: ./.github/workflows/kitmaker_portal.yaml with: release_tag: ${{ needs.release-github.outputs.release-tag }} @@ -360,7 +374,12 @@ jobs: permissions: id-token: write contents: read - secrets: inherit # zizmor: ignore[secrets-inherit] + secrets: + KITMAKER_PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} + KITMAKER_PORTAL_ACC_OWNER_EMAIL: ${{ secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL }} + CUOPT_SLACK_BOT_TOKEN: ${{ secrets.CUOPT_SLACK_BOT_TOKEN }} + CUOPT_SLACK_CHANNEL_ID: ${{ secrets.CUOPT_SLACK_CHANNEL_ID }} + CUOPT_SLACK_MENTION_ID: ${{ secrets.CUOPT_SLACK_MENTION_ID }} uses: ./.github/workflows/kitmaker_portal.yaml with: release_tag: ${{ needs.release-github.outputs.release-tag }} @@ -374,7 +393,12 @@ jobs: permissions: id-token: write contents: read - secrets: inherit # zizmor: ignore[secrets-inherit] + secrets: + KITMAKER_PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} + KITMAKER_PORTAL_ACC_OWNER_EMAIL: ${{ secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL }} + CUOPT_SLACK_BOT_TOKEN: ${{ secrets.CUOPT_SLACK_BOT_TOKEN }} + CUOPT_SLACK_CHANNEL_ID: ${{ secrets.CUOPT_SLACK_CHANNEL_ID }} + CUOPT_SLACK_MENTION_ID: ${{ secrets.CUOPT_SLACK_MENTION_ID }} uses: ./.github/workflows/kitmaker_portal.yaml with: release_tag: ${{ needs.release-github.outputs.release-tag }} @@ -388,7 +412,12 @@ jobs: permissions: id-token: write contents: read - secrets: inherit # zizmor: ignore[secrets-inherit] + secrets: + KITMAKER_PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} + KITMAKER_PORTAL_ACC_OWNER_EMAIL: ${{ secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL }} + CUOPT_SLACK_BOT_TOKEN: ${{ secrets.CUOPT_SLACK_BOT_TOKEN }} + CUOPT_SLACK_CHANNEL_ID: ${{ secrets.CUOPT_SLACK_CHANNEL_ID }} + CUOPT_SLACK_MENTION_ID: ${{ secrets.CUOPT_SLACK_MENTION_ID }} uses: ./.github/workflows/kitmaker_portal.yaml with: release_tag: ${{ needs.release-github.outputs.release-tag }} @@ -402,7 +431,12 @@ jobs: permissions: id-token: write contents: read - secrets: inherit # zizmor: ignore[secrets-inherit] + secrets: + KITMAKER_PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} + KITMAKER_PORTAL_ACC_OWNER_EMAIL: ${{ secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL }} + CUOPT_SLACK_BOT_TOKEN: ${{ secrets.CUOPT_SLACK_BOT_TOKEN }} + CUOPT_SLACK_CHANNEL_ID: ${{ secrets.CUOPT_SLACK_CHANNEL_ID }} + CUOPT_SLACK_MENTION_ID: ${{ secrets.CUOPT_SLACK_MENTION_ID }} uses: ./.github/workflows/kitmaker_portal.yaml with: release_tag: ${{ needs.release-github.outputs.release-tag }} @@ -416,7 +450,12 @@ jobs: permissions: id-token: write contents: read - secrets: inherit # zizmor: ignore[secrets-inherit] + secrets: + KITMAKER_PORTAL_AUTHORIZATION: ${{ secrets.KITMAKER_PORTAL_AUTHORIZATION }} + KITMAKER_PORTAL_ACC_OWNER_EMAIL: ${{ secrets.KITMAKER_PORTAL_ACC_OWNER_EMAIL }} + CUOPT_SLACK_BOT_TOKEN: ${{ secrets.CUOPT_SLACK_BOT_TOKEN }} + CUOPT_SLACK_CHANNEL_ID: ${{ secrets.CUOPT_SLACK_CHANNEL_ID }} + CUOPT_SLACK_MENTION_ID: ${{ secrets.CUOPT_SLACK_MENTION_ID }} uses: ./.github/workflows/kitmaker_portal.yaml with: release_tag: ${{ needs.release-github.outputs.release-tag }} diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index 375a8cd651..dc32bc8a3d 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -30,6 +30,22 @@ on: publish wheels. type: boolean default: false + secrets: + KITMAKER_PORTAL_AUTHORIZATION: + description: Bearer token (bare, no 'Bearer ' prefix) for the Kitmaker Portal API. + required: true + KITMAKER_PORTAL_ACC_OWNER_EMAIL: + description: Fallback for the 'pic' input. Required if 'pic' isn't passed. + required: false + CUOPT_SLACK_BOT_TOKEN: + description: Slack bot token for the on-failure notification. Optional; skipped if unset. + required: false + CUOPT_SLACK_CHANNEL_ID: + description: Slack channel ID for the on-failure notification. Optional; skipped if unset. + required: false + CUOPT_SLACK_MENTION_ID: + description: Slack user or subteam ID to mention on failure. Optional. + required: false outputs: status_id: description: Kitmaker Portal status UUID created by this release. @@ -71,6 +87,7 @@ jobs: status_id: ${{ steps.kitmaker-release.outputs.status_id }} env: GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.release_tag }} PROJECT_NAME: ${{ inputs.project_name }} PROJECT_ID: ${{ inputs.project_id }} ASSET_PATTERN: ${{ inputs.wheel_asset_pattern }} @@ -78,7 +95,7 @@ jobs: steps: - name: Validate inputs run: | - [[ -n "${{ inputs.release_tag }}" ]] || { echo "::error::'release_tag' is required"; exit 1; } + [[ -n "$RELEASE_TAG" ]] || { echo "::error::'release_tag' is required"; exit 1; } [[ -n "$PROJECT_NAME" ]] || { echo "::error::'project_name' is required"; exit 1; } [[ -n "$PROJECT_ID" ]] || { echo "::error::'project_id' is required"; exit 1; } [[ -n "$ASSET_PATTERN" ]] || { echo "::error::'wheel_asset_pattern' is required"; exit 1; } @@ -87,11 +104,11 @@ jobs: - name: Collect wheel asset URLs from the GitHub Release id: assets env: - RELEASE_TAG: ${{ inputs.release_tag }} UPLOAD: ${{ inputs.upload }} run: | - urls_json=$(gh release view "$RELEASE_TAG" --repo "${{ github.repository }}" --json assets \ - --jq "[.assets[] | select(.name | test(\"$ASSET_PATTERN\")) | .url]") + assets_json=$(gh release view "$RELEASE_TAG" --repo "${{ github.repository }}" --json assets) + urls_json=$(echo "$assets_json" | jq -c --arg pattern "$ASSET_PATTERN" \ + '[.assets[] | select(.name | test($pattern)) | .url]') if [[ "$(echo "$urls_json" | jq 'length')" -eq 0 ]]; then echo "::error::No assets on release '$RELEASE_TAG' matched pattern '$ASSET_PATTERN'" @@ -138,7 +155,7 @@ jobs: body=$(jq -n --arg project_name "$PROJECT_NAME" --argjson payload "$PAYLOAD" \ '{project_name: $project_name, payload: $payload}') - http_response=$(curl -sS --retry 3 --retry-delay 5 -w '\n%{http_code}' \ + http_response=$(curl -sS --retry 3 --retry-delay 5 --connect-timeout 10 --max-time 60 -w '\n%{http_code}' \ -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ -H "Authorization: Bearer $PORTAL_AUTHORIZATION" \ -H "Content-Type: application/json" \ @@ -155,7 +172,10 @@ jobs: exit 1 fi - status_id=$(echo "$response" | jq -r '.release_uuid') + status_id=$(echo "$response" | jq -e -r '.release_uuid | strings | select(length > 0)') || { + echo "::error::Response missing a non-empty 'release_uuid'" + exit 1 + } echo "status_id=$status_id" >> "$GITHUB_OUTPUT" - name: Poll Kitmaker Portal status @@ -167,11 +187,11 @@ jobs: # Re-mint on every iteration: the poll loop can run up to 30 minutes and the # GitHub OIDC token minted once at job start can expire before a large wheel # (e.g. libcuopt) finishes processing on Kitmaker's side. - FERRY_TOKEN=$(curl -sS --retry 3 --retry-delay 5 \ + FERRY_TOKEN=$(curl -sS --retry 3 --retry-delay 5 --connect-timeout 10 --max-time 30 \ -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=charon.nvidia.com" | jq -r '.value') - http_response=$(curl -sS --retry 3 --retry-delay 5 -w '\n%{http_code}' \ + http_response=$(curl -sS --retry 3 --retry-delay 5 --connect-timeout 10 --max-time 60 -w '\n%{http_code}' \ -H "X-Charon-GHA-Token: $FERRY_TOKEN" \ -H "Authorization: Bearer $PORTAL_AUTHORIZATION" \ "http://127.0.0.1:8888/kitmaker-portal/api/v0/status/$STATUS_ID") @@ -184,7 +204,11 @@ jobs: exit 1 fi - status=$(echo "$response" | jq -r '.status') + status=$(echo "$response" | jq -e -r '.status | strings | select(length > 0)') || { + echo "::error::Response missing a non-empty 'status'" + echo "$response" + exit 1 + } echo "Kitmaker Portal release status: $status" case "$status" in completed) From c9cac427793f0496c303645e599144960fa5bec4 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 24 Aug 2026 15:17:02 -0500 Subject: [PATCH 24/25] Pin third-party actions to commit SHAs for zizmor's unpinned-uses check Repo policy requires every 'uses:' reference pinned to a full commit hash, not a mutable tag. Pins actions/download-artifact@v4, teleport-actions/setup@v1, teleport-actions/application-tunnel@v1, and actions/github-script@v7 to their current SHAs (with the resolved version in a trailing comment, matching this repo's existing convention). pre-commit (including zizmor) now passes clean. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yaml | 2 +- .github/workflows/kitmaker_portal.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ef5a22e9dc..d2ce0aff9d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -303,7 +303,7 @@ jobs: release-tag: ${{ steps.release.outputs.tag }} steps: - name: Download built wheels - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: "cuopt_wheel_*" path: dist diff --git a/.github/workflows/kitmaker_portal.yaml b/.github/workflows/kitmaker_portal.yaml index dc32bc8a3d..1b91e11ed2 100644 --- a/.github/workflows/kitmaker_portal.yaml +++ b/.github/workflows/kitmaker_portal.yaml @@ -123,13 +123,13 @@ jobs: echo "payload=$payload" >> "$GITHUB_OUTPUT" - name: Install Teleport - uses: teleport-actions/setup@v1 + uses: teleport-actions/setup@b638ff596557cc3959eb6b5287d5e58e0c8ac6a6 # v1.1.1 with: version: auto proxy: nv-stg-ps.teleport.sh:443 - name: Start staging Ferry tunnel - uses: teleport-actions/application-tunnel@v1 + uses: teleport-actions/application-tunnel@bb7a8fbfb67b85d26013554f10d71dd032c1c764 # v1.0.1 with: proxy: nv-stg-ps.teleport.sh:443 token: charon-gha-runners @@ -138,7 +138,7 @@ jobs: - name: Mint Ferry identity token id: ferry-token - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: script: | const token = await core.getIDToken('charon.nvidia.com'); From 92ce348d1fb78e9b9019834d89b8cb41428dabb3 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 24 Aug 2026 15:18:28 -0500 Subject: [PATCH 25/25] Exclude kitmaker_portal.yaml from pr.yaml's changed-files build/test triggers Same treatment as build.yaml, nightly.yaml, etc.: a CI-infrastructure file that doesn't affect product source, so a PR touching only it shouldn't trigger the full conda/wheel build+test matrix. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/pr.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3169519c0a..ce5a294ffe 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -94,6 +94,7 @@ jobs: - '!.github/workflows/cloud_ci.yaml' - '!.github/workflows/inactivity_reminder.yaml' - '!.github/workflows/issue_automation.yaml' + - '!.github/workflows/kitmaker_portal.yaml' - '!.github/workflows/nightly.yaml' - '!.github/workflows/request-nvskills-ci.yml' - '!.github/workflows/test.yaml' @@ -155,6 +156,7 @@ jobs: - '!.github/workflows/cloud_ci.yaml' - '!.github/workflows/inactivity_reminder.yaml' - '!.github/workflows/issue_automation.yaml' + - '!.github/workflows/kitmaker_portal.yaml' - '!.github/workflows/nightly.yaml' - '!.github/workflows/request-nvskills-ci.yml' - '!.github/workflows/test.yaml' @@ -227,6 +229,7 @@ jobs: - '!.github/workflows/cloud_ci.yaml' - '!.github/workflows/inactivity_reminder.yaml' - '!.github/workflows/issue_automation.yaml' + - '!.github/workflows/kitmaker_portal.yaml' - '!.github/workflows/nightly.yaml' - '!.github/workflows/request-nvskills-ci.yml' - '!.github/workflows/test.yaml' @@ -296,6 +299,7 @@ jobs: - '!.github/workflows/cloud_ci.yaml' - '!.github/workflows/inactivity_reminder.yaml' - '!.github/workflows/issue_automation.yaml' + - '!.github/workflows/kitmaker_portal.yaml' - '!.github/workflows/nightly.yaml' - '!.github/workflows/request-nvskills-ci.yml' - '!.github/workflows/test.yaml'