diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 390d6f88ae2..7fde2419e58 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -586,10 +586,14 @@ jobs: OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml) OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" - LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI") + OLD_WHEEL_ARTIFACT_PATTERN="${OLD_BASENAME}[0-9a-f]" + LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id \ + --branch "${OLD_BRANCH}" \ + --artifact "${OLD_WHEEL_ARTIFACT_PATTERN}" \ + NVIDIA/cuda-python "CI") PREV_BINDINGS_DIR="cuda_bindings/dist-prev" - gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python + gh run download "${LATEST_PRIOR_RUN_ID}" -p "${OLD_BASENAME}" -R NVIDIA/cuda-python rm -rf ${OLD_BASENAME}-tests # exclude cython test artifacts ls -al $OLD_BASENAME mkdir -p "${PREV_BINDINGS_DIR}" diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 8134a6844fd..2fd918e2859 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -219,9 +219,18 @@ jobs: OLD_BRANCH=${{ needs.compute-matrix.outputs.OLD_BRANCH }} OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" - LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI") + OLD_WHEEL_ARTIFACT_PATTERN="${OLD_BASENAME}[0-9a-f]" + LOOKUP_ARGS=( + --branch "${OLD_BRANCH}" + --artifact "${OLD_WHEEL_ARTIFACT_PATTERN}" + ) + if ${{ inputs.test-python }}; then + LOOKUP_ARGS+=(--artifact cuda-python-wheel) + fi + LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id \ + "${LOOKUP_ARGS[@]}" NVIDIA/cuda-python "CI") - gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python + gh run download "${LATEST_PRIOR_RUN_ID}" -p "${OLD_BASENAME}" -R NVIDIA/cuda-python rm -rf ${OLD_BASENAME}-tests # exclude cython test artifacts ls -al $OLD_BASENAME mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}" @@ -229,7 +238,7 @@ jobs: rmdir $OLD_BASENAME if ${{ inputs.test-python }}; then - gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python + gh run download "${LATEST_PRIOR_RUN_ID}" -p cuda-python-wheel -R NVIDIA/cuda-python ls -al cuda-python-wheel mv cuda-python-wheel/*.whl . rmdir cuda-python-wheel diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 04b290b1cd0..1af7c4b625b 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -199,9 +199,18 @@ jobs: run: | OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml) OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*" - LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI") + OLD_WHEEL_ARTIFACT_PATTERN="${OLD_BASENAME}[0-9a-f]" + LOOKUP_ARGS=( + --branch "${OLD_BRANCH}" + --artifact "${OLD_WHEEL_ARTIFACT_PATTERN}" + ) + if ${{ inputs.test-python }}; then + LOOKUP_ARGS+=(--artifact cuda-python-wheel) + fi + LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id \ + "${LOOKUP_ARGS[@]}" NVIDIA/cuda-python "CI") - gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python + gh run download "${LATEST_PRIOR_RUN_ID}" -p "${OLD_BASENAME}" -R NVIDIA/cuda-python rm -rf ${OLD_BASENAME}-tests # exclude cython test artifacts ls -al $OLD_BASENAME mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}" @@ -209,7 +218,7 @@ jobs: rmdir $OLD_BASENAME if ${{ inputs.test-python }}; then - gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python + gh run download "${LATEST_PRIOR_RUN_ID}" -p cuda-python-wheel -R NVIDIA/cuda-python ls -al cuda-python-wheel mv cuda-python-wheel/*.whl . rmdir cuda-python-wheel diff --git a/ci/tools/lookup-run-id b/ci/tools/lookup-run-id index bd8ba413974..cc106c4e4f3 100755 --- a/ci/tools/lookup-run-id +++ b/ci/tools/lookup-run-id @@ -8,7 +8,7 @@ # # Two modes: # --tag Find the successful CI run triggered by a tag push. -# --branch Find the latest successful CI run on a branch. +# --branch Find the latest qualifying successful CI run on a branch. # # Outputs the run ID on stdout. All diagnostic messages go to stderr. # When --head-sha is passed, a second line with the run's head SHA is printed. @@ -24,11 +24,14 @@ Usage: Options: --tag Find run by git tag (requires local git repo with the tag) --branch Find latest successful run on the given branch + --artifact Require an unexpired artifact matching this shell glob + (repeatable; branch mode only) --head-sha Also print the run's head commit SHA (second line) Examples: $0 --tag v13.0.1 NVIDIA/cuda-python $0 --branch main NVIDIA/cuda-python + $0 --branch 12.9.x --artifact 'cuda-bindings-python313-*' NVIDIA/cuda-python $0 --branch main --head-sha NVIDIA/cuda-python "CI" EOF exit 1 @@ -38,15 +41,21 @@ EOF MODE="" REF="" HEAD_SHA_FLAG=0 +ARTIFACT_PATTERNS=() while [[ $# -gt 0 ]]; do case "${1}" in --tag) + [[ $# -ge 2 ]] || usage [[ -n "${MODE}" && "${MODE}" != "tag" ]] && { echo "Error: --tag and --branch are mutually exclusive" >&2; exit 1; } MODE="tag"; REF="${2}"; shift 2 ;; --branch) + [[ $# -ge 2 ]] || usage [[ -n "${MODE}" && "${MODE}" != "branch" ]] && { echo "Error: --tag and --branch are mutually exclusive" >&2; exit 1; } MODE="branch"; REF="${2}"; shift 2 ;; + --artifact) + [[ $# -ge 2 ]] || usage + ARTIFACT_PATTERNS+=("${2}"); shift 2 ;; --head-sha) HEAD_SHA_FLAG=1; shift ;; -h|--help) @@ -69,6 +78,11 @@ WORKFLOW_NAME="${1:-CI}" if [[ -z "${REPOSITORY}" ]]; then usage; fi +if [[ "${MODE}" != "branch" && ${#ARTIFACT_PATTERNS[@]} -gt 0 ]]; then + echo "Error: --artifact is only supported with --branch" >&2 + exit 1 +fi + # ── Prerequisite checks ── if [[ -z "${GH_TOKEN:-}" ]]; then echo "Error: GH_TOKEN environment variable is required" >&2 @@ -86,17 +100,79 @@ done if [[ "${MODE}" == "branch" ]]; then echo "Looking up latest successful '${WORKFLOW_NAME}' run on branch: ${REF}" >&2 - RUN_ID=$(gh run list \ - -b "${REF}" \ - -L 1 \ - -w "${WORKFLOW_NAME}" \ - -s success \ - -R "${REPOSITORY}" \ - --json databaseId \ - | jq -r '.[0].databaseId // empty') + RUN_DATA=$(gh run list \ + --repo "${REPOSITORY}" \ + --branch "${REF}" \ + --workflow "${WORKFLOW_NAME}" \ + --status completed \ + --json databaseId,workflowName,status,conclusion,headSha,headBranch,createdAt,url \ + --limit 100) + + CANDIDATE_RUNS=$(echo "${RUN_DATA}" | jq -r \ + --arg branch "${REF}" ' + map(select( + .headBranch == $branch + and .conclusion == "success" + )) + | sort_by(.createdAt, .databaseId) + | reverse + | .[] + | [.databaseId, .headSha, .createdAt] + | @tsv + ') + + if [[ -z "${CANDIDATE_RUNS}" ]]; then + echo "Error: No successful '${WORKFLOW_NAME}' run found on branch '${REF}'" >&2 + exit 1 + fi + + if [[ ${#ARTIFACT_PATTERNS[@]} -gt 0 ]]; then + echo "Requiring unexpired artifacts matching:" >&2 + printf ' - %s\n' "${ARTIFACT_PATTERNS[@]}" >&2 + fi + + RUN_ID="" + HEAD_SHA="" + while IFS= read -r candidate; do + IFS=$'\t' read -r candidate_id candidate_sha candidate_created_at <<< "${candidate}" + + missing_patterns=() + if [[ ${#ARTIFACT_PATTERNS[@]} -gt 0 ]]; then + if ! ARTIFACT_NAMES=$(gh api --paginate \ + "repos/${REPOSITORY}/actions/runs/${candidate_id}/artifacts?per_page=100" \ + --jq '.artifacts[] | select(.expired == false) | .name'); then + echo "Error: Failed to list artifacts for run ${candidate_id}" >&2 + exit 1 + fi + + for pattern in "${ARTIFACT_PATTERNS[@]}"; do + pattern_matched=0 + while IFS= read -r artifact_name; do + # The caller supplies a shell glob, so the RHS must remain unquoted. + # shellcheck disable=SC2053 + if [[ -n "${artifact_name}" && "${artifact_name}" == ${pattern} ]]; then + pattern_matched=1 + break + fi + done <<< "${ARTIFACT_NAMES}" + if [[ ${pattern_matched} == 0 ]]; then + missing_patterns+=("${pattern}") + fi + done + fi + + if [[ ${#missing_patterns[@]} -gt 0 ]]; then + echo "Skipping run ${candidate_id} (${candidate_created_at}); missing unexpired artifact(s): ${missing_patterns[*]}" >&2 + continue + fi + + RUN_ID="${candidate_id}" + HEAD_SHA="${candidate_sha}" + break + done <<< "${CANDIDATE_RUNS}" if [[ -z "${RUN_ID}" ]]; then - echo "Error: No successful '${WORKFLOW_NAME}' run found on branch '${REF}'" >&2 + echo "Error: No successful '${WORKFLOW_NAME}' run on branch '${REF}' has all required artifacts" >&2 exit 1 fi @@ -104,10 +180,10 @@ if [[ "${MODE}" == "branch" ]]; then echo "${RUN_ID}" if [[ "${HEAD_SHA_FLAG}" == 1 ]]; then - HEAD_SHA=$(gh run view "${RUN_ID}" \ - -R "${REPOSITORY}" \ - --json headSha \ - | jq -r '.headSha') + if [[ -z "${HEAD_SHA}" ]]; then + echo "Error: Run ${RUN_ID} has no head SHA" >&2 + exit 1 + fi echo "Head SHA: ${HEAD_SHA}" >&2 echo "${HEAD_SHA}" fi diff --git a/ci/tools/tests/test_lookup_run_id.py b/ci/tools/tests/test_lookup_run_id.py new file mode 100644 index 00000000000..a61b2019a5c --- /dev/null +++ b/ci/tools/tests/test_lookup_run_id.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + +import pytest + +LOOKUP_RUN_ID = Path(__file__).parent.parent / "lookup-run-id" + +FAKE_GH = r"""#!/usr/bin/env python3 +import json +import os +import re +import sys + +args = sys.argv[1:] +if args[:2] == ["run", "list"]: + print(os.environ["FAKE_RUNS"]) + raise SystemExit(0) + +if args[:1] == ["api"]: + if "--paginate" not in args or "--jq" not in args: + print("artifact lookup must be paginated and filtered", file=sys.stderr) + raise SystemExit(2) + match = re.search(r"/runs/(\d+)/artifacts", " ".join(args)) + if match is None: + print("could not determine run ID", file=sys.stderr) + raise SystemExit(2) + artifacts_by_run = json.loads(os.environ["FAKE_ARTIFACTS"]) + artifacts = artifacts_by_run.get(match.group(1)) + if artifacts is None: + print("simulated artifact API failure", file=sys.stderr) + raise SystemExit(3) + for artifact in artifacts: + if not artifact.get("expired", False): + print(artifact["name"]) + raise SystemExit(0) + +print(f"unexpected gh arguments: {args!r}", file=sys.stderr) +raise SystemExit(2) +""" + + +def _run(run_id, created_at, *, branch="12.9.x", workflow="CI", conclusion="success"): + return { + "databaseId": run_id, + "workflowName": workflow, + "status": "completed", + "conclusion": conclusion, + "headSha": f"sha-{run_id}", + "headBranch": branch, + "createdAt": created_at, + "url": f"https://example.invalid/runs/{run_id}", + } + + +@pytest.fixture +def fake_gh(tmp_path): + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + gh = fake_bin / "gh" + gh.write_text(FAKE_GH, encoding="utf-8") + gh.chmod(0o755) + return fake_bin + + +def _lookup(fake_gh, runs, artifacts, *args, workflow="CI"): + env = os.environ.copy() + env.update( + { + "FAKE_ARTIFACTS": json.dumps(artifacts), + "FAKE_RUNS": json.dumps(runs), + "GH_TOKEN": "test-token", + "PATH": f"{fake_gh}{os.pathsep}{env['PATH']}", + } + ) + return subprocess.run( # noqa: S603 - invokes the repository script under test + [str(LOOKUP_RUN_ID), *args, "NVIDIA/cuda-python", workflow], + check=False, + capture_output=True, + env=env, + text=True, + ) + + +@pytest.mark.agent_authored(model="gpt-5.6") +class TestBranchLookup: + def test_selects_newest_run_with_filename_workflow_selector(self, fake_gh): + runs = [ + _run(100, "2026-08-10T12:00:00Z"), + _run(400, "2026-08-13T12:00:00Z", conclusion="failure"), + _run(300, "2026-08-12T12:00:00Z", branch="other"), + _run(200, "2026-08-11T12:00:00Z"), + ] + + result = _lookup( + fake_gh, + runs, + {}, + "--branch", + "12.9.x", + "--head-sha", + workflow="ci.yml", + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.splitlines() == ["200", "sha-200"] + + def test_falls_back_until_all_required_artifacts_are_unexpired(self, fake_gh): + bindings_pattern = "cuda-bindings-python315-cuda*-linux-64*[0-9a-f]" + runs = [ + _run(300, "2026-08-13T12:00:00Z"), + _run(200, "2026-08-12T12:00:00Z"), + _run(100, "2026-08-11T12:00:00Z"), + ] + artifacts = { + "300": [ + { + "name": "cuda-bindings-python315-cuda12.9.1-linux-64-abc123", + "expired": True, + }, + { + "name": "cuda-bindings-python315-cuda12.9.1-linux-64-abc123-tests", + "expired": False, + }, + {"name": "cuda-python-wheel", "expired": False}, + ], + "200": [ + { + "name": "cuda-bindings-python315-cuda12.9.1-linux-64-def456", + "expired": False, + } + ], + "100": [ + { + "name": "cuda-bindings-python315-cuda12.9.1-linux-64-fedcba", + "expired": False, + }, + {"name": "cuda-python-wheel", "expired": False}, + ], + } + + result = _lookup( + fake_gh, + runs, + artifacts, + "--branch", + "12.9.x", + "--artifact", + bindings_pattern, + "--artifact", + "cuda-python-wheel", + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "100" + assert "Skipping run 300" in result.stderr + assert "Skipping run 200" in result.stderr + + def test_reports_when_no_successful_run_has_required_artifacts(self, fake_gh): + runs = [_run(100, "2026-08-11T12:00:00Z")] + artifacts = { + "100": [ + { + "name": "cuda-bindings-python315-cuda12.9.1-linux-64-fedcba", + "expired": True, + } + ] + } + + result = _lookup( + fake_gh, + runs, + artifacts, + "--branch", + "12.9.x", + "--artifact", + "cuda-bindings-python315-cuda*-linux-64*[0-9a-f]", + ) + + assert result.returncode == 1 + assert "has all required artifacts" in result.stderr + + def test_propagates_artifact_api_failures(self, fake_gh): + runs = [_run(100, "2026-08-11T12:00:00Z")] + + result = _lookup( + fake_gh, + runs, + {}, + "--branch", + "12.9.x", + "--artifact", + "cuda-bindings-*", + ) + + assert result.returncode == 1 + assert "Failed to list artifacts for run 100" in result.stderr