diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 795192b2352..8ce33ab2fca 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -753,7 +753,7 @@ jobs: path: ./wheelhouse/*.whl deploy: - name: Deploy + name: Deploy (${{ matrix.group }}) needs: - build-tarball - build-wheels @@ -766,12 +766,54 @@ jobs: contents: write # IMPORTANT: mandatory for making GitHub Releases id-token: write # IMPORTANT: mandatory for trusted publishing & sigstore + # TAG is shared by the two release-existence steps. GITHUB_TOKEN stays scoped + # to the steps that need it rather than job-wide, so third-party actions in + # this job never see it in their environment. + env: + TAG: ${{ github.ref_name }} + + # The required-reviewer pypi environment gates this job, so a human must + # approve before anything is created or published. Release creation and + # publishing all live in this one gated matrix, so a release needs a single + # approval: the groups are pending together and a reviewer approves them in + # one review (see the strategy comment below). environment: name: pypi url: https://pypi.org/p/aiohttp + strategy: + # The PyPI publish and the Sigstore signing each mint one short-lived OIDC + # identity per job and reuse it for every file, so signing the whole dist + # set in a single job can outlast the token and fail partway through + # (pypa/gh-action-pypi-publish#307). Splitting the work across groups, each + # its own job with a fresh identity signing only its share, keeps every + # signing loop well under the token lifetime. + # + # The groups run in parallel and all target the pypi environment, so they + # are pending for approval at the same time and a reviewer approves them in + # a single review rather than one prompt per group. The first group + # (job-index 0) creates the GitHub Release and the others wait for it; each + # group only ever touches its own disjoint share of dists, so the + # concurrent Release asset uploads never collide. fail-fast is off and + # every step is idempotent, so a single failed group can be re-run on its + # own. + # + # Each label "N of M" self-encodes its own position and total, which is the + # only source of truth for the split. Keep `group` the sole matrix axis: an + # include/exclude entry would renumber strategy.job-index / job-total, but + # the label-derived split below stays correct as long as job-index 0 is the + # first label. + fail-fast: false + matrix: + group: + - 1 of 2 + - 2 of 2 + steps: - name: Checkout + # Only the release-creating group needs the repo (create-release reads + # CHANGES.rst and aiohttp/__init__.py); the others only touch dist/. + if: ${{ strategy.job-index == 0 }} uses: actions/checkout@v7 with: submodules: true @@ -784,32 +826,64 @@ jobs: path: dist pattern: dist-* merge-multiple: true - - name: Collected dists + - name: Select this group's distributions + # Keep only this group's share of the dists so the job signs a bounded set. + # index and count come from the "N of M" label, the single source of truth + # for the split; to add a group, extend the matrix list above (e.g. + # "1 of 3" .. "3 of 3"). + # + # The split is fully deterministic: the same built dists always sort the + # same way (LC_ALL=C, byte order, independent of runner locale) and land in + # the same group, so re-running a single failed group reprocesses exactly + # its own share and never touches another group's dists. + id: group + shell: bash + env: + GROUP: ${{ matrix.group }} run: | - tree dist + set -euo pipefail + index=$(( ${GROUP%% of *} - 1 )) + count=${GROUP##* of } + shopt -s nullglob + mapfile -t all < <(printf '%s\n' dist/*.whl dist/*.tar.gz | LC_ALL=C sort) + i=0 + inputs=() + for f in "${all[@]}"; do + if [ "$(( i % count ))" -eq "${index}" ]; then + inputs+=("${f}") + else + rm -f -- "${f}" + fi + i=$(( i + 1 )) + done + echo "Group ${GROUP} keeps ${#inputs[@]} of ${#all[@]} dist(s):" + printf ' %s\n' "${inputs[@]}" + echo "sigstore-inputs=${inputs[*]}" >> "${GITHUB_OUTPUT}" - name: Check whether the GitHub Release already exists - # Allows re-running the deploy job after a partial failure (e.g. PyPI - # upload error) without the Make Release step failing with HTTP 422 - # because the tag/release was created on a prior attempt. Treat - # only the literal `release not found` reply as "does not exist"; - # other failures (auth, rate-limit, network) re-raise so the job - # fails loudly instead of falling through to Make Release. + # The first group owns Release creation. Skipping Make Release when the + # release already exists lets the job be re-run after a partial failure + # without hitting HTTP 422. Query the API and branch on the HTTP status, + # not on prose: a 404 means "create it", any other failure (auth, + # rate-limit, network) re-raises so the job fails loudly. + if: ${{ strategy.job-index == 0 }} id: gh-release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ github.ref_name }} run: | - if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" \ - >/dev/null 2>err; then + set -euo pipefail + if gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" \ + --silent 2>err; then echo 'exists=true' >> "${GITHUB_OUTPUT}" - elif grep -qx 'release not found' err; then + elif grep -q 'HTTP 404' err; then echo 'exists=false' >> "${GITHUB_OUTPUT}" else cat err >&2 exit 1 fi - name: Make Release - if: steps.gh-release.outputs.exists != 'true' + # The first group creates the Release and uploads its share of the + # packages; the other groups add their packages and signatures below. + if: ${{ strategy.job-index == 0 && steps.gh-release.outputs.exists != 'true' }} uses: aio-libs/create-release@v1.6.6 with: changes_file: CHANGES.rst @@ -821,28 +895,52 @@ jobs: :issue:`(\d+)` fix_issue_repl: >- #\1 - - - name: >- - Publish 🐍📦 to PyPI + - name: Wait for the GitHub Release + # The other groups do not create the Release; they wait for the first + # group to create it before they publish or upload anything, so a failure + # to create the Release blocks the irreversible PyPI upload too. Only a + # 404 counts as "not yet"; any other API failure re-raises immediately + # instead of silently retrying for the whole timeout. + if: ${{ strategy.job-index != 0 }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + for _ in $(seq 1 150); do + if gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" \ + --silent 2>err; then + exit 0 + fi + if ! grep -q 'HTTP 404' err; then + cat err >&2 + exit 1 + fi + sleep 2 + done + echo "GitHub Release ${TAG} did not appear in time" >&2 + exit 1 + - name: Publish 🐍📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: - # Allow re-running the deploy job after a partial PyPI upload - # without failing on dists that were already published. + # Allow re-running after a partial PyPI upload without failing on + # dists that a prior attempt already published. skip-existing: true - - name: Sign the dists with Sigstore uses: sigstore/gh-action-sigstore-python@v3.4.0 with: - inputs: >- - ./dist/*.tar.gz - ./dist/*.whl - + inputs: ${{ steps.group.outputs.sigstore-inputs }} - name: Upload artifact signatures to GitHub Release # Confusingly, this action also supports updating releases, not - # just creating them. This is what we want here, since we've manually + # just creating them. This is what we want here, since the first group # created the release above. + # + # The groups run this concurrently against the same release, which is safe: + # each group's files are a disjoint share, so asset names never collide, and + # with no body/name inputs the action preserves the existing release + # metadata (it writes back what it reads) rather than clearing it, so the + # concurrent metadata updates are identical no-ops. The Wait step above + # guarantees the release (with its notes) already exists first. uses: softprops/action-gh-release@v3.0.2 with: - # dist/ contains the built packages, which smoketest-artifacts/ - # contains the signatures and certificates. + # dist/ holds this group's packages plus their Sigstore signatures. files: dist/** diff --git a/CHANGES/13226.contrib.rst b/CHANGES/13226.contrib.rst new file mode 100644 index 00000000000..9b21890e561 --- /dev/null +++ b/CHANGES/13226.contrib.rst @@ -0,0 +1 @@ +Split release publishing and signing across multiple jobs so each stays within the short-lived signing token lifetime, fixing intermittent release upload failures -- by :user:`bdraco`.