diff --git a/.github/scripts/verify_npm_publish.sh b/.github/scripts/verify_npm_publish.sh new file mode 100755 index 0000000000..3c83560f74 --- /dev/null +++ b/.github/scripts/verify_npm_publish.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# verify_npm_publish.sh +# +# Polls the public npm registry until every non-private workspace package is +# retrievable at the version currently declared in its package.json. +# +# This guards the release pipeline against the propagation race where +# `npm publish --workspaces` has returned but the new versions are not yet +# served by registry.npmjs.org. Downstream consumers -- notably the Lambda +# layer build's `npm i @aws-lambda-powertools/@` inside +# `cdk synth` -- would otherwise fail with the misleading error +# `npm error code ETARGET / No matching version found`. +# +# Run from the repository root, after `npm publish --workspaces`. +# See .github/workflows/make-release.yml (publish-npm job). + +set -euo pipefail + +REGISTRY="${NPM_REGISTRY:-https://registry.npmjs.org}" +# Ceiling on the propagation wait; the loop returns as soon as all versions resolve. +TIMEOUT_SECONDS="${VERIFY_TIMEOUT_SECONDS:-420}" +# Delay between polling rounds (seconds). +SLEEP_SECONDS="${VERIFY_SLEEP_SECONDS:-15}" + +# Enumerate every workspace as "@", dropping private workspaces +# (testing-utils, layers, code-snippets, sample-app) which are never published. +mapfile -t packages < <( + npm pkg get name version private --workspaces --json \ + | jq -r 'to_entries[] + | select(.value.private != true) + | "\(.value.name)@\(.value.version)"' +) + +if [ "${#packages[@]}" -eq 0 ]; then + echo "::error::verify_npm_publish: no publishable workspaces found; refusing to continue" + exit 1 +fi + +echo "Waiting for ${#packages[@]} package version(s) to become available on ${REGISTRY}:" +printf ' - %s\n' "${packages[@]}" + +# Returns 0 only when the version document is served (GET 200) AND its +# advertised tarball is downloadable (HEAD 200). `-f` maps any HTTP >= 400 +# (404 while absent, 5xx during an incident) to a non-zero exit. `-S` is +# omitted so the expected 404s while a version propagates don't spam stderr. +is_available() { + local pkg="$1" version="$2" body tarball + body=$(curl -fs --max-time 30 "${REGISTRY}/${pkg}/${version}") || return 1 + tarball=$(printf '%s' "$body" | jq -r '.dist.tarball // empty') + [ -n "$tarball" ] || return 1 + curl -fsI --max-time 30 -o /dev/null "$tarball" +} + +deadline=$(( $(date +%s) + TIMEOUT_SECONDS )) +pending=("${packages[@]}") + +while [ "${#pending[@]}" -gt 0 ]; do + still_pending=() + for entry in "${pending[@]}"; do + # Scoped names begin with '@', so split on the *last* '@' (the version + # separator): "@aws-lambda-powertools/commons@2.35.0" -> name / version. + pkg="${entry%@*}" + version="${entry##*@}" + if is_available "$pkg" "$version"; then + echo " available: ${entry}" + else + still_pending+=("$entry") + fi + done + pending=("${still_pending[@]}") + + [ "${#pending[@]}" -eq 0 ] && break + + now=$(date +%s) + if [ "$now" -ge "$deadline" ]; then + echo "::error::verify_npm_publish: timed out after ${TIMEOUT_SECONDS}s waiting for the following package version(s) to appear on ${REGISTRY}:" + printf '::error:: - %s\n' "${pending[@]}" + echo "::error::These versions were accepted by 'npm publish' but are not yet served by the registry. Continuing would make the Lambda layer build's 'npm i' fail with a misleading 'npm error code ETARGET / No matching version found'. Re-run the release once propagation completes, or check for an npm registry incident at https://status.npmjs.org." + exit 1 + fi + + echo "Still waiting on ${#pending[@]} package(s); retrying in ${SLEEP_SECONDS}s (deadline in $(( deadline - now ))s)..." + sleep "$SLEEP_SECONDS" +done + +echo "All ${#packages[@]} package version(s) are available on ${REGISTRY}." diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml index 57917dd99b..62dee26dff 100644 --- a/.github/workflows/make-release.yml +++ b/.github/workflows/make-release.yml @@ -64,6 +64,13 @@ jobs: run: | VERSION=$(cat packages/commons/package.json | jq .version -r) echo RELEASE_VERSION="$VERSION" >> "$GITHUB_OUTPUT" + # Block until registry.npmjs.org serves every just-published, non-private + # workspace version. Because publish_layer and create_tag both gate on + # `needs: publish-npm`, this stops the layer build's `cdk synth`/`npm i` + # from starting before the versions exist and failing with a misleading + # ETARGET (see issue #5564). + - name: Verify npm registry propagation + run: bash .github/scripts/verify_npm_publish.sh # This job creates a new git tag using the released version (v1.18.1) create_tag: diff --git a/layers/src/layer-publisher-stack.ts b/layers/src/layer-publisher-stack.ts index 7ed00b9227..7e3fe9016b 100644 --- a/layers/src/layer-publisher-stack.ts +++ b/layers/src/layer-publisher-stack.ts @@ -1,5 +1,6 @@ -import { execSync } from 'node:child_process'; +import { execFileSync, execSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; +import { readdirSync } from 'node:fs'; import { dirname, join, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { CfnOutput, RemovalPolicy, Stack, type StackProps } from 'aws-cdk-lib'; @@ -16,6 +17,49 @@ import type { Construct } from 'constructs'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); +const INSTALL_MAX_ATTEMPTS = 5; +const INSTALL_BASE_DELAY_MS = 5_000; + +/** + * Synchronous sleep for CDK's `tryBundle`, which is called synchronously and + * cannot await. Parks the thread on a never-notified `Atomics.wait` lock, so it + * blocks without spinning the CPU or spawning a `sleep` subprocess. + */ +const sleepSync = (ms: number): void => { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +}; + +/** + * Runs `execFileSync`, retrying with exponential backoff so the layer install + * can ride out npm registry propagation lag for freshly published versions (#5564). + */ +const execFileWithRetry = ( + file: string, + args: string[], + maxAttempts: number = INSTALL_MAX_ATTEMPTS +): void => { + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + execFileSync(file, args); + return; + } catch (error) { + if (attempt === maxAttempts) { + console.error( + `Command failed after ${maxAttempts} attempts: ${file} ${args.join(' ')}` + ); + throw error; + } + const delayMs = INSTALL_BASE_DELAY_MS * 2 ** (attempt - 1); + console.warn( + `Command failed (attempt ${attempt}/${maxAttempts}), retrying in ${ + delayMs / 1000 + }s` + ); + sleepSync(delayMs); + } + } +}; + export interface LayerPublisherStackProps extends StackProps { readonly layerName?: string; readonly powertoolsPackageVersion?: string; @@ -132,11 +176,6 @@ export class LayerPublisherStack extends Stack { `mv aws-lambda-powertools-${util}-*.tgz ${tmpBuildDir}` ); } - modulesToInstall.push( - ...utilities.map((util) => - join(tmpBuildDir, `aws-lambda-powertools-${util}-*.tgz`) - ) - ); filesToRemove.push( ...utilities.map((util) => join(`aws-lambda-powertools-${util}-*.tgz`) @@ -166,10 +205,31 @@ export class LayerPublisherStack extends Stack { buildFromLocal && execSync(buildCommands.join(' && '), { cwd: projectRoot }); - // Phase 3: Install dependencies to tmp folder - execSync( - `npm i --prefix ${tmpBuildDir} ${modulesToInstall.join(' ')}` - ); + // Phase 3: Install dependencies to tmp folder. Retry with backoff + // to absorb npm registry propagation delays for freshly published + // versions (see issue #5564). Args are passed to execFileSync as an + // array (no shell), so the local .tgz paths are resolved here rather + // than relying on shell glob expansion. + if (buildFromLocal) { + for (const util of utilities) { + const prefix = `aws-lambda-powertools-${util}-`; + const tarball = readdirSync(tmpBuildDir).find( + (name) => name.startsWith(prefix) && name.endsWith('.tgz') + ); + if (tarball === undefined) { + throw new Error( + `Could not find packed tarball for ${util} in ${tmpBuildDir}` + ); + } + modulesToInstall.push(join(tmpBuildDir, tarball)); + } + } + execFileWithRetry('npm', [ + 'i', + '--prefix', + tmpBuildDir, + ...modulesToInstall, + ]); // Phase 4: Remove unnecessary files execSync(