From c214d105463d9a66859ca9e3baa1abc44613b0af Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Thu, 2 Apr 2026 11:14:07 -0700 Subject: [PATCH 1/5] Add release PR validation and tag automation --- .github/workflows/release-tags.yml | 59 ++++++++++++ .github/workflows/validate-release-pr.yml | 109 ++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 .github/workflows/release-tags.yml create mode 100644 .github/workflows/validate-release-pr.yml diff --git a/.github/workflows/release-tags.yml b/.github/workflows/release-tags.yml new file mode 100644 index 0000000..b23a49f --- /dev/null +++ b/.github/workflows/release-tags.yml @@ -0,0 +1,59 @@ +name: Move release tags + +on: + push: + branches: + - main + paths: + - .release-source.json + +permissions: + contents: write + +jobs: + move-release-tags: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Skip non-release commits + id: release-metadata + run: | + set -euo pipefail + + if [[ ! -f .release-source.json ]]; then + echo "release_commit=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + version="$(jq -r '.version' .release-source.json)" + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid release version in .release-source.json: $version" + exit 1 + fi + + echo "release_commit=true" >> "$GITHUB_OUTPUT" + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "major=${version%%.*}" >> "$GITHUB_OUTPUT" + echo "minor=${version%.*}" >> "$GITHUB_OUTPUT" + + - name: Update floating and exact version tags + if: ${{ steps.release-metadata.outputs.release_commit == 'true' }} + env: + VERSION: ${{ steps.release-metadata.outputs.version }} + MAJOR: ${{ steps.release-metadata.outputs.major }} + MINOR: ${{ steps.release-metadata.outputs.minor }} + run: | + set -euo pipefail + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + git tag -fa "v${VERSION}" -m "Release v${VERSION}" + git tag -fa "v${MINOR}" -m "Release v${MINOR}" + git tag -fa "v${MAJOR}" -m "Release v${MAJOR}" + + git push --force origin "refs/tags/v${VERSION}" "refs/tags/v${MINOR}" "refs/tags/v${MAJOR}" diff --git a/.github/workflows/validate-release-pr.yml b/.github/workflows/validate-release-pr.yml new file mode 100644 index 0000000..bd96fbe --- /dev/null +++ b/.github/workflows/validate-release-pr.yml @@ -0,0 +1,109 @@ +name: Validate release PR + +on: + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + validate-release-pr: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Validate maintenance PR scope + if: ${{ !startsWith(github.head_ref, 'release/code-scan-action-v') }} + run: | + set -euo pipefail + + while IFS= read -r file; do + [[ -z "$file" ]] && continue + if [[ ! "$file" =~ ^\.github/ ]]; then + echo "Maintenance PRs may only change .github/* files. Unexpected file: $file" + exit 1 + fi + done < <(git diff --name-only origin/main...HEAD) + + echo "Branch ${{ github.head_ref }} is not a generated release branch; artifact mirror files are unchanged." + + - name: Validate generated release payload + if: ${{ startsWith(github.head_ref, 'release/code-scan-action-v') }} + env: + HEAD_REF: ${{ github.head_ref }} + run: | + set -euo pipefail + + if [[ ! -f .release-source.json ]]; then + echo "Missing .release-source.json in generated release PR" + exit 1 + fi + + jq -e ' + .repository == "promptfoo/promptfoo" + and .packagePath == "code-scan-action" + and (.sourceSha | test("^[0-9a-f]{40}$")) + and (.sourceTag | type == "string" and length > 0) + and (.version | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) + ' .release-source.json > /dev/null + + version="$(jq -r '.version' .release-source.json)" + source_sha="$(jq -r '.sourceSha' .release-source.json)" + source_tag="$(jq -r '.sourceTag' .release-source.json)" + + if [[ "$HEAD_REF" != "release/code-scan-action-v${version}" ]]; then + echo "Branch name $HEAD_REF does not match release version $version" + exit 1 + fi + + if [[ "$source_tag" != "code-scan-action-${version}" ]]; then + echo "Source tag $source_tag does not match release version $version" + exit 1 + fi + + while IFS= read -r file; do + [[ -z "$file" ]] && continue + if [[ ! "$file" =~ ^(action\.yml|README\.md|CHANGELOG\.md|\.release-source\.json|dist(/.*)?)$ ]]; then + echo "Unexpected generated release file: $file" + exit 1 + fi + done < <(git diff --name-only origin/main...HEAD) + + source_dir="$RUNNER_TEMP/promptfoo-source" + expected_dir="$RUNNER_TEMP/code-scan-action-expected" + + rm -rf "$source_dir" "$expected_dir" + git clone https://github.com/promptfoo/promptfoo.git "$source_dir" + + cd "$source_dir" + git checkout --detach "$source_sha" + git merge-base --is-ancestor "$source_sha" origin/main + + resolved_source_tag_sha="$(git rev-parse "${source_tag}^{commit}")" + if [[ "$resolved_source_tag_sha" != "$source_sha" ]]; then + echo "Source tag $source_tag points to $resolved_source_tag_sha, expected $source_sha" + exit 1 + fi + + npm install -g npm@latest + npm ci + npm ci --prefix code-scan-action + npm run tsc --prefix code-scan-action + npm run build --prefix code-scan-action + + mkdir -p "$expected_dir" + cp code-scan-action/action.yml "$expected_dir/action.yml" + cp code-scan-action/README.md "$expected_dir/README.md" + cp code-scan-action/CHANGELOG.md "$expected_dir/CHANGELOG.md" + cp -R code-scan-action/dist "$expected_dir/dist" + cp "$GITHUB_WORKSPACE/.release-source.json" "$expected_dir/.release-source.json" + + if ! diff -qr --exclude='.git' --exclude='.github' "$expected_dir" "$GITHUB_WORKSPACE"; then + echo "Generated mirror contents do not match promptfoo/promptfoo@$source_sha" + exit 1 + fi From 19d72cfe30f8518dba3be5a26077090b6dfbfee5 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Thu, 2 Apr 2026 11:19:49 -0700 Subject: [PATCH 2/5] Create GitHub releases for mirrored action tags --- .github/workflows/release-tags.yml | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.github/workflows/release-tags.yml b/.github/workflows/release-tags.yml index b23a49f..15c1b79 100644 --- a/.github/workflows/release-tags.yml +++ b/.github/workflows/release-tags.yml @@ -57,3 +57,45 @@ jobs: git tag -fa "v${MAJOR}" -m "Release v${MAJOR}" git push --force origin "refs/tags/v${VERSION}" "refs/tags/v${MINOR}" "refs/tags/v${MAJOR}" + + - name: Create or update GitHub release + if: ${{ steps.release-metadata.outputs.release_commit == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ steps.release-metadata.outputs.version }} + run: | + set -euo pipefail + + notes_file="$RUNNER_TEMP/release-notes.md" + awk -v version="$VERSION" ' + $0 ~ "^## \\[" version "\\]" { + in_section=1 + next + } + in_section && /^## \[/ { + exit + } + in_section { + print + } + ' CHANGELOG.md > "$notes_file" + + if [[ ! -s "$notes_file" ]]; then + printf 'Release v%s\n' "$VERSION" > "$notes_file" + fi + + if gh release view "v${VERSION}" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then + gh release edit "v${VERSION}" \ + --repo "$GITHUB_REPOSITORY" \ + --title "v${VERSION}" \ + --notes-file "$notes_file" \ + --target "$GITHUB_SHA" \ + --verify-tag + else + gh release create "v${VERSION}" \ + --repo "$GITHUB_REPOSITORY" \ + --title "v${VERSION}" \ + --notes-file "$notes_file" \ + --target "$GITHUB_SHA" \ + --verify-tag + fi From 0a285fafa381b5a87671dd02721b7ca2cd9ecb9b Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Thu, 2 Apr 2026 11:31:13 -0700 Subject: [PATCH 3/5] Address mirror release workflow review feedback --- .github/workflows/release-tags.yml | 21 +++++++++++++++++++++ .github/workflows/validate-release-pr.yml | 11 +++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-tags.yml b/.github/workflows/release-tags.yml index 15c1b79..2127f00 100644 --- a/.github/workflows/release-tags.yml +++ b/.github/workflows/release-tags.yml @@ -10,6 +10,10 @@ on: permissions: contents: write +concurrency: + group: release-tags-main + cancel-in-progress: false + jobs: move-release-tags: runs-on: ubuntu-latest @@ -19,8 +23,25 @@ jobs: with: fetch-depth: 0 + - name: Skip stale workflow executions + id: freshness + run: | + set -euo pipefail + + git fetch origin main + current_main_sha="$(git rev-parse origin/main)" + + if [[ "$GITHUB_SHA" != "$current_main_sha" ]]; then + echo "release_commit=false" >> "$GITHUB_OUTPUT" + echo "Skipping stale run for $GITHUB_SHA; origin/main is $current_main_sha" + exit 0 + fi + + echo "release_commit=true" >> "$GITHUB_OUTPUT" + - name: Skip non-release commits id: release-metadata + if: ${{ steps.freshness.outputs.release_commit == 'true' }} run: | set -euo pipefail diff --git a/.github/workflows/validate-release-pr.yml b/.github/workflows/validate-release-pr.yml index bd96fbe..6476bc0 100644 --- a/.github/workflows/validate-release-pr.yml +++ b/.github/workflows/validate-release-pr.yml @@ -103,7 +103,14 @@ jobs: cp -R code-scan-action/dist "$expected_dir/dist" cp "$GITHUB_WORKSPACE/.release-source.json" "$expected_dir/.release-source.json" - if ! diff -qr --exclude='.git' --exclude='.github' "$expected_dir" "$GITHUB_WORKSPACE"; then - echo "Generated mirror contents do not match promptfoo/promptfoo@$source_sha" + for file in action.yml README.md CHANGELOG.md .release-source.json; do + if ! diff -u "$expected_dir/$file" "$GITHUB_WORKSPACE/$file"; then + echo "Generated mirror file $file does not match promptfoo/promptfoo@$source_sha" + exit 1 + fi + done + + if ! diff -qr "$expected_dir/dist" "$GITHUB_WORKSPACE/dist"; then + echo "Generated mirror dist/ does not match promptfoo/promptfoo@$source_sha" exit 1 fi From be1c5cef60610185c9524f64ac9b39f9c187e28f Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Thu, 2 Apr 2026 11:32:59 -0700 Subject: [PATCH 4/5] Recheck main tip before writing release tags --- .github/workflows/release-tags.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/release-tags.yml b/.github/workflows/release-tags.yml index 2127f00..1909b4a 100644 --- a/.github/workflows/release-tags.yml +++ b/.github/workflows/release-tags.yml @@ -77,6 +77,13 @@ jobs: git tag -fa "v${MINOR}" -m "Release v${MINOR}" git tag -fa "v${MAJOR}" -m "Release v${MAJOR}" + git fetch origin main + current_main_sha="$(git rev-parse origin/main)" + if [[ "$GITHUB_SHA" != "$current_main_sha" ]]; then + echo "Skipping stale tag update for $GITHUB_SHA; origin/main is $current_main_sha" + exit 0 + fi + git push --force origin "refs/tags/v${VERSION}" "refs/tags/v${MINOR}" "refs/tags/v${MAJOR}" - name: Create or update GitHub release @@ -105,6 +112,13 @@ jobs: printf 'Release v%s\n' "$VERSION" > "$notes_file" fi + git fetch origin main + current_main_sha="$(git rev-parse origin/main)" + if [[ "$GITHUB_SHA" != "$current_main_sha" ]]; then + echo "Skipping stale release update for $GITHUB_SHA; origin/main is $current_main_sha" + exit 0 + fi + if gh release view "v${VERSION}" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then gh release edit "v${VERSION}" \ --repo "$GITHUB_REPOSITORY" \ From 825e5167d4fb64f5b65cf3d65823b0b775b2062a Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Mon, 13 Apr 2026 12:32:47 -0700 Subject: [PATCH 5/5] Use release metadata to detect stale tag runs --- .github/workflows/release-tags.yml | 34 ++++++++++-------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/.github/workflows/release-tags.yml b/.github/workflows/release-tags.yml index 1909b4a..0e8706e 100644 --- a/.github/workflows/release-tags.yml +++ b/.github/workflows/release-tags.yml @@ -23,25 +23,8 @@ jobs: with: fetch-depth: 0 - - name: Skip stale workflow executions - id: freshness - run: | - set -euo pipefail - - git fetch origin main - current_main_sha="$(git rev-parse origin/main)" - - if [[ "$GITHUB_SHA" != "$current_main_sha" ]]; then - echo "release_commit=false" >> "$GITHUB_OUTPUT" - echo "Skipping stale run for $GITHUB_SHA; origin/main is $current_main_sha" - exit 0 - fi - - echo "release_commit=true" >> "$GITHUB_OUTPUT" - - name: Skip non-release commits id: release-metadata - if: ${{ steps.freshness.outputs.release_commit == 'true' }} run: | set -euo pipefail @@ -58,6 +41,7 @@ jobs: echo "release_commit=true" >> "$GITHUB_OUTPUT" echo "version=$version" >> "$GITHUB_OUTPUT" + echo "source_sha=$(jq -r '.sourceSha' .release-source.json)" >> "$GITHUB_OUTPUT" echo "major=${version%%.*}" >> "$GITHUB_OUTPUT" echo "minor=${version%.*}" >> "$GITHUB_OUTPUT" @@ -65,6 +49,7 @@ jobs: if: ${{ steps.release-metadata.outputs.release_commit == 'true' }} env: VERSION: ${{ steps.release-metadata.outputs.version }} + SOURCE_SHA: ${{ steps.release-metadata.outputs.source_sha }} MAJOR: ${{ steps.release-metadata.outputs.major }} MINOR: ${{ steps.release-metadata.outputs.minor }} run: | @@ -78,9 +63,10 @@ jobs: git tag -fa "v${MAJOR}" -m "Release v${MAJOR}" git fetch origin main - current_main_sha="$(git rev-parse origin/main)" - if [[ "$GITHUB_SHA" != "$current_main_sha" ]]; then - echo "Skipping stale tag update for $GITHUB_SHA; origin/main is $current_main_sha" + main_version="$(git show origin/main:.release-source.json | jq -r '.version')" + main_source_sha="$(git show origin/main:.release-source.json | jq -r '.sourceSha')" + if [[ "$VERSION" != "$main_version" || "$SOURCE_SHA" != "$main_source_sha" ]]; then + echo "Skipping stale tag update for release $VERSION; origin/main now points to $main_version from $main_source_sha" exit 0 fi @@ -91,6 +77,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} VERSION: ${{ steps.release-metadata.outputs.version }} + SOURCE_SHA: ${{ steps.release-metadata.outputs.source_sha }} run: | set -euo pipefail @@ -113,9 +100,10 @@ jobs: fi git fetch origin main - current_main_sha="$(git rev-parse origin/main)" - if [[ "$GITHUB_SHA" != "$current_main_sha" ]]; then - echo "Skipping stale release update for $GITHUB_SHA; origin/main is $current_main_sha" + main_version="$(git show origin/main:.release-source.json | jq -r '.version')" + main_source_sha="$(git show origin/main:.release-source.json | jq -r '.sourceSha')" + if [[ "$VERSION" != "$main_version" || "$SOURCE_SHA" != "$main_source_sha" ]]; then + echo "Skipping stale release update for release $VERSION; origin/main now points to $main_version from $main_source_sha" exit 0 fi