From 3330909cb950aa4072246fb8fe47c4a8a5e64925 Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 16:06:57 +0400 Subject: [PATCH 01/13] modify gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index abc298b..088c37e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ DerivedData/ .netrc Package.resolved -/Scripts/benchmarks +.benchmarkBaselines/ From 211b58d93a3a9833c81825fc5877ab8a177c3dcc Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 16:07:14 +0400 Subject: [PATCH 02/13] add benchmark workflow --- .github/workflows/run-benchmark.yml | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/run-benchmark.yml diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml new file mode 100644 index 0000000..0aed295 --- /dev/null +++ b/.github/workflows/run-benchmark.yml @@ -0,0 +1,45 @@ +name: Run benchmark + +on: + pull_request: + branches: + - main +jobs: + benchmark: + permissions: + contents: read + + runs-on: macos-26 + + steps: + - name: Checkout main + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: base + + - name: Update baseline + run: swift package --allow-writing-to-package-directory benchmark baseline update main + + - name: Upload baseline + uses: actions/upload-artifact@v7 + with: + name: benchmark-baseline + path: base/.benchmarkBaselines + + - name: Checkout current + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: current + - name: Download baseline + uses: actions/download-artifact@v8 + with: + name: benchmark-baseline + path: .benchmarkBaselines + + - name: Compare results + run: swift package benchmark baseline compare main \ + --filter ".*inlinestring16.*" \ + --format markdown \ + --no-progress From d6d80ac82ea030d88863c687609e4c124f571688 Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 16:11:07 +0400 Subject: [PATCH 03/13] remove path on checkout --- .github/workflows/run-benchmark.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index 0aed295..344921b 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -16,7 +16,6 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ github.event.pull_request.base.sha }} - path: base - name: Update baseline run: swift package --allow-writing-to-package-directory benchmark baseline update main @@ -36,7 +35,6 @@ jobs: uses: actions/download-artifact@v8 with: name: benchmark-baseline - path: .benchmarkBaselines - name: Compare results run: swift package benchmark baseline compare main \ From 82604a05de92adc61427ca2be1757d74872c0466 Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 16:17:50 +0400 Subject: [PATCH 04/13] fix issues --- .github/workflows/run-benchmark.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index 344921b..f2e9eec 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -18,23 +18,27 @@ jobs: ref: ${{ github.event.pull_request.base.sha }} - name: Update baseline - run: swift package --allow-writing-to-package-directory benchmark baseline update main + run: swift package --allow-writing-to-package-directory \ + benchmark baseline update main \ + --no-progress - name: Upload baseline uses: actions/upload-artifact@v7 with: name: benchmark-baseline - path: base/.benchmarkBaselines + path: .benchmarkBaselines - name: Checkout current uses: actions/checkout@v7 with: ref: ${{ github.event.pull_request.head.sha }} path: current + - name: Download baseline uses: actions/download-artifact@v8 with: name: benchmark-baseline + path: .benchmarkBaselines - name: Compare results run: swift package benchmark baseline compare main \ From 748f5d40ff02cb27c43b60da6874c6d4c629e0fc Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 16:19:38 +0400 Subject: [PATCH 05/13] fix issues --- .github/workflows/run-benchmark.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index f2e9eec..6f49d6e 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -18,8 +18,7 @@ jobs: ref: ${{ github.event.pull_request.base.sha }} - name: Update baseline - run: swift package --allow-writing-to-package-directory \ - benchmark baseline update main \ + run: swift package --allow-writing-to-package-directory benchmark baseline update main \ --no-progress - name: Upload baseline From bfcf549d98ab1992c72265e7ead7591f0e6b2cec Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 16:22:27 +0400 Subject: [PATCH 06/13] fix issues --- .github/workflows/run-benchmark.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index 6f49d6e..570d4af 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -18,8 +18,7 @@ jobs: ref: ${{ github.event.pull_request.base.sha }} - name: Update baseline - run: swift package --allow-writing-to-package-directory benchmark baseline update main \ - --no-progress + run: swift package --allow-writing-to-package-directory benchmark baseline update main --no-progress - name: Upload baseline uses: actions/upload-artifact@v7 From 250d8e0e2cb1c53bfec70ae89a7eb98efc6edde5 Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 18:17:00 +0400 Subject: [PATCH 07/13] add debug --- .github/workflows/run-benchmark.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index 570d4af..5e90383 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -18,7 +18,17 @@ jobs: ref: ${{ github.event.pull_request.base.sha }} - name: Update baseline - run: swift package --allow-writing-to-package-directory benchmark baseline update main --no-progress + run: swift package --allow-writing-to-package-directory benchmark baseline update main --no-progress + - name: Debug baseline + run: | + echo "PWD:" + pwd + + echo "Directories:" + find . -type d -name "*benchmark*" -o -name ".benchmarkBaselines" + + echo "Files:" + find . -type f -name "results.json" - name: Upload baseline uses: actions/upload-artifact@v7 @@ -30,7 +40,6 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ github.event.pull_request.head.sha }} - path: current - name: Download baseline uses: actions/download-artifact@v8 From 1359cc22c872ffa9092e1481cba993eacbe4b789 Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 18:18:06 +0400 Subject: [PATCH 08/13] add debug --- .github/workflows/run-benchmark.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index 5e90383..c00528f 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -19,14 +19,13 @@ jobs: - name: Update baseline run: swift package --allow-writing-to-package-directory benchmark baseline update main --no-progress + - name: Debug baseline run: | echo "PWD:" pwd - echo "Directories:" find . -type d -name "*benchmark*" -o -name ".benchmarkBaselines" - echo "Files:" find . -type f -name "results.json" From 007000ce653ec1b9579dd3a0b6718132ec549661 Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 18:19:16 +0400 Subject: [PATCH 09/13] add debug --- .github/workflows/run-benchmark.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index c00528f..802908c 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -21,13 +21,13 @@ jobs: run: swift package --allow-writing-to-package-directory benchmark baseline update main --no-progress - name: Debug baseline - run: | - echo "PWD:" - pwd - echo "Directories:" - find . -type d -name "*benchmark*" -o -name ".benchmarkBaselines" - echo "Files:" - find . -type f -name "results.json" + run: | + echo "PWD:" + pwd + echo "Directories:" + find . -type d -name "*benchmark*" -o -name ".benchmarkBaselines" + echo "Files:" + find . -type f -name "results.json" - name: Upload baseline uses: actions/upload-artifact@v7 From 1433e4c75f65be8bd46966097b3130a80524ab20 Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 18:24:17 +0400 Subject: [PATCH 10/13] fix artifact upload --- .github/workflows/run-benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index 802908c..c78a90c 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -34,6 +34,7 @@ jobs: with: name: benchmark-baseline path: .benchmarkBaselines + include-hidden-files: true - name: Checkout current uses: actions/checkout@v7 From 9157867f2fe6260b602f47bd0ecb5ff30a47ea03 Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 18:30:40 +0400 Subject: [PATCH 11/13] fix tiny --- .github/workflows/run-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index c78a90c..f55888f 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -48,7 +48,7 @@ jobs: path: .benchmarkBaselines - name: Compare results - run: swift package benchmark baseline compare main \ + run: | swift package benchmark baseline compare main \ --filter ".*inlinestring16.*" \ --format markdown \ --no-progress From 4d7ae1b38155c7ed9d13f29b804770ffb51b16df Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 18:31:56 +0400 Subject: [PATCH 12/13] fix tiny --- .github/workflows/run-benchmark.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index f55888f..2a3ba97 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -48,7 +48,8 @@ jobs: path: .benchmarkBaselines - name: Compare results - run: | swift package benchmark baseline compare main \ - --filter ".*inlinestring16.*" \ - --format markdown \ - --no-progress + run: | + swift package benchmark baseline compare main \ + --filter ".*inlinestring16.*" \ + --format markdown \ + --no-progress From b0810da9c1aaf3154bf6826f2b94020b93112c9f Mon Sep 17 00:00:00 2001 From: "Ivan A." Date: Fri, 14 Aug 2026 19:03:02 +0400 Subject: [PATCH 13/13] add summary, comment, fail condition --- .github/workflows/run-benchmark.yml | 108 +++++++++++++++++++++++++--- 1 file changed, 98 insertions(+), 10 deletions(-) diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index 2a3ba97..02a480a 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -8,6 +8,7 @@ jobs: benchmark: permissions: contents: read + pull-requests: write runs-on: macos-26 @@ -20,15 +21,6 @@ jobs: - name: Update baseline run: swift package --allow-writing-to-package-directory benchmark baseline update main --no-progress - - name: Debug baseline - run: | - echo "PWD:" - pwd - echo "Directories:" - find . -type d -name "*benchmark*" -o -name ".benchmarkBaselines" - echo "Files:" - find . -type f -name "results.json" - - name: Upload baseline uses: actions/upload-artifact@v7 with: @@ -52,4 +44,100 @@ jobs: swift package benchmark baseline compare main \ --filter ".*inlinestring16.*" \ --format markdown \ - --no-progress + --no-progress | tee comparison.md + + - name: Extract results + run: | + awk ' + /^### .* metrics$/ { + name = $0 + sub(/^### /, "", name) + sub(/ metrics$/, "", name) + } + /^\|[[:space:]]*Improvement %[[:space:]]*\|/ { + split($0, values, "|") + p50 = values[5] + gsub(/^[[:space:]]+|[[:space:]]+$/, "", p50) + printf "%s\t%s%%\n", name, p50 + } + ' comparison.md + + - name: Build benchmark summary + id: benchmark-summary + run: | + awk -v threshold=1.10 ' + BEGIN { + print "## Benchmark summary" > "benchmark-summary.md" + print "" >> "benchmark-summary.md" + print "| Benchmark | p50 |" >> "benchmark-summary.md" + print "|---|---:|" >> "benchmark-summary.md" + + print "## Significant benchmark changes" > "benchmark-comment.md" + print "" >> "benchmark-comment.md" + print "| Benchmark | p50 |" >> "benchmark-comment.md" + print "|---|---:|" >> "benchmark-comment.md" + + significant = 0 + has_regression = 0 + } + + /^### .* metrics$/ { + name = $0 + sub(/^### /, "", name) + sub(/ metrics$/, "", name) + } + + /^\|[[:space:]]*Improvement %[[:space:]]*\|/ { + split($0, values, "|") + + improvement = values[5] + gsub(/^[[:space:]]+|[[:space:]]+$/, "", improvement) + improvement += 0 + + if (improvement > 0) { + ratio = 1 / (1 - improvement / 100) + result = sprintf("%.2fx faster", ratio) + } else if (improvement < 0) { + ratio = 1 - improvement / 100 + result = sprintf("%.2fx slower", ratio) + } else { + ratio = 1 + result = "1.00x" + } + + printf "| `%s` | %s |\n", name, result >> "benchmark-summary.md" + + if (ratio > threshold) { + printf "| `%s` | %s |\n", name, result >> "benchmark-comment.md" + significant = 1 + + if (improvement < 0) { + has_regression = 1 + } + } + } + + END { + if (significant == 0) { + print "" >> "benchmark-comment.md" + print "No significant changes." >> "benchmark-comment.md" + } + + print "has_regression=" has_regression >> ENVIRON["GITHUB_OUTPUT"] + } + ' comparison.md + + - name: Publish summary + run: cat benchmark-summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: Comment PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr comment "${{ github.event.pull_request.number }}" \ + --repo "${{ github.repository }}" \ + --body-file benchmark-comment.md + + - name: Fail on regression + if: steps.benchmark-summary.outputs.has_regression == '1' + run: exit 1