Skip to content

Commit 228047a

Browse files
ileitchileitch
committed
CI benchmarks
1 parent e683568 commit 228047a

1 file changed

Lines changed: 118 additions & 0 deletions

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
name: Benchmark
2+
on:
3+
issue_comment:
4+
types: [created]
5+
pull_request:
6+
types: [labeled]
7+
8+
jobs:
9+
benchmark:
10+
name: Benchmark
11+
if: |
12+
(github.event_name == 'issue_comment' && github.event.issue.pull_request && contains(github.event.comment.body, '/benchmark')) ||
13+
(github.event_name == 'pull_request' && github.event.label.name == 'benchmark')
14+
runs-on: macos-26
15+
permissions:
16+
contents: read
17+
pull-requests: write
18+
steps:
19+
- name: Get PR number
20+
id: pr-number
21+
run: |
22+
if [ "${{ github.event_name }}" = "issue_comment" ]; then
23+
echo "number=${{ github.event.issue.number }}" >> "$GITHUB_OUTPUT"
24+
else
25+
echo "number=${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT"
26+
fi
27+
28+
- name: Get PR details
29+
id: pr
30+
env:
31+
GH_TOKEN: ${{ github.token }}
32+
run: |
33+
PR_DATA=$(gh api repos/${{ github.repository }}/pulls/${{ steps.pr-number.outputs.number }})
34+
HEAD_SHA=$(echo "$PR_DATA" | jq -r '.head.sha')
35+
HEAD_REF=$(echo "$PR_DATA" | jq -r '.head.ref')
36+
BASE_REF=$(echo "$PR_DATA" | jq -r '.base.ref')
37+
echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
38+
echo "head_ref=$HEAD_REF" >> "$GITHUB_OUTPUT"
39+
echo "base_ref=$BASE_REF" >> "$GITHUB_OUTPUT"
40+
41+
- name: Add reaction to comment
42+
if: github.event_name == 'issue_comment'
43+
env:
44+
GH_TOKEN: ${{ github.token }}
45+
run: |
46+
gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
47+
-f content='+1'
48+
49+
- uses: actions/checkout@master
50+
with:
51+
ref: ${{ steps.pr.outputs.head_sha }}
52+
fetch-depth: 0
53+
54+
- name: Select Xcode version
55+
run: sudo xcode-select -s /Applications/Xcode_26.0.app
56+
57+
- name: Install hyperfine
58+
run: brew install hyperfine
59+
60+
- name: Get merge-base
61+
id: commits
62+
run: |
63+
git fetch origin ${{ steps.pr.outputs.base_ref }}
64+
MERGE_BASE=$(git merge-base HEAD origin/${{ steps.pr.outputs.base_ref }})
65+
echo "merge_base=$MERGE_BASE" >> "$GITHUB_OUTPUT"
66+
echo "Head SHA: ${{ steps.pr.outputs.head_sha }}"
67+
echo "Merge-base SHA: $MERGE_BASE"
68+
69+
- name: Build HEAD (Release)
70+
run: swift build -c release
71+
72+
- name: Benchmark HEAD
73+
run: |
74+
hyperfine --warmup 3 --export-json head-benchmark.json './.build/release/periphery scan --quiet --skip-build'
75+
76+
- name: Checkout and build merge-base
77+
run: |
78+
git checkout ${{ steps.commits.outputs.merge_base }}
79+
swift build -c release
80+
81+
- name: Benchmark merge-base
82+
run: |
83+
hyperfine --warmup 3 --export-json base-benchmark.json './.build/release/periphery scan --quiet --skip-build'
84+
85+
- name: Post results to PR
86+
env:
87+
GH_TOKEN: ${{ github.token }}
88+
run: |
89+
HEAD_MEAN=$(jq '.results[0].mean' head-benchmark.json)
90+
BASE_MEAN=$(jq '.results[0].mean' base-benchmark.json)
91+
HEAD_STDDEV=$(jq '.results[0].stddev' head-benchmark.json)
92+
BASE_STDDEV=$(jq '.results[0].stddev' base-benchmark.json)
93+
94+
CHANGE=$(echo "scale=2; (($HEAD_MEAN - $BASE_MEAN) / $BASE_MEAN) * 100" | bc)
95+
96+
COMMENT="## Benchmark Results
97+
98+
| Commit | Mean (s) | Std Dev (s) |
99+
|--------|----------|-------------|
100+
| Merge-base (\`$(echo ${{ steps.commits.outputs.merge_base }} | cut -c1-7)\`) | $(printf "%.3f" $BASE_MEAN) | ±$(printf "%.3f" $BASE_STDDEV) |
101+
| HEAD (\`$(echo ${{ steps.pr.outputs.head_sha }} | cut -c1-7)\`) | $(printf "%.3f" $HEAD_MEAN) | ±$(printf "%.3f" $HEAD_STDDEV) |
102+
103+
**Change: $(printf "%+.1f%%" $CHANGE)**"
104+
105+
gh pr comment ${{ steps.pr-number.outputs.number }} --body "$COMMENT"
106+
107+
- name: Remove benchmark label
108+
if: github.event_name == 'pull_request'
109+
env:
110+
GH_TOKEN: ${{ github.token }}
111+
run: |
112+
gh pr edit ${{ steps.pr-number.outputs.number }} --remove-label benchmark
113+
114+
- name: Upload benchmark results
115+
uses: actions/upload-artifact@v4
116+
with:
117+
name: benchmark-results
118+
path: "*-benchmark.json"

0 commit comments

Comments
 (0)