Skip to content

Commit c94c91b

Browse files
ileitchileitch
committed
CI benchmarks
1 parent e683568 commit c94c91b

2 files changed

Lines changed: 286 additions & 0 deletions

File tree

‎.github/benchmark-projects.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"projects": [
3+
{
4+
"name": "Alamofire",
5+
"url": "https://github.com/Alamofire/Alamofire.git",
6+
"ref": "5.10.0"
7+
},
8+
{
9+
"name": "stripe-ios",
10+
"url": "https://github.com/stripe/stripe-ios.git",
11+
"ref": "25.3.1"
12+
}
13+
]
14+
}

‎.github/workflows/benchmark.yml‎

Lines changed: 272 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,272 @@
1+
name: Benchmark
2+
on:
3+
issue_comment:
4+
types: [created]
5+
pull_request:
6+
types: [labeled]
7+
8+
jobs:
9+
setup:
10+
name: Setup
11+
if: |
12+
(github.event_name == 'issue_comment' && github.event.issue.pull_request && contains(github.event.comment.body, '/benchmark')) ||
13+
(github.event_name == 'pull_request' && github.event.label.name == 'benchmark')
14+
runs-on: ubuntu-latest
15+
permissions:
16+
contents: read
17+
pull-requests: write
18+
outputs:
19+
matrix: ${{ steps.matrix.outputs.matrix }}
20+
pr_number: ${{ steps.pr-number.outputs.number }}
21+
head_sha: ${{ steps.pr.outputs.head_sha }}
22+
merge_base: ${{ steps.commits.outputs.merge_base }}
23+
steps:
24+
- name: Get PR number
25+
id: pr-number
26+
run: |
27+
if [ "${{ github.event_name }}" = "issue_comment" ]; then
28+
echo "number=${{ github.event.issue.number }}" >> "$GITHUB_OUTPUT"
29+
else
30+
echo "number=${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT"
31+
fi
32+
33+
- name: Get PR details
34+
id: pr
35+
env:
36+
GH_TOKEN: ${{ github.token }}
37+
run: |
38+
PR_DATA=$(gh api repos/${{ github.repository }}/pulls/${{ steps.pr-number.outputs.number }})
39+
HEAD_SHA=$(echo "$PR_DATA" | jq -r '.head.sha')
40+
HEAD_REF=$(echo "$PR_DATA" | jq -r '.head.ref')
41+
BASE_REF=$(echo "$PR_DATA" | jq -r '.base.ref')
42+
echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
43+
echo "head_ref=$HEAD_REF" >> "$GITHUB_OUTPUT"
44+
echo "base_ref=$BASE_REF" >> "$GITHUB_OUTPUT"
45+
46+
- name: Add reaction to comment
47+
if: github.event_name == 'issue_comment'
48+
env:
49+
GH_TOKEN: ${{ github.token }}
50+
run: |
51+
gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
52+
-f content='+1'
53+
54+
- uses: actions/checkout@master
55+
with:
56+
ref: ${{ steps.pr.outputs.head_sha }}
57+
fetch-depth: 0
58+
59+
- name: Get merge-base
60+
id: commits
61+
run: |
62+
git fetch origin ${{ steps.pr.outputs.base_ref }}
63+
MERGE_BASE=$(git merge-base HEAD origin/${{ steps.pr.outputs.base_ref }})
64+
echo "merge_base=$MERGE_BASE" >> "$GITHUB_OUTPUT"
65+
66+
- name: Generate matrix
67+
id: matrix
68+
run: |
69+
# Prepend hardcoded periphery (self) project to the list from config
70+
PERIPHERY='{"name": "periphery", "self": true}'
71+
MATRIX=$(jq -c --argjson periphery "$PERIPHERY" '{project: ([$periphery] + .projects)}' .github/benchmark-projects.json)
72+
echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"
73+
74+
build-head:
75+
name: Build HEAD
76+
needs: setup
77+
runs-on: macos-26
78+
permissions:
79+
contents: read
80+
steps:
81+
- uses: actions/checkout@master
82+
with:
83+
ref: ${{ needs.setup.outputs.head_sha }}
84+
85+
- name: Select Xcode version
86+
run: sudo xcode-select -s /Applications/Xcode_26.2.0.app
87+
88+
- name: Build
89+
run: swift build -c release --product periphery
90+
91+
- name: Upload build
92+
uses: actions/upload-artifact@v4
93+
with:
94+
name: periphery-head
95+
path: .build/release/periphery
96+
97+
build-base:
98+
name: Build merge-base
99+
needs: setup
100+
runs-on: macos-26
101+
permissions:
102+
contents: read
103+
steps:
104+
- uses: actions/checkout@master
105+
with:
106+
ref: ${{ needs.setup.outputs.merge_base }}
107+
108+
- name: Select Xcode version
109+
run: sudo xcode-select -s /Applications/Xcode_26.2.0.app
110+
111+
- name: Build
112+
run: swift build -c release --product periphery
113+
114+
- name: Upload build
115+
uses: actions/upload-artifact@v4
116+
with:
117+
name: periphery-base
118+
path: .build/release/periphery
119+
120+
benchmark:
121+
name: Benchmark (${{ matrix.project.name }})
122+
needs: [setup, build-head, build-base]
123+
runs-on: macos-26
124+
permissions:
125+
contents: read
126+
strategy:
127+
fail-fast: false
128+
matrix: ${{ fromJson(needs.setup.outputs.matrix) }}
129+
steps:
130+
- uses: actions/checkout@master
131+
if: ${{ matrix.project.self }}
132+
with:
133+
ref: ${{ needs.setup.outputs.head_sha }}
134+
fetch-depth: 0
135+
136+
- name: Select Xcode version
137+
run: sudo xcode-select -s /Applications/Xcode_26.2.0.app
138+
139+
- name: Download HEAD build
140+
uses: actions/download-artifact@v4
141+
with:
142+
name: periphery-head
143+
path: .build/release
144+
145+
- name: Make binary executable
146+
run: chmod +x .build/release/periphery
147+
148+
- name: Install hyperfine
149+
run: brew install hyperfine
150+
151+
- name: Clone target project
152+
if: ${{ !matrix.project.self }}
153+
run: |
154+
git clone --depth 1 --branch ${{ matrix.project.ref }} ${{ matrix.project.url }} /tmp/target-project
155+
156+
- name: Build project
157+
working-directory: ${{ matrix.project.self && '.' || '/tmp/target-project' }}
158+
run: swift build
159+
160+
- name: Benchmark HEAD (self)
161+
if: ${{ matrix.project.self }}
162+
run: |
163+
hyperfine --show-output --warmup 3 --export-json head-benchmark.json \
164+
'./.build/release/periphery scan --quiet --skip-build'
165+
166+
- name: Benchmark HEAD (external)
167+
if: ${{ !matrix.project.self }}
168+
working-directory: /tmp/target-project
169+
run: |
170+
hyperfine --show-output --warmup 3 --export-json ${{ github.workspace }}/head-benchmark.json \
171+
'${{ github.workspace }}/.build/release/periphery scan --quiet --skip-build'
172+
173+
- name: Download merge-base build
174+
uses: actions/download-artifact@v4
175+
with:
176+
name: periphery-base
177+
path: .build/release
178+
179+
- name: Make binary executable
180+
run: chmod +x .build/release/periphery
181+
182+
- name: Benchmark merge-base (self)
183+
if: ${{ matrix.project.self }}
184+
run: |
185+
hyperfine --show-output --warmup 3 --export-json base-benchmark.json \
186+
'./.build/release/periphery scan --quiet --skip-build'
187+
188+
- name: Benchmark merge-base (external)
189+
if: ${{ !matrix.project.self }}
190+
working-directory: /tmp/target-project
191+
run: |
192+
hyperfine --show-output --warmup 3 --export-json ${{ github.workspace }}/base-benchmark.json \
193+
'${{ github.workspace }}/.build/release/periphery scan --quiet --skip-build'
194+
195+
- name: Generate result summary
196+
run: |
197+
HEAD_MEAN=$(jq '.results[0].mean' head-benchmark.json)
198+
BASE_MEAN=$(jq '.results[0].mean' base-benchmark.json)
199+
HEAD_STDDEV=$(jq '.results[0].stddev' head-benchmark.json)
200+
BASE_STDDEV=$(jq '.results[0].stddev' base-benchmark.json)
201+
CHANGE=$(echo "scale=2; (($HEAD_MEAN - $BASE_MEAN) / $BASE_MEAN) * 100" | bc)
202+
203+
jq -n \
204+
--arg name "${{ matrix.project.name }}" \
205+
--argjson head_mean "$HEAD_MEAN" \
206+
--argjson base_mean "$BASE_MEAN" \
207+
--argjson head_stddev "$HEAD_STDDEV" \
208+
--argjson base_stddev "$BASE_STDDEV" \
209+
--argjson change "$CHANGE" \
210+
'{name: $name, head_mean: $head_mean, base_mean: $base_mean, head_stddev: $head_stddev, base_stddev: $base_stddev, change: $change}' \
211+
> result-${{ matrix.project.name }}.json
212+
213+
- name: Upload benchmark results
214+
uses: actions/upload-artifact@v4
215+
with:
216+
name: benchmark-${{ matrix.project.name }}
217+
path: |
218+
head-benchmark.json
219+
base-benchmark.json
220+
result-${{ matrix.project.name }}.json
221+
222+
summary:
223+
name: Post Results
224+
needs: [setup, benchmark]
225+
runs-on: ubuntu-latest
226+
permissions:
227+
pull-requests: write
228+
steps:
229+
- name: Download all artifacts
230+
uses: actions/download-artifact@v4
231+
with:
232+
pattern: benchmark-*
233+
merge-multiple: true
234+
235+
- name: Generate comment
236+
id: comment
237+
run: |
238+
echo "## Benchmark Results" > comment.md
239+
echo "" >> comment.md
240+
echo "Comparing \`${{ needs.setup.outputs.merge_base }}\` (merge-base) → \`${{ needs.setup.outputs.head_sha }}\` (HEAD)" >> comment.md
241+
echo "" >> comment.md
242+
echo "| Project | Merge-base (s) | HEAD (s) | Change |" >> comment.md
243+
echo "|---------|----------------|----------|--------|" >> comment.md
244+
245+
for f in result-*.json; do
246+
NAME=$(jq -r '.name' "$f")
247+
BASE_MEAN=$(jq -r '.base_mean' "$f")
248+
HEAD_MEAN=$(jq -r '.head_mean' "$f")
249+
BASE_STDDEV=$(jq -r '.base_stddev' "$f")
250+
HEAD_STDDEV=$(jq -r '.head_stddev' "$f")
251+
CHANGE=$(jq -r '.change' "$f")
252+
253+
printf "| %s | %.3f ±%.3f | %.3f ±%.3f | %+.1f%% |\n" \
254+
"$NAME" "$BASE_MEAN" "$BASE_STDDEV" "$HEAD_MEAN" "$HEAD_STDDEV" "$CHANGE" >> comment.md
255+
done
256+
257+
- name: Post comment
258+
env:
259+
GH_TOKEN: ${{ github.token }}
260+
run: |
261+
gh pr comment ${{ needs.setup.outputs.pr_number }} \
262+
--repo ${{ github.repository }} \
263+
--body-file comment.md
264+
265+
- name: Remove benchmark label
266+
if: github.event_name == 'pull_request'
267+
env:
268+
GH_TOKEN: ${{ github.token }}
269+
run: |
270+
gh pr edit ${{ needs.setup.outputs.pr_number }} \
271+
--repo ${{ github.repository }} \
272+
--remove-label benchmark

0 commit comments

Comments
 (0)