Skip to content

Commit bc499ea

Browse files
committed
Integrate hyperlight-ci into CI workflows and Just recipes
- Add cargo alias (`cargo ci`) for convenient hyperlight-ci invocation - Update dep_benchmarks workflow to use `cargo ci bench` and generate a markdown report via `cargo ci bench-report`, posting results as a PR comment per hypervisor/cpu matrix entry - Add benchmarks job to ValidatePullRequest workflow with hypervisor and cpu matrix, gated behind docs-only and build-guests checks - Grant pull-requests: write permission for PR comment posting - Simplify Justfile bench recipes to delegate to `cargo ci bench` - Update benchmarking docs to reflect the new workflow Signed-off-by: Jorge Prendes <jorge.prendes@gmail.com>
1 parent fde3365 commit bc499ea

5 files changed

Lines changed: 191 additions & 6 deletions

File tree

.cargo/config.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[alias] # command aliases
2+
ci = ["run", "--quiet", "--package=hyperlight-ci", "--"]

.github/workflows/ValidatePullRequest.yml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,27 @@ jobs:
140140
docs_only: ${{ needs.docs-pr.outputs.docs-only }}
141141
secrets: inherit
142142

143+
# Run benchmarks and post results as PR comment
144+
benchmarks:
145+
needs:
146+
- docs-pr
147+
- build-guests
148+
# Required because update-guest-locks is skipped on non-dependabot PRs,
149+
# and a skipped dependency transitively skips all downstream jobs.
150+
# See: https://github.com/actions/runner/issues/2205
151+
if: ${{ !cancelled() && !failure() }}
152+
strategy:
153+
fail-fast: false
154+
matrix:
155+
hypervisor: ['hyperv-ws2025', mshv3, kvm]
156+
cpu: [amd, intel]
157+
uses: ./.github/workflows/dep_benchmarks.yml
158+
secrets: inherit
159+
with:
160+
docs_only: ${{ needs.docs-pr.outputs.docs-only }}
161+
hypervisor: ${{ matrix.hypervisor }}
162+
cpu: ${{ matrix.cpu }}
163+
143164
spelling:
144165
name: spell check with typos
145166
runs-on: ubuntu-latest
@@ -167,6 +188,7 @@ jobs:
167188
- build-test
168189
- run-examples
169190
- fuzzing
191+
- benchmarks
170192
- spelling
171193
- license-headers
172194
if: always()

.github/workflows/dep_benchmarks.yml

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ on:
5656
required: false
5757
type: number
5858
default: 5
59-
6059
env:
6160
CARGO_TERM_COLOR: always
6261
RUST_BACKTRACE: full
@@ -133,7 +132,17 @@ jobs:
133132
continue-on-error: true
134133

135134
- name: Run benchmarks
136-
run: just bench-ci main
135+
run: just bench-ci
136+
137+
- name: Create benchmarks report
138+
run: cargo ci bench-report > target/criterion/benchmark.md
139+
140+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
141+
with:
142+
name: benchmark-report_${{ runner.os }}_${{ inputs.hypervisor }}_${{ inputs.cpu }}
143+
path: target/criterion/benchmark.md
144+
if-no-files-found: error
145+
retention-days: 1
137146

138147
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
139148
with:
Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
2+
3+
# Posts combined benchmark results as a single PR comment.
4+
#
5+
# This uses the workflow_run trigger so it runs with default-branch privileges,
6+
# which guarantees write access to pull requests regardless of repository or
7+
# organisation permission settings.
8+
9+
name: Post Benchmark Comment
10+
11+
on:
12+
workflow_run:
13+
workflows: ["Validate Pull Request"]
14+
types: [completed]
15+
16+
permissions:
17+
pull-requests: write
18+
actions: read
19+
20+
jobs:
21+
post-comment:
22+
if: >-
23+
github.event.workflow_run.event == 'pull_request' &&
24+
github.event.workflow_run.conclusion == 'success'
25+
runs-on: ubuntu-latest
26+
steps:
27+
- name: Download benchmark report artifacts
28+
uses: actions/github-script@v9
29+
with:
30+
script: |
31+
const fs = require('fs');
32+
const path = require('path');
33+
34+
// List artifacts from the triggering workflow run
35+
const artifacts = await github.paginate(
36+
github.rest.actions.listWorkflowRunArtifacts,
37+
{
38+
owner: context.repo.owner,
39+
repo: context.repo.repo,
40+
run_id: ${{ github.event.workflow_run.id }},
41+
}
42+
);
43+
44+
const reportArtifacts = artifacts.filter(a => a.name.startsWith('benchmark-report_'));
45+
if (reportArtifacts.length === 0) {
46+
console.log('No benchmark report artifacts found.');
47+
return;
48+
}
49+
50+
fs.mkdirSync('reports', { recursive: true });
51+
52+
for (const artifact of reportArtifacts) {
53+
const zip = await github.rest.actions.downloadArtifact({
54+
owner: context.repo.owner,
55+
repo: context.repo.repo,
56+
artifact_id: artifact.id,
57+
archive_format: 'zip',
58+
});
59+
60+
const dir = path.join('reports', artifact.name);
61+
fs.mkdirSync(dir, { recursive: true });
62+
fs.writeFileSync(path.join(dir, 'artifact.zip'), Buffer.from(zip.data));
63+
}
64+
65+
- name: Extract artifacts
66+
run: |
67+
for dir in reports/benchmark-report_*/; do
68+
if [ -f "$dir/artifact.zip" ]; then
69+
unzip -o "$dir/artifact.zip" -d "$dir"
70+
rm "$dir/artifact.zip"
71+
fi
72+
done
73+
74+
- name: Find PR number
75+
id: pr
76+
uses: actions/github-script@v9
77+
with:
78+
script: |
79+
// Get the PR associated with this workflow run
80+
const run = await github.rest.actions.getWorkflowRun({
81+
owner: context.repo.owner,
82+
repo: context.repo.repo,
83+
run_id: ${{ github.event.workflow_run.id }},
84+
});
85+
86+
// pull_requests array on the run object
87+
const prs = run.data.pull_requests;
88+
if (prs && prs.length > 0) {
89+
core.setOutput('number', prs[0].number);
90+
return;
91+
}
92+
93+
// Fallback: search for a PR matching the head branch/sha
94+
const head_sha = run.data.head_sha;
95+
const pulls = await github.rest.pulls.list({
96+
owner: context.repo.owner,
97+
repo: context.repo.repo,
98+
state: 'open',
99+
head: `${context.repo.owner}:${run.data.head_branch}`,
100+
});
101+
102+
const match = pulls.data.find(p => p.head.sha === head_sha);
103+
if (match) {
104+
core.setOutput('number', match.number);
105+
} else {
106+
core.setOutput('number', '');
107+
console.log('Could not determine PR number, skipping comment.');
108+
}
109+
110+
- name: Post combined benchmark results to PR
111+
if: ${{ steps.pr.outputs.number != '' }}
112+
uses: actions/github-script@v9
113+
with:
114+
script: |
115+
const fs = require('fs');
116+
const path = require('path');
117+
118+
const reportsDir = 'reports';
119+
if (!fs.existsSync(reportsDir)) {
120+
console.log('No benchmark reports found, skipping comment.');
121+
return;
122+
}
123+
124+
// Collect all report files from subdirectories
125+
const sections = [];
126+
const dirs = fs.readdirSync(reportsDir).sort();
127+
for (const dir of dirs) {
128+
const mdPath = path.join(reportsDir, dir, 'benchmark.md');
129+
if (!fs.existsSync(mdPath)) continue;
130+
131+
// Extract hypervisor/cpu from artifact name: benchmark-report_OS_hypervisor_cpu
132+
const parts = dir.replace('benchmark-report_', '').split('_');
133+
const os = parts[0];
134+
const hypervisor = parts.slice(1, -1).join('_');
135+
const cpu = parts[parts.length - 1];
136+
const label = `${hypervisor} / ${cpu} (${os})`;
137+
138+
const content = fs.readFileSync(mdPath, 'utf8').trim();
139+
sections.push(`<details>\n<summary><b>${label}</b></summary>\n\n${content}\n\n</details>`);
140+
}
141+
142+
if (sections.length === 0) {
143+
console.log('No benchmark report content found, skipping comment.');
144+
return;
145+
}
146+
147+
const body = `## Benchmark Results\n\n${sections.join('\n\n')}`;
148+
149+
await github.rest.issues.createComment({
150+
owner: context.repo.owner,
151+
repo: context.repo.repo,
152+
issue_number: Number('${{ steps.pr.outputs.number }}'),
153+
body: body,
154+
});

Justfile

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -401,12 +401,10 @@ bench-download os hypervisor cpu tag="":
401401

402402
# Warning: compares to and then OVERWRITES the given baseline
403403
bench-ci baseline features="":
404-
@# Benchmarks are always run with release builds for meaningful results
405-
cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }}
404+
cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose --save-baseline {{ baseline }}
406405

407406
bench features="":
408-
@# Benchmarks are always run with release builds for meaningful results
409-
cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose
407+
cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose
410408

411409
###############
412410
### FUZZING ###

0 commit comments

Comments
 (0)