pr-benchmark.yml

# PR Benchmark Workflow
#
# This workflow benchmarks PRs from github.com/fluffylabs/typeberry by:
# 1. Downloading a docker image artifact from a typeberry PR's workflow run
# 2. Running picofuzz tests against that docker image
# 3. Comparing results with baseline statistics from https://typeberry.fluffylabs.dev/
# 4. Posting a comment to the PR with the comparison results
#
# Usage:
#   Trigger manually with workflow_dispatch:
#     - pr_number: The PR number from fluffylabs/typeberry (e.g., "704")
#
#   Or trigger via repository_dispatch with type "benchmark-pr":
#     client_payload:
#       pr_number: "704"
#
# Required Secrets:
#   - TYPEBERRY_PAT: GitHub Personal Access Token with permissions to:
#     * Read artifacts from fluffylabs/typeberry (repo:public_repo or repo:read)
#     * Post comments to PRs in fluffylabs/typeberry (repo:public_repo or issues:write)
#     Note: GITHUB_TOKEN is used as fallback but may not have sufficient permissions
#
name: Benchmark Typeberry PR

on:
  workflow_dispatch:
    inputs:
      pr_number:
        description: 'PR number from fluffylabs/typeberry'
        required: true
        type: string
  repository_dispatch:
    types: [benchmark-pr]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.inputs.pr_number || github.event.client_payload.pr_number }}
  cancel-in-progress: true

jobs:
  setup-docker-image:
    name: Setup Docker Image
    runs-on: self-hosted
    outputs:
      pr_number: ${{ steps.set-vars.outputs.pr_number }}
      workflow_run_id: ${{ steps.find-workflow-run.outputs.workflow_run_id }}
      is_perf_runner: ${{ steps.detect-perf.outputs.is_perf }}
    steps:
      - uses: actions/checkout@v6

      - name: Detect if running on perf hardware
        id: detect-perf
        uses: ./.github/actions/detect-perf

      - name: Set variables
        id: set-vars
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            echo "pr_number=${{ github.event.inputs.pr_number }}" >> $GITHUB_OUTPUT
          else
            echo "pr_number=${{ github.event.client_payload.pr_number }}" >> $GITHUB_OUTPUT
          fi

      - name: Find latest build-docker workflow run
        id: find-workflow-run
        env:
          GH_TOKEN: ${{ secrets.TYPEBERRY_PAT }}
          PR_NUMBER: ${{ steps.set-vars.outputs.pr_number }}
        run: |
          # Get PR details to find the head branch
          PR_DATA=$(curl -s \
            -H "Authorization: token $GH_TOKEN" \
            -H "Accept: application/vnd.github.v3+json" \
            "https://api.github.com/repos/fluffylabs/typeberry/pulls/$PR_NUMBER")

          HEAD_BRANCH=$(echo "$PR_DATA" | jq -r '.head.ref')
          echo "PR #$PR_NUMBER branch: $HEAD_BRANCH"

          # Get the workflow ID for build-docker
          WORKFLOW_ID=$(curl -s \
            -H "Authorization: token $GH_TOKEN" \
            -H "Accept: application/vnd.github.v3+json" \
            "https://api.github.com/repos/fluffylabs/typeberry/actions/workflows" \
            | jq -r '.workflows[] | select(.name == "Build - Docker" or (.path | endswith("build-docker.yml"))) | .id' \
            | head -n 1)

          if [ -z "$WORKFLOW_ID" ]; then
            echo "Error: Could not find build-docker workflow"
            exit 1
          fi

          echo "build-docker workflow ID: $WORKFLOW_ID"

          # Find the latest successful workflow run for this branch with artifacts
          WORKFLOW_RUN_ID=$(curl -s \
            -H "Authorization: token $GH_TOKEN" \
            -H "Accept: application/vnd.github.v3+json" \
            "https://api.github.com/repos/fluffylabs/typeberry/actions/workflows/$WORKFLOW_ID/runs?branch=$HEAD_BRANCH&status=success&per_page=10" \
            | jq -r '.workflow_runs[] | select(.artifacts_url != null) | .id' \
            | head -n 1)

          if [ -z "$WORKFLOW_RUN_ID" ]; then
            echo "Error: Could not find successful build-docker workflow run for branch $HEAD_BRANCH"
            echo "Please ensure the build-docker workflow has completed successfully for this PR"
            exit 1
          fi

          echo "Found workflow run ID: $WORKFLOW_RUN_ID"
          echo "workflow_run_id=$WORKFLOW_RUN_ID" >> $GITHUB_OUTPUT

      - name: Download docker image artifact
        uses: actions/download-artifact@v8
        with:
          name: typeberry-docker-image
          path: ./docker-image
          github-token: ${{ secrets.TYPEBERRY_PAT || secrets.GITHUB_TOKEN }}
          repository: fluffylabs/typeberry
          run-id: ${{ steps.find-workflow-run.outputs.workflow_run_id }}

      - name: Load docker image
        run: |
          gunzip ./docker-image/typeberry-image.tar.gz
          docker load -i ./docker-image/typeberry-image.tar
          # Tag the image as latest so existing tests work
          IMAGE_ID=$(docker images --format "{{.ID}}" typeberry | head -n 1)
          if [ -z "$IMAGE_ID" ]; then
            echo "Error: No typeberry image found after loading"
            exit 1
          fi
          docker tag $IMAGE_ID ghcr.io/fluffylabs/typeberry:latest
          docker images | grep typeberry

  benchmark:
    name: Run Benchmark Tests
    needs: setup-docker-image
    runs-on: self-hosted
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: 'recursive'

      - name: Detect if running on perf hardware
        id: detect-perf
        uses: ./.github/actions/detect-perf

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: '24'
          cache: 'npm'

      - name: Run benchmark tests and generate report
        run: npm start -w picofuzz-benchmark

      - name: Upload benchmark report
        uses: actions/upload-artifact@v7
        with:
          name: benchmark-report
          path: ./benchmark-report.md
        if: always()

      - name: Upload CSV results
        uses: actions/upload-artifact@v7
        with:
          name: picofuzz-results
          path: ./picofuzz-result/*.csv
          retention-days: 90
        # Only upload results from perf hardware for aggregation
        if: always() && steps.detect-perf.outputs.is_perf == 'true'

  report:
    name: Post Results to PR
    needs: [benchmark, setup-docker-image]
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Generate GitHub App token
        id: generate-token
        uses: actions/create-github-app-token@v3
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          owner: fluffylabs
          repositories: typeberry

      - name: Download benchmark report
        uses: actions/download-artifact@v8
        with:
          name: benchmark-report

      - name: Post or update comment on PR
        uses: actions/github-script@v9
        with:
          github-token: ${{ steps.generate-token.outputs.token }}
          script: |
            const fs = require('fs');
            const report = fs.readFileSync('benchmark-report.md', 'utf8');

            // Add a marker to identify our comments
            const marker = '<!-- typeberry-benchmark-report -->';
            const body = marker + '\n' + report;

            const owner = 'fluffylabs';
            const repo = 'typeberry';
            const issue_number = ${{ needs.setup-docker-image.outputs.pr_number }};

            // Find existing comment
            const { data: comments } = await github.rest.issues.listComments({
              owner,
              repo,
              issue_number,
            });

            const existingComment = comments.find(comment =>
              comment.body && comment.body.includes(marker)
            );

            if (existingComment) {
              // Update existing comment
              await github.rest.issues.updateComment({
                owner,
                repo,
                comment_id: existingComment.id,
                body
              });
              console.log(`Updated existing comment: ${existingComment.html_url}`);
            } else {
              // Create new comment
              const { data: newComment } = await github.rest.issues.createComment({
                owner,
                repo,
                issue_number,
                body
              });
              console.log(`Created new comment: ${newComment.html_url}`);
            }