diff --git a/.github/ltp/ltp-expected.txt b/.github/ltp/ltp-expected.txt new file mode 100644 index 0000000000000..829dd5d5033ce --- /dev/null +++ b/.github/ltp/ltp-expected.txt @@ -0,0 +1,13 @@ +# LTP Expected Results +# +# Format: test_name PASS|FAIL|CONF +# +# Tests listed as FAIL here are known broken and will NOT block CI. +# Tests listed as CONF are expected to be skipped due to kernel config. +# Any test not listed that regresses from PASS->FAIL will block PR creation. +# +# Add entries here when you intentionally accept a known failure: +# broken_test FAIL # upstream LTP bug #1234, fix pending +# +# This file starts empty — populate it as you learn which tests are +# flaky or permanently broken in this kernel/QEMU environment. diff --git a/.github/scripts/create-pr-body-multiarch.sh b/.github/scripts/create-pr-body-multiarch.sh index 616228bb7477c..353b98ebce212 100755 --- a/.github/scripts/create-pr-body-multiarch.sh +++ b/.github/scripts/create-pr-body-multiarch.sh @@ -1,127 +1,103 @@ #!/bin/bash # Script to create PR body using named arguments -# Usage: create-pr-body.sh --arch ARCH --build-time TIME --total-time TIME --passed N --failed N [--arch ...] --run-id ID --comparison SECTION --repo REPO [--commit-file FILE] +# Usage: create-pr-body.sh --arch ARCH --build-time TIME --total-time TIME +# [--kselftest-passed N --kselftest-failed N --kselftest-status TEXT] +# [--ltp-passed N --ltp-failed N --ltp-status TEXT] +# [--arch ...] --run-id ID --repo REPO --compared-against BRANCH +# [--ltp-details TEXT] [--commit-file FILE] set -euo pipefail -# Arrays to track architectures and their data declare -a ARCHS=() declare -A ARCH_DATA -# Global parameters RUN_ID="" -COMPARISON_SECTION="" REPO="" COMMIT_MESSAGE_FILE="" +COMPARED_AGAINST="" +LTP_DETAILS="" CURRENT_ARCH="" while [[ $# -gt 0 ]]; do case "$1" in --arch) - [[ $# -lt 2 ]] && { echo "Error: --arch requires a value" >&2; exit 1; } CURRENT_ARCH="$2" - # Only add to ARCHS array if not already present if [[ ! " ${ARCHS[@]:-} " =~ " ${CURRENT_ARCH} " ]]; then ARCHS+=("$CURRENT_ARCH") fi shift 2 ;; --build-time) - [[ $# -lt 2 ]] && { echo "Error: --build-time requires a value" >&2; exit 1; } - [[ -z "$CURRENT_ARCH" ]] && { echo "Error: --arch must be specified before --build-time" >&2; exit 1; } - ARCH_DATA["${CURRENT_ARCH}_build_time"]="$2" - shift 2 - ;; + ARCH_DATA["${CURRENT_ARCH}_build_time"]="$2"; shift 2 ;; --total-time) - [[ $# -lt 2 ]] && { echo "Error: --total-time requires a value" >&2; exit 1; } - [[ -z "$CURRENT_ARCH" ]] && { echo "Error: --arch must be specified before --total-time" >&2; exit 1; } - ARCH_DATA["${CURRENT_ARCH}_total_time"]="$2" - shift 2 - ;; - --passed) - [[ $# -lt 2 ]] && { echo "Error: --passed requires a value" >&2; exit 1; } - [[ -z "$CURRENT_ARCH" ]] && { echo "Error: --arch must be specified before --passed" >&2; exit 1; } - ARCH_DATA["${CURRENT_ARCH}_passed"]="$2" - shift 2 - ;; - --failed) - [[ $# -lt 2 ]] && { echo "Error: --failed requires a value" >&2; exit 1; } - [[ -z "$CURRENT_ARCH" ]] && { echo "Error: --arch must be specified before --failed" >&2; exit 1; } - ARCH_DATA["${CURRENT_ARCH}_failed"]="$2" - shift 2 - ;; + ARCH_DATA["${CURRENT_ARCH}_total_time"]="$2"; shift 2 ;; + --kselftest-passed) + ARCH_DATA["${CURRENT_ARCH}_kselftest_passed"]="$2"; shift 2 ;; + --kselftest-failed) + ARCH_DATA["${CURRENT_ARCH}_kselftest_failed"]="$2"; shift 2 ;; + --kselftest-status) + ARCH_DATA["${CURRENT_ARCH}_kselftest_status"]="$2"; shift 2 ;; + --ltp-passed) + ARCH_DATA["${CURRENT_ARCH}_ltp_passed"]="$2"; shift 2 ;; + --ltp-failed) + ARCH_DATA["${CURRENT_ARCH}_ltp_failed"]="$2"; shift 2 ;; + --ltp-status) + ARCH_DATA["${CURRENT_ARCH}_ltp_status"]="$2"; shift 2 ;; --run-id) - [[ $# -lt 2 ]] && { echo "Error: --run-id requires a value" >&2; exit 1; } - RUN_ID="$2" - shift 2 - ;; - --comparison) - [[ $# -lt 2 ]] && { echo "Error: --comparison requires a value" >&2; exit 1; } - COMPARISON_SECTION="$2" - shift 2 - ;; + RUN_ID="$2"; shift 2 ;; --repo) - [[ $# -lt 2 ]] && { echo "Error: --repo requires a value" >&2; exit 1; } - REPO="$2" - shift 2 - ;; + REPO="$2"; shift 2 ;; + --compared-against) + COMPARED_AGAINST="$2"; shift 2 ;; + --ltp-details) + LTP_DETAILS="$2"; shift 2 ;; --commit-file) - [[ $# -lt 2 ]] && { echo "Error: --commit-file requires a value" >&2; exit 1; } - COMMIT_MESSAGE_FILE="$2" - shift 2 - ;; + COMMIT_MESSAGE_FILE="$2"; shift 2 ;; *) - echo "Error: Unknown option: $1" >&2 - echo "Usage: $0 --arch ARCH --build-time TIME --total-time TIME --passed N --failed N [--arch ...] --run-id ID --comparison SECTION --repo REPO [--commit-file FILE]" >&2 - exit 1 - ;; + echo "Error: Unknown option: $1" >&2; exit 1 ;; esac done -# Validate required parameters [[ ${#ARCHS[@]} -eq 0 ]] && { echo "Error: At least one --arch required" >&2; exit 1; } [[ -z "$RUN_ID" ]] && { echo "Error: --run-id required" >&2; exit 1; } -[[ -z "$COMPARISON_SECTION" ]] && { echo "Error: --comparison required" >&2; exit 1; } [[ -z "$REPO" ]] && { echo "Error: --repo required" >&2; exit 1; } [[ -z "$COMMIT_MESSAGE_FILE" ]] && COMMIT_MESSAGE_FILE="/tmp/commit_message.txt" -# Check if commit message file exists if [ ! -f "$COMMIT_MESSAGE_FILE" ]; then echo "Error: Commit message file not found: $COMMIT_MESSAGE_FILE" >&2 exit 1 fi -# Validate each arch has all required data for arch in "${ARCHS[@]}"; do [[ -z "${ARCH_DATA[${arch}_build_time]:-}" ]] && { echo "Error: Missing --build-time for $arch" >&2; exit 1; } [[ -z "${ARCH_DATA[${arch}_total_time]:-}" ]] && { echo "Error: Missing --total-time for $arch" >&2; exit 1; } - [[ -z "${ARCH_DATA[${arch}_passed]:-}" ]] && { echo "Error: Missing --passed for $arch" >&2; exit 1; } - [[ -z "${ARCH_DATA[${arch}_failed]:-}" ]] && { echo "Error: Missing --failed for $arch" >&2; exit 1; } done -# Convert seconds to minutes for better readability convert_time() { - local seconds="${1%s}" # Remove 's' suffix if present + local seconds="${1%s}" local minutes=$((seconds / 60)) local remaining_seconds=$((seconds % 60)) echo "${minutes}m ${remaining_seconds}s" } -# Determine if multi-arch MULTIARCH=false -if [ ${#ARCHS[@]} -gt 1 ]; then - MULTIARCH=true -fi +[ ${#ARCHS[@]} -gt 1 ] && MULTIARCH=true -# Convert times for all architectures for arch in "${ARCHS[@]}"; do ARCH_DATA["${arch}_build_time_readable"]=$(convert_time "${ARCH_DATA[${arch}_build_time]}") ARCH_DATA["${arch}_total_time_readable"]=$(convert_time "${ARCH_DATA[${arch}_total_time]}") done -# Generate PR body +# Check if any arch has kselftest or LTP data +HAS_KSELFTEST=false +HAS_LTP=false +for arch in "${ARCHS[@]}"; do + [ -n "${ARCH_DATA[${arch}_kselftest_passed]:-}" ] && HAS_KSELFTEST=true + [ -n "${ARCH_DATA[${arch}_ltp_passed]:-}" ] && HAS_LTP=true +done + cat << EOF ## Summary This PR has been automatically created after successful completion of all CI stages. @@ -140,71 +116,78 @@ cat << EOF ### ✅ Build Stage EOF -# Build Stage - conditional formatting if [ "$MULTIARCH" = true ]; then - cat << EOF - -| Architecture | Build Time | Total Time | -|--------------|------------|------------| -EOF + echo "" + echo "| Architecture | Build Time | Total Time |" + echo "|--------------|------------|------------|" for arch in "${ARCHS[@]}"; do echo "| ${arch} | ${ARCH_DATA[${arch}_build_time_readable]} | ${ARCH_DATA[${arch}_total_time_readable]} |" done else ARCH1="${ARCHS[0]}" - cat << EOF -- Status: Passed (${ARCH1}) -- Build Time: ${ARCH_DATA[${ARCH1}_build_time_readable]} -- Total Time: ${ARCH_DATA[${ARCH1}_total_time_readable]} -EOF + echo "- Status: Passed (${ARCH1})" + echo "- Build Time: ${ARCH_DATA[${ARCH1}_build_time_readable]}" + echo "- Total Time: ${ARCH_DATA[${ARCH1}_total_time_readable]}" fi -cat << EOF +echo "" +echo "- [View build logs](https://github.com/${REPO}/actions/runs/${RUN_ID})" -- [View build logs](https://github.com/${REPO}/actions/runs/${RUN_ID}) +cat << EOF ### ✅ Boot Verification EOF -# Boot Verification - conditional formatting if [ "$MULTIARCH" = true ]; then echo "- Status: Passed (all architectures)" else echo "- Status: Passed (${ARCHS[0]})" fi +echo "- [View boot logs](https://github.com/${REPO}/actions/runs/${RUN_ID})" -cat << EOF -- [View boot logs](https://github.com/${REPO}/actions/runs/${RUN_ID}) +if [ "$HAS_KSELFTEST" = true ]; then + cat << EOF ### ✅ Kernel Selftests + +| Architecture | Passed | Failed | Compared Against | Status | +|--------------|--------|--------|-----------------|--------| EOF + for arch in "${ARCHS[@]}"; do + passed="${ARCH_DATA[${arch}_kselftest_passed]:-N/A}" + failed="${ARCH_DATA[${arch}_kselftest_failed]:-N/A}" + status="${ARCH_DATA[${arch}_kselftest_status]:-⚠️ No baseline available}" + echo "| ${arch} | ${passed} | ${failed} | ${COMPARED_AGAINST:-N/A} | ${status} |" + done + echo "" + echo "- [View kselftest logs](https://github.com/${REPO}/actions/runs/${RUN_ID})" +fi -# Kernel Selftests - conditional formatting -if [ "$MULTIARCH" = true ]; then +if [ "$HAS_LTP" = true ]; then cat << EOF -| Architecture | Passed | Failed | -|--------------|---------|--------| +### ✅ LTP Results + +| Architecture | Passed | Failed | Compared Against | Status | +|--------------|--------|--------|-----------------|--------| EOF for arch in "${ARCHS[@]}"; do - echo "| ${arch} | ${ARCH_DATA[${arch}_passed]} | ${ARCH_DATA[${arch}_failed]} |" + ltp_passed="${ARCH_DATA[${arch}_ltp_passed]:-N/A}" + ltp_failed="${ARCH_DATA[${arch}_ltp_failed]:-N/A}" + ltp_status="${ARCH_DATA[${arch}_ltp_status]:-⚠️ No baseline available}" + echo "| ${arch} | ${ltp_passed} | ${ltp_failed} | ${COMPARED_AGAINST:-N/A} | ${ltp_status} |" done -else - ARCH1="${ARCHS[0]}" - cat << EOF + echo "" + echo "- [View LTP logs](https://github.com/${REPO}/actions/runs/${RUN_ID})" -- **Architecture:** ${ARCH1} -- **Passed:** ${ARCH_DATA[${ARCH1}_passed]} -- **Failed:** ${ARCH_DATA[${ARCH1}_failed]} -EOF + if [ -n "$LTP_DETAILS" ]; then + echo "" + echo "$LTP_DETAILS" + fi fi cat << EOF -- [View kselftest logs](https://github.com/${REPO}/actions/runs/${RUN_ID}) - -${COMPARISON_SECTION} - --- 🤖 This PR was automatically generated by GitHub Actions Run ID: ${RUN_ID} diff --git a/.github/workflows/kernel-build-and-test-multiarch-trigger.yml b/.github/workflows/kernel-build-and-test-multiarch-trigger.yml index 02160ee0bf4b4..0fbb04171d5c1 100644 --- a/.github/workflows/kernel-build-and-test-multiarch-trigger.yml +++ b/.github/workflows/kernel-build-and-test-multiarch-trigger.yml @@ -18,6 +18,11 @@ on: required: false type: boolean default: false + skip_ltp: + description: 'Skip the LTP test stage' + required: false + type: boolean + default: false permissions: contents: read @@ -44,6 +49,7 @@ jobs: ARCHITECTURES: ${{ inputs.architectures }} SKIP_KABI: ${{ inputs.skip_kabi }} SKIP_KSELFTESTS: ${{ inputs.skip_kselftests }} + SKIP_LTP: ${{ inputs.skip_ltp }} COMMIT_MSG: ${{ github.event.head_commit.message }} PR_TITLE: ${{ github.event.pull_request.title }} run: | @@ -138,6 +144,12 @@ jobs: exit 1 fi + # Validate skip_ltp - must be exactly 'true' or 'false' + if ! [[ "$SKIP_LTP" =~ ^(true|false)$ ]]; then + echo "❌ Invalid skip_ltp value: $SKIP_LTP" + exit 1 + fi + # Pass validated values to environment echo "IS_PR=$IS_PR" >> "$GITHUB_ENV" echo "BASE_REF=$BASE_REF" >> "$GITHUB_ENV" @@ -146,6 +158,7 @@ jobs: echo "ARCHITECTURES=$ARCHITECTURES" >> "$GITHUB_ENV" echo "SKIP_KABI=$SKIP_KABI" >> "$GITHUB_ENV" echo "SKIP_KSELFTESTS=$SKIP_KSELFTESTS" >> "$GITHUB_ENV" + echo "SKIP_LTP=$SKIP_LTP" >> "$GITHUB_ENV" - name: Clone base branch if: github.event_name == 'pull_request' @@ -194,6 +207,7 @@ jobs: echo "$ARCHITECTURES" > pr_metadata/architectures.txt echo "$SKIP_KABI" > pr_metadata/skip_kabi.txt echo "$SKIP_KSELFTESTS" > pr_metadata/skip_kselftests.txt + echo "$SKIP_LTP" > pr_metadata/skip_ltp.txt echo "$IS_PR" > pr_metadata/is_pr.txt # Create a checksum of metadata for integrity verification diff --git a/.github/workflows/kernel-build-and-test-multiarch.yml b/.github/workflows/kernel-build-and-test-multiarch.yml index 22314c160994a..abbd2ce457311 100644 --- a/.github/workflows/kernel-build-and-test-multiarch.yml +++ b/.github/workflows/kernel-build-and-test-multiarch.yml @@ -36,6 +36,7 @@ jobs: architectures: ${{ steps.pr_metadata.outputs.architectures }} skip_kabi: ${{ steps.pr_metadata.outputs.skip_kabi }} skip_kselftests: ${{ steps.pr_metadata.outputs.skip_kselftests }} + skip_ltp: ${{ steps.pr_metadata.outputs.skip_ltp }} is_pr: ${{ steps.pr_metadata.outputs.is_pr }} steps: @@ -93,6 +94,7 @@ jobs: ARCHITECTURES=$(cat pr_metadata/architectures.txt) SKIP_KABI=$(cat pr_metadata/skip_kabi.txt) SKIP_KSELFTESTS=$(cat pr_metadata/skip_kselftests.txt) + SKIP_LTP=$(cat pr_metadata/skip_ltp.txt 2>/dev/null || echo "false") IS_PR=$(cat pr_metadata/is_pr.txt) # === CRITICAL VALIDATION: Prevent command injection === @@ -183,6 +185,12 @@ jobs: exit 1 fi + # Validate skip_ltp - must be exactly 'true' or 'false' + if ! [[ "$SKIP_LTP" =~ ^(true|false)$ ]]; then + echo "❌ Security: Invalid skip_ltp value: $SKIP_LTP" + exit 1 + fi + # === All validation passed - safe to use === echo "✅ All metadata validation passed" @@ -196,6 +204,7 @@ jobs: echo "architectures=$ARCHITECTURES" >> $GITHUB_OUTPUT echo "skip_kabi=$SKIP_KABI" >> $GITHUB_OUTPUT echo "skip_kselftests=$SKIP_KSELFTESTS" >> $GITHUB_OUTPUT + echo "skip_ltp=$SKIP_LTP" >> $GITHUB_OUTPUT echo "is_pr=$IS_PR" >> $GITHUB_OUTPUT - name: Upload head_ref for baseline search @@ -318,6 +327,8 @@ jobs: -v "$PWD/kernel-container-build/build-container/image_from_container.sh":/usr/local/bin/image_from_container.sh:ro \ -v "$PWD/kernel-container-build/container/kernel_build.sh":/usr/libexec/kernel_build.sh:ro \ -v "$PWD/kernel-container-build/container/check_kabi.sh":/usr/libexec/check_kabi.sh:ro \ + -v "$PWD/kernel-container-build/container/build_kselftests.sh":/usr/libexec/build_kselftests.sh:ro \ + -v "$PWD/kernel-container-build/container/build_ltp.sh":/usr/libexec/build_ltp.sh:ro \ --security-opt label=disable \ pulp.prod.ciq.dev/ciq/cicd/lts-images/builder \ /usr/local/build-scripts/build_kernel.sh "${BUILD_ARGS[@]}" 2>&1 | tee output/kernel-build.log @@ -332,17 +343,28 @@ jobs: path: output/kernel-build.log retention-days: 7 - # Upload qcow2 image for next stages - - name: Upload qcow2 image + # Upload kselftests qcow2 image + - name: Upload kselftests qcow2 image uses: actions/upload-artifact@v4 if: always() with: - name: kernel-qcow2-image-${{ matrix.arch }} + name: kernel-qcow2-kselftests-${{ matrix.arch }} path: | - output/*.qcow2 + output/*-kselftests.qcow2 output/last_build_image.txt retention-days: 7 + # Upload LTP qcow2 image + - name: Upload LTP qcow2 image + uses: actions/upload-artifact@v4 + if: ${{ always() && needs.pre-setup.outputs.skip_ltp == 'false' }} + with: + name: kernel-qcow2-ltp-${{ matrix.arch }} + path: | + output/*-ltp.qcow2 + output/last_ltp_image.txt + retention-days: 7 + boot: name: Boot verification (${{ matrix.arch }}) runs-on: ${{ matrix.runner }} @@ -382,7 +404,7 @@ jobs: - name: Download qcow2 image uses: actions/download-artifact@v4 with: - name: kernel-qcow2-image-${{ matrix.arch }} + name: kernel-qcow2-kselftests-${{ matrix.arch }} path: output # Boot verification test @@ -449,7 +471,7 @@ jobs: - name: Download qcow2 image uses: actions/download-artifact@v4 with: - name: kernel-qcow2-image-${{ matrix.arch }} + name: kernel-qcow2-kselftests-${{ matrix.arch }} path: output # Run kselftests @@ -478,10 +500,299 @@ jobs: output/dmesg-*.log retention-days: 7 + test-ltp: + name: Run LTP (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + needs: [pre-setup, setup, boot] + if: ${{ needs.pre-setup.outputs.skip_ltp == 'false' }} + continue-on-error: true # LTP results are informational only - do not fail the workflow + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} + + steps: + - name: Generate GitHub App token + id: generate_token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + repositories: | + kernel-container-build + + - name: Checkout kernel-container-build (test branch) + uses: actions/checkout@v4 + with: + repository: ctrliq/kernel-container-build + ref: automated-testing-v1 + path: kernel-container-build + token: ${{ steps.generate_token.outputs.token }} + + - name: Install host dependencies + run: | + set -euxo pipefail + sudo apt-get update + sudo apt-get install -y fuse3 cpu-checker podman + sudo modprobe fuse + if [ -e /dev/kvm ]; then + sudo chmod 0666 /dev/kvm + fi + + - name: Download qcow2 image + uses: actions/download-artifact@v4 + with: + name: kernel-qcow2-ltp-${{ matrix.arch }} + path: output + + - name: Execute LTP + run: | + set -euxo pipefail + chmod +x kernel-container-build/build-container/*.sh + podman run --rm --pull=always \ + --privileged \ + --device=/dev/fuse \ + $([ -e /dev/kvm ] && echo "--device=/dev/kvm") \ + -v "$PWD/output":/output \ + -v "$PWD/kernel-container-build/build-container":/usr/local/build-scripts:ro \ + --security-opt label=disable \ + pulp.prod.ciq.dev/ciq/cicd/lts-images/builder \ + /usr/local/build-scripts/test_ltp.sh + + - name: Parse LTP results + if: always() + run: | + set -euxo pipefail + chmod +x kernel-container-build/build-container/parse_ltp_results.sh + LOG=$(ls output/ltp-*.log 2>/dev/null | head -1) + if [ -n "$LOG" ]; then + kernel-container-build/build-container/parse_ltp_results.sh "$LOG" output/ltp-results.json + else + echo "::warning::No LTP log found to parse" + fi + + - name: Upload LTP logs + uses: actions/upload-artifact@v4 + if: always() + with: + name: ltp-logs-${{ matrix.arch }} + path: | + output/ltp-*.log + output/ltp-results.json + output/dmesg-ltp-*.log + retention-days: 7 + + compare-ltp: + name: Compare LTP results (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + needs: [pre-setup, setup, build, boot, test-ltp] + if: ${{ always() && needs.pre-setup.outputs.skip_ltp == 'false' && needs.build.result == 'success' && needs.boot.result == 'success' }} + continue-on-error: true # LTP results are informational only - do not fail the workflow + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} + outputs: + comparison_status_x86_64: ${{ matrix.arch == 'x86_64' && steps.ltp_comparison.outputs.comparison_status || '' }} + comparison_status_aarch64: ${{ matrix.arch == 'aarch64' && steps.ltp_comparison.outputs.comparison_status || '' }} + regressions_x86_64: ${{ matrix.arch == 'x86_64' && steps.ltp_comparison.outputs.regressions || '' }} + regressions_aarch64: ${{ matrix.arch == 'aarch64' && steps.ltp_comparison.outputs.regressions || '' }} + fixes_x86_64: ${{ matrix.arch == 'x86_64' && steps.ltp_comparison.outputs.fixes || '' }} + fixes_aarch64: ${{ matrix.arch == 'aarch64' && steps.ltp_comparison.outputs.fixes || '' }} + passed_x86_64: ${{ matrix.arch == 'x86_64' && steps.ltp_comparison.outputs.passed || '' }} + passed_aarch64: ${{ matrix.arch == 'aarch64' && steps.ltp_comparison.outputs.passed || '' }} + failed_x86_64: ${{ matrix.arch == 'x86_64' && steps.ltp_comparison.outputs.failed || '' }} + failed_aarch64: ${{ matrix.arch == 'aarch64' && steps.ltp_comparison.outputs.failed || '' }} + + steps: + - name: Checkout kernel source + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Checkout kernel-container-build (test branch) + id: generate_token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + repositories: | + kernel-container-build + + - name: Checkout kernel-container-build + uses: actions/checkout@v4 + with: + repository: ctrliq/kernel-container-build + ref: automated-testing-v1 + path: kernel-container-build + token: ${{ steps.generate_token.outputs.token }} + + - name: Install GitHub CLI + run: | + if ! command -v gh &> /dev/null; then + sudo apt-get update && sudo apt-get install -y gh + fi + + - name: Generate GitHub App token for comparison + id: generate_token_compare + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + repositories: | + kernel-src-tree + kernel-container-build + + - name: Download current LTP results + if: needs.test-ltp.result == 'success' + uses: actions/download-artifact@v4 + with: + name: ltp-logs-${{ matrix.arch }} + path: output-current + + - name: Determine base branch + id: base_branch + env: + GH_TOKEN: ${{ steps.generate_token_compare.outputs.token }} + HEAD_REF: ${{ needs.pre-setup.outputs.head_ref }} + BASE_REF: ${{ needs.pre-setup.outputs.base_ref }} + PR_NUMBER: ${{ needs.pre-setup.outputs.pr_number }} + run: | + # Reuse same base branch logic as compare-results job + BASE_BRANCH="" + BRANCH_NAME="$HEAD_REF" + VALID_BASES="ciqlts9_2 ciqlts9_4 ciqlts8_6 ciqlts9_6 ciq-6.12.y ciq-6.18.y ciqcbr7_9" + + EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --state open --json number,baseRefName --jq '.[0]' || echo "") + if [ -n "$EXISTING_PR" ] && [ "$EXISTING_PR" != "null" ]; then + BASE_BRANCH=$(echo "$EXISTING_PR" | jq -r '.baseRefName') + elif [ -n "$BASE_REF" ]; then + BASE_BRANCH="$BASE_REF" + elif [[ "$BRANCH_NAME" =~ ^\{[^}]+\}_(rlc-[0-9]+/.+)$ ]]; then + BASE_BRANCH="${BASH_REMATCH[1]}" + elif [[ "$BRANCH_NAME" =~ \{[^}]+\}[_-](.+) ]]; then + EXTRACTED_BASE="${BASH_REMATCH[1]}" + if echo "$VALID_BASES" | grep -wq "$EXTRACTED_BASE"; then + BASE_BRANCH="$EXTRACTED_BASE" + fi + fi + + echo "base_branch=$BASE_BRANCH" >> $GITHUB_OUTPUT + + - name: Download baseline LTP results from last merged PR + if: steps.base_branch.outputs.base_branch != '' + env: + GH_TOKEN: ${{ steps.generate_token_compare.outputs.token }} + run: | + set +e + BASE_BRANCH="${{ steps.base_branch.outputs.base_branch }}" + CURRENT_RUN_ID="${{ github.run_id }}" + + SUCCESSFUL_RUNS=$(gh run list \ + --workflow kernel-build-and-test-multiarch.yml \ + --status success \ + --limit 50 \ + --json databaseId,headBranch,createdAt) + + if [ -z "$SUCCESSFUL_RUNS" ] || [ "$SUCCESSFUL_RUNS" = "[]" ]; then + echo "::warning::No successful workflow runs found" + exit 0 + fi + + while read -r run; do + RUN_ID=$(echo "$run" | jq -r '.databaseId') + HEAD_BRANCH=$(echo "$run" | jq -r '.headBranch') + + [ "$RUN_ID" = "$CURRENT_RUN_ID" ] && continue + + EXTRACTED_BASE="" + if [[ "$HEAD_BRANCH" =~ ^\{[^}]+\}_(rlc-[0-9]+/.+)$ ]]; then + EXTRACTED_BASE="${BASH_REMATCH[1]}" + elif [[ "$HEAD_BRANCH" =~ \{[^}]+\}[_-](.+) ]]; then + EXTRACTED_BASE="${BASH_REMATCH[1]}" + fi + + if [ "$EXTRACTED_BASE" = "$BASE_BRANCH" ]; then + PR_INFO=$(gh pr list --head "$HEAD_BRANCH" --base "$BASE_BRANCH" --state merged --json number,mergedAt,baseRefName --jq '.[0]' 2>/dev/null || echo "") + if [ -n "$PR_INFO" ] && [ "$PR_INFO" != "null" ]; then + ARTIFACT_ID=$(gh api "repos/${{ github.repository }}/actions/runs/$RUN_ID/artifacts" \ + --jq ".artifacts[] | select(.name == \"ltp-logs-${{ matrix.arch }}\" and .expired == false) | .id" \ + | tail -1) + if [ -n "$ARTIFACT_ID" ]; then + mkdir -p output-previous + if gh api "repos/${{ github.repository }}/actions/artifacts/$ARTIFACT_ID/zip" > /tmp/baseline-ltp.zip 2>/dev/null && \ + unzip -q /tmp/baseline-ltp.zip -d output-previous 2>/dev/null; then + rm -f /tmp/baseline-ltp.zip + exit 0 + fi + rm -f /tmp/baseline-ltp.zip + fi + fi + fi + done < <(echo "$SUCCESSFUL_RUNS" | jq -c '.[]') + + echo "::warning::No baseline LTP results found from merged PRs targeting $BASE_BRANCH" + continue-on-error: true + timeout-minutes: 3 + + - name: Compare LTP results + id: ltp_comparison + run: | + chmod +x kernel-container-build/build-container/compare_ltp_results.sh + + CURR_JSON=$(ls output-current/ltp-results.json 2>/dev/null | head -1) + PREV_JSON=$(ls output-previous/ltp-results.json 2>/dev/null | head -1) + EXPECTED_FILE=".github/ltp-expected.txt" + + if [ -z "$CURR_JSON" ]; then + echo "::warning::No current LTP results JSON found, skipping comparison" + echo "comparison_status=skipped" >> $GITHUB_OUTPUT + echo "regressions=" >> $GITHUB_OUTPUT + echo "fixes=" >> $GITHUB_OUTPUT + echo "passed=" >> $GITHUB_OUTPUT + echo "failed=" >> $GITHUB_OUTPUT + exit 0 + fi + + # Count passed/failed from current results + PASSED=$(jq '[to_entries[] | select(.value == "PASS")] | length' "$CURR_JSON") + FAILED=$(jq '[to_entries[] | select(.value == "FAIL")] | length' "$CURR_JSON") + echo "passed=$PASSED" >> $GITHUB_OUTPUT + echo "failed=$FAILED" >> $GITHUB_OUTPUT + + if [ -z "$PREV_JSON" ]; then + echo "::notice::No baseline LTP results found - first run or artifacts expired" + echo "comparison_status=skipped" >> $GITHUB_OUTPUT + echo "regressions=" >> $GITHUB_OUTPUT + echo "fixes=" >> $GITHUB_OUTPUT + exit 0 + fi + + EXPECTED_ARG="" + [ -f "$EXPECTED_FILE" ] && EXPECTED_ARG="$EXPECTED_FILE" + + set +e + COMPARE_OUTPUT=$(kernel-container-build/build-container/compare_ltp_results.sh "$PREV_JSON" "$CURR_JSON" $EXPECTED_ARG 2>&1) + COMPARE_EXIT=$? + set -e + + echo "$COMPARE_OUTPUT" + + REGRESSIONS=$(echo "$COMPARE_OUTPUT" | grep '^ - ' | sed 's/^ - //' | paste -sd '|' -) + FIXES=$(echo "$COMPARE_OUTPUT" | grep '^ + ' | sed 's/^ + //' | paste -sd '|' -) + + echo "regressions=$REGRESSIONS" >> $GITHUB_OUTPUT + echo "fixes=$FIXES" >> $GITHUB_OUTPUT + + if [ $COMPARE_EXIT -ne 0 ]; then + echo "::warning::LTP regressions detected (informational only)" + echo "comparison_status=failed" >> $GITHUB_OUTPUT + else + echo "comparison_status=passed" >> $GITHUB_OUTPUT + fi + compare-results: name: Compare with previous run (${{ matrix.arch }}) runs-on: ${{ matrix.runner }} - needs: [pre-setup, setup, build, boot, test-kselftest] + needs: [pre-setup, setup, build, boot, test-kselftest, test-ltp] if: ${{ always() && needs.build.result == 'success' && needs.boot.result == 'success' }} strategy: fail-fast: false @@ -793,7 +1104,7 @@ jobs: create-pr: name: Create Pull Request runs-on: kernel-build - needs: [pre-setup, setup, build, boot, test-kselftest, compare-results] + needs: [pre-setup, setup, build, boot, test-kselftest, compare-results, test-ltp, compare-ltp] if: | always() && needs.build.result == 'success' && @@ -818,7 +1129,8 @@ jobs: ARCHITECTURES: ${{ needs.pre-setup.outputs.architectures }} run: | # Skip PR if any required stage failed - # test-kselftest is optional when skip_kselftests is true (result will be 'skipped') + # test-kselftest is optional when skip_kselftests input is true + # LTP failures/regressions are informational only - do not block PR creation KSELFTEST_RESULT="${{ needs.test-kselftest.result }}" if [ "${{ needs.build.result }}" != "success" ] || \ [ "${{ needs.boot.result }}" != "success" ] || \ @@ -831,25 +1143,26 @@ jobs: ARCHS="$ARCHITECTURES" REGRESSION_DETECTED=false - # Check x86_64 regression if enabled + # Check x86_64 regressions if enabled if echo "$ARCHS" | grep -q "x86_64"; then if [ "${{ needs.compare-results.outputs.comparison_status_x86_64 }}" = "failed" ]; then - echo "x86_64: Test regression detected" + echo "x86_64: kselftest regression detected" REGRESSION_DETECTED=true fi fi - # Check aarch64 regression if enabled + # Check aarch64 regressions if enabled if echo "$ARCHS" | grep -q "aarch64"; then if [ "${{ needs.compare-results.outputs.comparison_status_aarch64 }}" = "failed" ]; then - echo "aarch64: Test regression detected" + echo "aarch64: kselftest regression detected" REGRESSION_DETECTED=true fi fi - # Skip PR if any regression was detected (but allow if comparison was skipped/unavailable) + # Note: LTP regressions are shown in PR body but do not block PR creation + if [ "$REGRESSION_DETECTED" = "true" ]; then - echo "Test regression detected, skipping PR creation" + echo "Regression detected, skipping PR creation" exit 1 fi @@ -1054,7 +1367,28 @@ jobs: HAS_X86="${{ steps.detect_arch.outputs.has_x86_64 }}" HAS_ARM="${{ steps.detect_arch.outputs.has_aarch64 }}" - # Determine comparison status message + # Helper functions + kselftest_status_text() { + case "$1" in + passed) echo "✅ No regressions" ;; + failed) echo "❌ Regression detected" ;; + *) echo "⚠️ No baseline available" ;; + esac + } + + ltp_format_list() { + echo "$1" | tr '|' '\n' | sed 's/^/- /' + } + + ltp_status_text() { + case "$1" in + passed) echo "✅ No regressions" ;; + failed) echo "❌ $(echo "$2" | tr '|' '\n' | wc -l | tr -d ' ') regressions" ;; + *) echo "⚠️ No baseline available" ;; + esac + } + + # Kselftest comparison status COMPARISON_STATUS_X86="${{ needs.compare-results.outputs.comparison_status_x86_64 }}" COMPARISON_STATUS_ARM="${{ needs.compare-results.outputs.comparison_status_aarch64 }}" @@ -1064,57 +1398,42 @@ jobs: [ -z "$COMPARISON_STATUS_ARM" ] && COMPARISON_STATUS_ARM="skipped" fi - # Create comparison section - use printf to avoid code block formatting - COMPARISON_SECTION="### Test Comparison" - - # Add x86_64 section if enabled - if [ "$HAS_X86" = "true" ]; then - if [ "$COMPARISON_STATUS_X86" = "passed" ]; then - COMPARISON_SECTION="$COMPARISON_SECTION - - **x86_64:** - - ✅ Status: Passed - Within acceptable threshold (±3 tests) - - Compared against: $BASE_BRANCH" - elif [ "$COMPARISON_STATUS_X86" = "skipped" ]; then - COMPARISON_SECTION="$COMPARISON_SECTION - - **x86_64:** - - ⚠️ Status: Skipped - No baseline available" - else - COMPARISON_SECTION="$COMPARISON_SECTION - - **x86_64:** - - ❌ Status: Failed - Regression detected" + # LTP comparison data + LTP_STATUS_X86="${{ needs.compare-ltp.outputs.comparison_status_x86_64 }}" + LTP_STATUS_ARM="${{ needs.compare-ltp.outputs.comparison_status_aarch64 }}" + LTP_REGRESSIONS_X86="${{ needs.compare-ltp.outputs.regressions_x86_64 }}" + LTP_FIXES_X86="${{ needs.compare-ltp.outputs.fixes_x86_64 }}" + LTP_REGRESSIONS_ARM="${{ needs.compare-ltp.outputs.regressions_aarch64 }}" + LTP_FIXES_ARM="${{ needs.compare-ltp.outputs.fixes_aarch64 }}" + LTP_PASSED_X86="${{ needs.compare-ltp.outputs.passed_x86_64 }}" + LTP_FAILED_X86="${{ needs.compare-ltp.outputs.failed_x86_64 }}" + LTP_PASSED_ARM="${{ needs.compare-ltp.outputs.passed_aarch64 }}" + LTP_FAILED_ARM="${{ needs.compare-ltp.outputs.failed_aarch64 }}" + + # Build LTP details (regressions/fixes as bullet lists below the table) + LTP_DETAILS="" + if [ "${{ needs.pre-setup.outputs.skip_ltp }}" != "true" ]; then + if [ "$HAS_X86" = "true" ] && [ "$LTP_STATUS_X86" = "failed" ]; then + LTP_DETAILS="$LTP_DETAILS"$'\n'"**x86_64 regressions:**"$'\n'"$(ltp_format_list "$LTP_REGRESSIONS_X86")" fi - fi - - # Add aarch64 section if enabled - if [ "$HAS_ARM" = "true" ]; then - if [ "$COMPARISON_STATUS_ARM" = "passed" ]; then - COMPARISON_SECTION="$COMPARISON_SECTION - - **aarch64:** - - ✅ Status: Passed - Within acceptable threshold (±3 tests) - - Compared against: $BASE_BRANCH" - elif [ "$COMPARISON_STATUS_ARM" = "skipped" ]; then - COMPARISON_SECTION="$COMPARISON_SECTION - - **aarch64:** - - ⚠️ Status: Skipped - No baseline available" - else - COMPARISON_SECTION="$COMPARISON_SECTION - - **aarch64:** - - ❌ Status: Failed - Regression detected" + if [ "$HAS_X86" = "true" ] && [ -n "$LTP_FIXES_X86" ]; then + LTP_DETAILS="$LTP_DETAILS"$'\n'"**x86_64 newly passing:**"$'\n'"$(ltp_format_list "$LTP_FIXES_X86")" + fi + if [ "$HAS_ARM" = "true" ] && [ "$LTP_STATUS_ARM" = "failed" ]; then + LTP_DETAILS="$LTP_DETAILS"$'\n'"**aarch64 regressions:**"$'\n'"$(ltp_format_list "$LTP_REGRESSIONS_ARM")" + fi + if [ "$HAS_ARM" = "true" ] && [ -n "$LTP_FIXES_ARM" ]; then + LTP_DETAILS="$LTP_DETAILS"$'\n'"**aarch64 newly passing:**"$'\n'"$(ltp_format_list "$LTP_FIXES_ARM")" fi fi - # Build script arguments with named parameters + # Build script arguments SCRIPT_ARGS=( --run-id "${{ github.run_id }}" - --comparison "$COMPARISON_SECTION" --repo "${{ github.repository }}" --commit-file "/tmp/commit_message.txt" + --compared-against "$BASE_BRANCH" + --ltp-details "$LTP_DETAILS" ) # Add x86_64 architecture if enabled @@ -1123,9 +1442,22 @@ jobs: --arch x86_64 --build-time "${{ steps.build_info.outputs.build_time_x86_64 }}" --total-time "${{ steps.build_info.outputs.total_time_x86_64 }}" - --passed "${{ steps.stats.outputs.passed_x86_64 }}" - --failed "${{ steps.stats.outputs.failed_x86_64 }}" ) + # Only add kselftest data if not skipped + if [ "${{ needs.pre-setup.outputs.skip_kselftests }}" != "true" ]; then + SCRIPT_ARGS+=( + --kselftest-passed "${{ steps.stats.outputs.passed_x86_64 }}" + --kselftest-failed "${{ steps.stats.outputs.failed_x86_64 }}" + --kselftest-status "$(kselftest_status_text "$COMPARISON_STATUS_X86")" + ) + fi + if [ "${{ needs.pre-setup.outputs.skip_ltp }}" != "true" ]; then + SCRIPT_ARGS+=( + --ltp-passed "${LTP_PASSED_X86:-N/A}" + --ltp-failed "${LTP_FAILED_X86:-N/A}" + --ltp-status "$(ltp_status_text "$LTP_STATUS_X86" "$LTP_REGRESSIONS_X86")" + ) + fi fi # Add aarch64 architecture if enabled @@ -1134,9 +1466,21 @@ jobs: --arch aarch64 --build-time "${{ steps.build_info.outputs.build_time_aarch64 }}" --total-time "${{ steps.build_info.outputs.total_time_aarch64 }}" - --passed "${{ steps.stats.outputs.passed_aarch64 }}" - --failed "${{ steps.stats.outputs.failed_aarch64 }}" ) + if [ "${{ needs.pre-setup.outputs.skip_kselftests }}" != "true" ]; then + SCRIPT_ARGS+=( + --kselftest-passed "${{ steps.stats.outputs.passed_aarch64 }}" + --kselftest-failed "${{ steps.stats.outputs.failed_aarch64 }}" + --kselftest-status "$(kselftest_status_text "$COMPARISON_STATUS_ARM")" + ) + fi + if [ "${{ needs.pre-setup.outputs.skip_ltp }}" != "true" ]; then + SCRIPT_ARGS+=( + --ltp-passed "${LTP_PASSED_ARM:-N/A}" + --ltp-failed "${LTP_FAILED_ARM:-N/A}" + --ltp-status "$(ltp_status_text "$LTP_STATUS_ARM" "$LTP_REGRESSIONS_ARM")" + ) + fi fi # Call script with named arguments