From 971813d5d1d1be73e0d3da700eeb0f2839809017 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 8 May 2026 10:32:06 -0400 Subject: [PATCH] Add code coverage grading workflow --- .github/workflows/codecov.yml | 56 ++++++++++- scripts/coverage-grade.sh | 174 ++++++++++++++++++++++++++++++++++ 2 files changed, 229 insertions(+), 1 deletion(-) create mode 100755 scripts/coverage-grade.sh diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 0ee10baa385b..6aadfe620b28 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -21,6 +21,7 @@ on: [pull_request, push] permissions: contents: read + pull-requests: write # required to post/update the grade comment on PRs concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -28,7 +29,7 @@ concurrency: jobs: build: - if: github.repository == 'apache/cloudstack' + if: github.repository == 'apache/cloudstack' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) name: codecov runs-on: ubuntu-22.04 steps: @@ -57,3 +58,56 @@ jobs: verbose: true name: codecov token: ${{ secrets.CODECOV_TOKEN }} + + - name: Compute Coverage Grade + id: grade + run: bash scripts/coverage-grade.sh client/target/site/jacoco-aggregate/jacoco.xml + + # Posts a new comment on every push so coverage history is preserved across the PR timeline. + # On push events (no PR number) this step is skipped automatically. + - name: Post Coverage Grade Comment on PR + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const grade = '${{ steps.grade.outputs.coverage_grade }}'; + const label = '${{ steps.grade.outputs.coverage_grade_label }}'; + const linePct = '${{ steps.grade.outputs.line_coverage }}'; + const branchPct = '${{ steps.grade.outputs.branch_coverage }}'; + const emojiMap = { A: 'šŸŸ¢', B: 'šŸŸ”', C: 'šŸŸ ', D: 'šŸ”“', F: 'ā›”' }; + const emoji = emojiMap[grade] ?? 'ā“'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + + const branchRow = branchPct !== 'N/A' + ? `| Branch coverage | **${branchPct}%** |` + : ''; + + const body = [ + `## ${emoji} Test Coverage Grade: \`${grade}\` ā€” ${label}`, + '', + '| Metric | Value |', + '|--------|-------|', + `| Line coverage | **${linePct}%** |`, + branchRow, + '', + '### Grade Scale', + '| Grade | Line Coverage | Meaning |', + '|-------|--------------|---------|', + '| šŸŸ¢ A | ā‰„ 80% | Excellent - this code sleeps well at night šŸ˜“ |', + '| šŸŸ” B | 60-79% | Good - almost there, don\'t stop now šŸ˜‰ |', + '| šŸŸ  C | 40-59% | Acceptable - your code is wearing a seatbelt, but no airbags šŸ˜¬ |', + '| šŸ”“ D | 20-39% | Marginal - boldly shipping where no test has gone before šŸ–– |', + '| ā›” F | < 20% | Failing - tests? what tests? šŸ”„ |', + '', + '> Branch coverage is shown as a secondary signal. Grade is determined by **line coverage**.', + `> [View full Actions run](${runUrl})`, + ].filter(l => l !== undefined).join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body, + }); + console.log('Posted coverage grade comment'); diff --git a/scripts/coverage-grade.sh b/scripts/coverage-grade.sh new file mode 100755 index 000000000000..fc7edc8dcb99 --- /dev/null +++ b/scripts/coverage-grade.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# coverage-grade.sh +# +# Parses the JaCoCo aggregate XML report and outputs an Aā€“F coverage grade. +# +# Usage: +# ./scripts/coverage-grade.sh [path/to/jacoco.xml] +# +# Exit codes: +# 0 ā€“ grade is D or above (line coverage >= 20%) +# 1 ā€“ grade is F (line coverage < 20%) +# +# Environment variables (optional, used when writing GitHub outputs): +# GITHUB_OUTPUT ā€“ set automatically by GitHub Actions +# GITHUB_STEP_SUMMARY ā€“ set automatically by GitHub Actions + +set -euo pipefail + +JACOCO_XML="${1:-client/target/site/jacoco-aggregate/jacoco.xml}" + +if [[ ! -f "$JACOCO_XML" ]]; then + echo "ERROR: JaCoCo report not found at: $JACOCO_XML" >&2 + exit 2 +fi + +# --------------------------------------------------------------------------- +# Parse LINE and BRANCH counters from the top-level element using +# Python's built-in xml.etree.ElementTree (no extra dependencies needed). +# --------------------------------------------------------------------------- +read -r LINE_COVERED LINE_MISSED BRANCH_COVERED BRANCH_MISSED < <(python3 - "$JACOCO_XML" <<'PYEOF' +import sys, xml.etree.ElementTree as ET + +tree = ET.parse(sys.argv[1]) +root = tree.getroot() + +lc = lm = bc = bm = 0 +# Sum counters from all children so we get the true aggregate, +# avoiding any duplicate top-level counter that some JaCoCo versions emit. +for pkg in root.iter('package'): + for counter in pkg.findall('counter'): + t = counter.get('type') + if t == 'LINE': + lc += int(counter.get('covered', 0)) + lm += int(counter.get('missed', 0)) + elif t == 'BRANCH': + bc += int(counter.get('covered', 0)) + bm += int(counter.get('missed', 0)) + +print(lc, lm, bc, bm) +PYEOF +) + +# --------------------------------------------------------------------------- +# Compute percentages +# --------------------------------------------------------------------------- +line_total=$(( LINE_COVERED + LINE_MISSED )) +branch_total=$(( BRANCH_COVERED + BRANCH_MISSED )) + +if (( line_total == 0 )); then + echo "ERROR: No LINE counters found in $JACOCO_XML ā€“ was the build run with -P quality?" >&2 + exit 2 +fi + +# Use awk for floating-point arithmetic +LINE_PCT=$(awk "BEGIN { printf \"%.2f\", ($LINE_COVERED / $line_total) * 100 }") + +if (( branch_total > 0 )); then + BRANCH_PCT=$(awk "BEGIN { printf \"%.2f\", ($BRANCH_COVERED / $branch_total) * 100 }") +else + BRANCH_PCT="N/A" +fi + +# --------------------------------------------------------------------------- +# Assign grade based on LINE coverage +# +# A ā‰„ 80% Excellent +# B 60ā€“79% Good +# C 40ā€“59% Acceptable +# D 20ā€“39% Marginal (meets minimum gate) +# F < 20% Failing +# --------------------------------------------------------------------------- +LINE_INT=$(awk "BEGIN { printf \"%d\", $LINE_PCT }") # truncate, not round + +if (( LINE_INT >= 80 )); then GRADE="A"; EMOJI="šŸŸ¢"; LABEL="Excellent" +elif (( LINE_INT >= 60 )); then GRADE="B"; EMOJI="šŸŸ”"; LABEL="Good" +elif (( LINE_INT >= 40 )); then GRADE="C"; EMOJI="šŸŸ "; LABEL="Acceptable" +elif (( LINE_INT >= 20 )); then GRADE="D"; EMOJI="šŸ”“"; LABEL="Marginal" +else GRADE="F"; EMOJI="ā›”"; LABEL="Failing" +fi + +# --------------------------------------------------------------------------- +# Human-readable output (always printed to stdout) +# --------------------------------------------------------------------------- +echo "ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”" +echo "ā”‚ CloudStack Test Coverage Report ā”‚" +echo "ā”œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤" +printf "ā”‚ Grade : %s %-5s %-20s ā”‚\n" "$EMOJI" "$GRADE" "($LABEL)" +printf "ā”‚ Line coverage: %6s%% (%d / %d lines)%*sā”‚\n" \ + "$LINE_PCT" "$LINE_COVERED" "$line_total" \ + $(( 14 - ${#LINE_COVERED} - ${#line_total} )) " " +if [[ "$BRANCH_PCT" != "N/A" ]]; then + printf "ā”‚ Branch cov. : %6s%% (%d / %d branches)%*sā”‚\n" \ + "$BRANCH_PCT" "$BRANCH_COVERED" "$branch_total" \ + $(( 11 - ${#BRANCH_COVERED} - ${#branch_total} )) " " +else + printf "ā”‚ Branch cov. : N/A (no branch data) ā”‚\n" +fi +echo "ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜" +echo "" +echo "Grade scale: A ā‰„80% B 60-79% C 40-59% D 20-39% F <20% (line coverage)" + +# --------------------------------------------------------------------------- +# GitHub Actions: write outputs and step summary +# --------------------------------------------------------------------------- +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + { + echo "coverage_grade=$GRADE" + echo "coverage_grade_label=$LABEL" + echo "line_coverage=$LINE_PCT" + echo "branch_coverage=$BRANCH_PCT" + } >> "$GITHUB_OUTPUT" +fi + +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + { + echo "## $EMOJI Test Coverage Grade: **$GRADE** ā€” $LABEL" + echo "" + echo "| Metric | Covered | Total | Percentage |" + echo "|--------|---------|-------|------------|" + echo "| Line coverage | $LINE_COVERED | $line_total | **${LINE_PCT}%** |" + if [[ "$BRANCH_PCT" != "N/A" ]]; then + echo "| Branch coverage | $BRANCH_COVERED | $branch_total | **${BRANCH_PCT}%** |" + fi + echo "" + echo "### Grade Scale" + echo "| Grade | Line Coverage | Meaning |" + echo "|-------|--------------|---------|" + echo "| šŸŸ¢ A | ā‰„ 80% | Excellent - this code sleeps well at night šŸ˜“ |" + echo "| šŸŸ” B | 60-79% | Good - almost there, don't stop now šŸ˜‰ |" + echo "| šŸŸ  C | 40-59% | Acceptable - your code is wearing a seatbelt, but no airbags šŸ˜¬ |" + echo "| šŸ”“ D | 20-39% | Marginal - boldly shipping where no test has gone before šŸ–– |" + echo "| ā›” F | < 20% | tests? what tests? šŸ”„ |" + echo "" + echo "> Branch coverage is shown as a secondary signal. Grade is based on line coverage." + } >> "$GITHUB_STEP_SUMMARY" +fi + +# --------------------------------------------------------------------------- +# Exit non-zero for grade F so the CI job can be configured to fail +# --------------------------------------------------------------------------- +if [[ "$GRADE" == "F" ]]; then + echo "" + echo "ā›” FAIL: Line coverage ${LINE_PCT}% is below the minimum threshold of 20%." >&2 + exit 1 +fi + +exit 0