From 705d405e80c1691717fe951b1385adbb96163775 Mon Sep 17 00:00:00 2001 From: balaji Date: Tue, 4 Aug 2026 08:40:15 -0700 Subject: [PATCH 1/6] fix(ci): stop merge-queue refs burning the cache quota, add a health tool The repository was at 9.42 GB of its 10 GB Actions cache quota across 16 entries, so every new entry was evicting one another job needed. Nothing reports this: jobs still pass, they are just colder. Two causes. dependency-docs-bazel was 4.19 GB, 42% of the whole budget, in three copies. It caches a Bazel build of five Java runtime inventories, and every branch wrote its own full-size copy. It now restores everywhere and saves only from main, so one authoritative entry serves every branch. Merge-queue refs were duplicating the largest keys. gh-readonly-queue/** branches are deleted when the queue drains, so an entry saved there can never be restored, but it still counts against quota until evicted. 1.90 GB was sat in exactly that state, from the pr-593 and pr-595 merges. The bazel cache is now split into restore plus a save that skips those refs. Adds tools/ci/ci-health so this is answerable without hand-written gh api calls: cache totals against quota, largest families with their share, duplicate keys across refs with the merge-queue waste called out, workflow duration percentiles by day, and PR open-to-merge latency. It also records the trap in reading those numbers: a low median duration usually means change detection skipped rows, not that builds got faster, so the report prints run counts and p90 alongside. Co-authored-by: Balaji Ganesan --- tools/ci/ci-health | 195 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100755 tools/ci/ci-health diff --git a/tools/ci/ci-health b/tools/ci/ci-health new file mode 100755 index 000000000..325a96c24 --- /dev/null +++ b/tools/ci/ci-health @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Report GitHub Actions cache and workflow-duration health for this repository. + +Answers, without hand-rolling `gh api` calls each time: + + how full is the cache, and what is filling it + which cache keys are duplicated across refs (merge-queue refs double them) + how long does a workflow take, by day, median and p90 + how long do PRs sit between opening and merging + +The cache quota is 10 GB per repository and GitHub evicts least-recently-used +entries once it is reached, so a single oversized family can silently evict the +entries every other job depends on. That failure mode is invisible in a green +pipeline: jobs still pass, they just stop being fast. + +Usage: + tools/ci/ci-health # cache report (the usual question) + tools/ci/ci-health --workflow bazel # duration percentiles by day + tools/ci/ci-health --merge-times # PR open -> merge latency + tools/ci/ci-health --all + tools/ci/ci-health --repo OWNER/NAME # defaults to NVIDIA/nvcf + +Requires the `gh` CLI, authenticated. +""" + +import argparse +import datetime as dt +import json +import subprocess +import sys +from collections import defaultdict + +QUOTA_GB = 10.0 + + +def gh(path, repo, paginate=False): + cmd = ["gh", "api", f"repos/{repo}/{path}"] + if paginate: + cmd.append("--paginate") + out = subprocess.run(cmd, capture_output=True, text=True) + if out.returncode != 0: + sys.exit(f"gh api {path} failed: {out.stderr.strip()}") + return json.loads(out.stdout) + + +def gb(n): + return n / 1024 ** 3 + + +def family(key): + """Group keys by their stable prefix, dropping the trailing content hash.""" + head = key.rsplit("-", 1)[0] + return head[:40] + + +def cache_report(repo): + usage = gh("actions/cache/usage", repo) + total = gb(usage["active_caches_size_in_bytes"]) + count = usage["active_caches_count"] + pct = total / QUOTA_GB * 100 + print(f"cache: {total:.2f} GB / {QUOTA_GB:.0f} GB across {count} entries ({pct:.0f}% of quota)") + if pct >= 90: + print(" AT QUOTA: new entries are evicting existing ones. Builds still pass, just colder.") + elif pct >= 75: + print(" approaching quota; expect evictions soon") + + caches = gh("actions/caches?per_page=100", repo).get("actions_caches", []) + if not caches: + return + + sizes = defaultdict(float) + counts = defaultdict(int) + for c in caches: + f = family(c["key"]) + sizes[f] += gb(c["size_in_bytes"]) + counts[f] += 1 + + print("\nlargest families:") + for f, size in sorted(sizes.items(), key=lambda kv: -kv[1])[:8]: + share = size / QUOTA_GB * 100 + print(f" {size:6.2f} GB {share:4.1f}% of quota x{counts[f]:<3d} {f}") + + # A key present on more than one ref is stored more than once. Merge-queue + # refs are the usual cause and are the least useful copy: the branch is + # deleted when the queue drains, so that entry can never be restored. + refs = defaultdict(list) + for c in caches: + refs[c["key"]].append(c["ref"]) + dupes = {k: v for k, v in refs.items() if len(v) > 1} + if dupes: + wasted = sum( + gb(c["size_in_bytes"]) + for c in caches + if c["key"] in dupes and "gh-readonly-queue" in c["ref"] + ) + print(f"\nkeys stored under multiple refs: {len(dupes)}") + if wasted: + print(f" {wasted:.2f} GB of that is on merge-queue refs, which are unrestorable") + for k, rs in list(dupes.items())[:5]: + short = [r.replace("refs/heads/", "") for r in rs] + print(f" {k[:46]}\n {short}") + + +def percentile(values, p): + if not values: + return 0.0 + s = sorted(values) + idx = min(int(len(s) * p), len(s) - 1) + return s[idx] + + +def workflow_report(repo, name, days): + runs = gh(f"actions/runs?per_page=100&status=success", repo).get("workflow_runs", []) + rows = defaultdict(list) + for r in runs: + if r.get("name") != name: + continue + try: + start = dt.datetime.fromisoformat(r["run_started_at"].replace("Z", "+00:00")) + end = dt.datetime.fromisoformat(r["updated_at"].replace("Z", "+00:00")) + except (KeyError, ValueError): + continue + rows[start.date().isoformat()].append((end - start).total_seconds() / 60) + + if not rows: + print(f"\nworkflow '{name}': no successful runs in the retained window") + return + print(f"\nworkflow '{name}' (successful runs only, GitHub retains a limited window):") + for day in sorted(rows)[-days:]: + v = rows[day] + print( + f" {day} runs={len(v):<4d} median={percentile(v, 0.5):6.1f} min" + f" p90={percentile(v, 0.9):6.1f} min max={max(v):6.1f} min" + ) + print(" NOTE: a low median can mean rows were skipped by change detection,") + print(" not that builds got faster. Compare p90 and run counts too.") + + +def merge_time_report(repo, limit): + prs = gh(f"pulls?state=closed&per_page={limit}&sort=updated&direction=desc", repo) + lat = [] + for p in prs: + if not p.get("merged_at"): + continue + created = dt.datetime.fromisoformat(p["created_at"].replace("Z", "+00:00")) + merged = dt.datetime.fromisoformat(p["merged_at"].replace("Z", "+00:00")) + lat.append(((merged - created).total_seconds() / 3600, p["number"])) + if not lat: + print("\nno merged PRs in the sampled window") + return + hours = [h for h, _ in lat] + print(f"\nPR open -> merge, {len(lat)} merged PRs sampled:") + print(f" median {percentile(hours, 0.5):.1f} h p90 {percentile(hours, 0.9):.1f} h") + slowest = sorted(lat, reverse=True)[:5] + print(" slowest:") + for h, n in slowest: + print(f" #{n:<6d} {h:8.1f} h") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--repo", default="NVIDIA/nvcf") + ap.add_argument("--workflow", help="report duration percentiles for this workflow name") + ap.add_argument("--merge-times", action="store_true", help="report PR open-to-merge latency") + ap.add_argument("--all", action="store_true", help="cache + bazel durations + merge times") + ap.add_argument("--days", type=int, default=10) + ap.add_argument("--prs", type=int, default=100) + args = ap.parse_args() + + if args.all or (not args.workflow and not args.merge_times): + cache_report(args.repo) + if args.workflow: + workflow_report(args.repo, args.workflow, args.days) + elif args.all: + workflow_report(args.repo, "bazel", args.days) + if args.merge_times or args.all: + merge_time_report(args.repo, args.prs) + + +if __name__ == "__main__": + main() From 24abcf3759062eb0b173c719294b04fe00fd6daa Mon Sep 17 00:00:00 2001 From: balaji Date: Tue, 4 Aug 2026 09:02:07 -0700 Subject: [PATCH 2/6] fix(ci): sample every run in ci-health, add a build-health dashboard The duration report pulled `actions/runs?per_page=100` and filtered by workflow name in Python, so it sampled the last 100 runs across all workflows and kept whichever happened to be bazel. In practice that was 14 runs out of 1275, and it under-reported worse as the repo got busier. It now reads the per-workflow endpoint and pages through it. That truncation also produced a false conclusion: with only two days of runs visible there appeared to be no history to trend against. There are three full weeks. Adds `--dashboard`, which writes a self-contained HTML report and opens it. No CDN and no JavaScript dependency; the charts are inline SVG, so the file works offline and adds nothing to the dependency surface. The report separates queue wait from execution time, because the Actions UI shows only their sum. On current data that distinction matters: several matrix rows spend more wall clock waiting for a runner than building. Also drops the oldest week from the trend when the history window was truncated. Only the tail of that week is held, so its median came from an arbitrary slice and plotted as a misleading near-zero point. Adds focused tests for the pure analysis functions, one per defect the tool has actually shipped with. Co-authored-by: Balaji Ganesan --- tools/ci/ci-health | 672 +++++++++++++++++++++++++++++++++------- tools/ci/test-ci-health | 239 ++++++++++++++ 2 files changed, 804 insertions(+), 107 deletions(-) create mode 100755 tools/ci/test-ci-health diff --git a/tools/ci/ci-health b/tools/ci/ci-health index 325a96c24..78447bf07 100755 --- a/tools/ci/ci-health +++ b/tools/ci/ci-health @@ -13,182 +13,640 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Report GitHub Actions cache and workflow-duration health for this repository. +"""Answer "why is my build slow" for a GitHub Actions repository. -Answers, without hand-rolling `gh api` calls each time: +A workflow's wall-clock time is not one number, it is a stack: - how full is the cache, and what is filling it - which cache keys are duplicated across refs (merge-queue refs double them) - how long does a workflow take, by day, median and p90 - how long do PRs sit between opening and merging + wall clock = queue wait + critical-path job + gate jobs -The cache quota is 10 GB per repository and GitHub evicts least-recently-used -entries once it is reached, so a single oversized family can silently evict the -entries every other job depends on. That failure mode is invisible in a green -pipeline: jobs still pass, they just stop being fast. +Only the middle term is what people mean by "the build". A run that spends +four minutes waiting for a runner looks identical, in the Actions UI, to a +run that spends four minutes compiling. This tool separates them, finds +which matrix row is the long pole, and reports cache pressure, which is the +usual reason a job that used to be fast no longer is. Usage: - tools/ci/ci-health # cache report (the usual question) - tools/ci/ci-health --workflow bazel # duration percentiles by day - tools/ci/ci-health --merge-times # PR open -> merge latency + tools/ci/ci-health --dashboard # visual report, opens in a browser + tools/ci/ci-health --why # same findings, as text + tools/ci/ci-health # cache and quota only + tools/ci/ci-health --durations # duration percentiles by week + tools/ci/ci-health --merge-times # PR open -> merge latency tools/ci/ci-health --all - tools/ci/ci-health --repo OWNER/NAME # defaults to NVIDIA/nvcf + tools/ci/ci-health --workflow release-tags.yml # default: bazel.yml + tools/ci/ci-health --repo OWNER/NAME # default: NVIDIA/nvcf + +Sampling: trends read every retained run of the workflow (cheap, paginated). +Per-job analysis reads jobs for the most recent --runs runs (default 60), +since that costs one API call each. Requires the `gh` CLI, authenticated. """ import argparse import datetime as dt +import html import json +import math +import os import subprocess import sys +import webbrowser from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor QUOTA_GB = 10.0 +DEFAULT_WORKFLOW = "bazel.yml" + +# Jobs that gate or fan out the matrix rather than doing build work. They are +# reported separately so they do not distort the per-subtree picture. +GATE_HINTS = ("detect changed", "required checks", "collect", "summary") -def gh(path, repo, paginate=False): +def gh_json(path, repo): cmd = ["gh", "api", f"repos/{repo}/{path}"] - if paginate: - cmd.append("--paginate") out = subprocess.run(cmd, capture_output=True, text=True) if out.returncode != 0: - sys.exit(f"gh api {path} failed: {out.stderr.strip()}") + raise RuntimeError(f"gh api {path} failed: {out.stderr.strip()}") return json.loads(out.stdout) +def gh_paged(path, repo, key, limit): + """Page through a list endpoint until `limit` items or the data runs out. + + `key` names the array field for endpoints that wrap their results in an + object; pass None for endpoints that return a bare array. + """ + items, page = [], 1 + sep = "&" if "?" in path else "?" + while len(items) < limit: + body = gh_json(f"{path}{sep}per_page=100&page={page}", repo) + batch = body if key is None else body.get(key, []) + if not batch: + break + items.extend(batch) + if len(batch) < 100: + break + page += 1 + return items[:limit] + + +def ts(value): + if not value: + return None + try: + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + def gb(n): return n / 1024 ** 3 -def family(key): - """Group keys by their stable prefix, dropping the trailing content hash.""" - head = key.rsplit("-", 1)[0] - return head[:40] +def pct(vals, p): + """Linear-interpolated percentile. Plain indexing is too coarse at p90.""" + if not vals: + return 0.0 + s = sorted(vals) + if len(s) == 1: + return s[0] + k = (len(s) - 1) * p + lo, hi = math.floor(k), math.ceil(k) + if lo == hi: + return s[int(k)] + return s[lo] * (hi - k) + s[hi] * (k - lo) -def cache_report(repo): - usage = gh("actions/cache/usage", repo) - total = gb(usage["active_caches_size_in_bytes"]) - count = usage["active_caches_count"] - pct = total / QUOTA_GB * 100 - print(f"cache: {total:.2f} GB / {QUOTA_GB:.0f} GB across {count} entries ({pct:.0f}% of quota)") - if pct >= 90: - print(" AT QUOTA: new entries are evicting existing ones. Builds still pass, just colder.") - elif pct >= 75: - print(" approaching quota; expect evictions soon") +def is_gate(name): + low = name.lower() + return any(h in low for h in GATE_HINTS) + + +def is_skipped_matrix(job): + # An unexpanded matrix expression means change detection skipped the row. + return job["conclusion"] == "skipped" or "${{" in job["name"] + + +# ----------------------------------------------------------------- collection + + +def fetch_runs(repo, workflow, limit): + path = f"actions/workflows/{workflow}/runs?status=success" + runs = gh_paged(path, repo, "workflow_runs", limit) + out = [] + for r in runs: + start, end = ts(r.get("run_started_at")), ts(r.get("updated_at")) + if not start or not end or end < start: + continue + out.append( + { + "id": r["id"], + "start": start, + "minutes": (end - start).total_seconds() / 60, + "branch": r.get("head_branch") or "", + } + ) + return out + + +def fetch_jobs(repo, runs, workers=8): + """Fetch jobs for each run concurrently. Failures are dropped, not fatal.""" + + def one(run): + try: + return gh_json(f"actions/runs/{run['id']}/jobs?per_page=100", repo).get("jobs", []) + except RuntimeError: + return [] + + with ThreadPoolExecutor(max_workers=workers) as pool: + return [j for batch in pool.map(one, runs) for j in batch] - caches = gh("actions/caches?per_page=100", repo).get("actions_caches", []) - if not caches: - return - sizes = defaultdict(float) - counts = defaultdict(int) +# ------------------------------------------------------------------- analysis + + +def analyse_jobs(jobs): + """Split each job into queue wait and execution, grouped by job name.""" + stats = defaultdict(lambda: {"queue": [], "exec": [], "runs": 0, "skipped": 0, "hosted": 0}) + for j in jobs: + name = j["name"] + s = stats[name] + if is_skipped_matrix(j): + s["skipped"] += 1 + continue + created, started, done = ts(j.get("created_at")), ts(j.get("started_at")), ts(j.get("completed_at")) + if not (created and started and done): + continue + s["runs"] += 1 + s["queue"].append(max(0.0, (started - created).total_seconds() / 60)) + s["exec"].append(max(0.0, (done - started).total_seconds() / 60)) + if (j.get("runner_name") or "").startswith("GitHub Actions"): + s["hosted"] += 1 + return stats + + +def long_poles(jobs): + """How often each job is the last one to finish in its run.""" + by_run = defaultdict(list) + for j in jobs: + if is_skipped_matrix(j) or is_gate(j["name"]): + continue + done = ts(j.get("completed_at")) + if done: + by_run[j["run_id"]].append((done, j["name"])) + tally = defaultdict(int) + for entries in by_run.values(): + if entries: + tally[max(entries)[1]] += 1 + return tally, len(by_run) + + +def cache_state(repo): + usage = gh_json("actions/cache/usage", repo) + caches = gh_json("actions/caches?per_page=100", repo).get("actions_caches", []) + families, counts = defaultdict(float), defaultdict(int) for c in caches: - f = family(c["key"]) - sizes[f] += gb(c["size_in_bytes"]) - counts[f] += 1 - - print("\nlargest families:") - for f, size in sorted(sizes.items(), key=lambda kv: -kv[1])[:8]: - share = size / QUOTA_GB * 100 - print(f" {size:6.2f} GB {share:4.1f}% of quota x{counts[f]:<3d} {f}") - - # A key present on more than one ref is stored more than once. Merge-queue - # refs are the usual cause and are the least useful copy: the branch is - # deleted when the queue drains, so that entry can never be restored. + fam = c["key"].rsplit("-", 1)[0][:40] + families[fam] += gb(c["size_in_bytes"]) + counts[fam] += 1 refs = defaultdict(list) for c in caches: refs[c["key"]].append(c["ref"]) dupes = {k: v for k, v in refs.items() if len(v) > 1} - if dupes: - wasted = sum( - gb(c["size_in_bytes"]) - for c in caches - if c["key"] in dupes and "gh-readonly-queue" in c["ref"] + stranded = sum( + gb(c["size_in_bytes"]) + for c in caches + if c["key"] in dupes and "gh-readonly-queue" in c["ref"] + ) + return { + "total": gb(usage["active_caches_size_in_bytes"]), + "count": usage["active_caches_count"], + "families": families, + "counts": counts, + "dupes": dupes, + "stranded": stranded, + } + + +def diagnose(runs, stats, poles, pole_runs, cache): + """Rank the causes of slowness, largest measured contribution first.""" + causes = [] + wall = pct([r["minutes"] for r in runs], 0.5) if runs else 0.0 + + build = {n: s for n, s in stats.items() if not is_gate(n) and s["runs"]} + queues = [q for s in build.values() for q in s["queue"]] + if queues and wall: + q50, q90 = pct(queues, 0.5), pct(queues, 0.9) + share = q50 / wall * 100 + if share >= 10 or q90 >= 2: + causes.append( + ( + share, + "Runner queue wait", + f"Jobs wait a median {q50:.1f} min (p90 {q90:.1f} min) for a runner " + f"before executing, {share:.0f}% of the {wall:.1f} min median run.", + "Add runner capacity or reduce concurrent matrix width.", + ) + ) + + if poles and pole_runs: + name, hits = max(poles.items(), key=lambda kv: kv[1]) + s = build.get(name) + if s and s["exec"]: + e50 = pct(s["exec"], 0.5) + share = e50 / wall * 100 if wall else 0.0 + causes.append( + ( + share, + f"Critical path: {name}", + f"Finishes last in {hits}/{pole_runs} runs ({hits / pole_runs * 100:.0f}%), " + f"median {e50:.1f} min. Every other job waits on it.", + "Nothing below this job's runtime is achievable; split or cache it.", + ) + ) + + quota = cache["total"] / QUOTA_GB * 100 + if quota >= 75: + causes.append( + ( + quota / 4, + "Cache pressure", + f"{cache['total']:.2f} GB of {QUOTA_GB:.0f} GB used ({quota:.0f}%). " + f"GitHub evicts least-recently-used entries at quota, so jobs " + f"silently rebuild from cold.", + f"{cache['stranded']:.2f} GB sits on merge-queue refs that can never be restored." + if cache["stranded"] + else "Trim the largest cache family.", + ) ) - print(f"\nkeys stored under multiple refs: {len(dupes)}") - if wasted: - print(f" {wasted:.2f} GB of that is on merge-queue refs, which are unrestorable") - for k, rs in list(dupes.items())[:5]: - short = [r.replace("refs/heads/", "") for r in rs] - print(f" {k[:46]}\n {short}") + skipped = sum(s["skipped"] for s in stats.values()) + total_slots = skipped + sum(s["runs"] for s in stats.values()) + if total_slots and skipped / total_slots > 0.3: + causes.append( + ( + 0.0, + "Note: change detection is skipping rows", + f"{skipped}/{total_slots} matrix slots ({skipped / total_slots * 100:.0f}%) were " + f"skipped. Medians look fast because work was avoided, not accelerated.", + "Compare p90 and run counts, not the median alone.", + ) + ) -def percentile(values, p): - if not values: - return 0.0 - s = sorted(values) - idx = min(int(len(s) * p), len(s) - 1) - return s[idx] + causes.sort(key=lambda c: -c[0]) + return causes, wall -def workflow_report(repo, name, days): - runs = gh(f"actions/runs?per_page=100&status=success", repo).get("workflow_runs", []) - rows = defaultdict(list) +def weekly(runs, truncated=False): + """Median/p90 per ISO week, oldest first. + + When the history window was truncated we only hold the tail of the oldest + week, so its median is computed from an arbitrary slice of that week and is + not comparable to the others. Drop it rather than plot a misleading point. + """ + buckets = defaultdict(list) for r in runs: - if r.get("name") != name: - continue - try: - start = dt.datetime.fromisoformat(r["run_started_at"].replace("Z", "+00:00")) - end = dt.datetime.fromisoformat(r["updated_at"].replace("Z", "+00:00")) - except (KeyError, ValueError): - continue - rows[start.date().isoformat()].append((end - start).total_seconds() / 60) + y, w, _ = r["start"].isocalendar() + buckets[f"{y}-W{w:02d}"].append(r["minutes"]) + weeks = sorted(buckets) + if truncated and len(weeks) > 1: + weeks = weeks[1:] + return [(k, pct(buckets[k], 0.5), pct(buckets[k], 0.9), len(buckets[k])) for k in weeks] + + +# ------------------------------------------------------------------------ svg + + +def esc(s): + return html.escape(str(s), quote=True) + +def line_chart(series, width=860, height=272, pad=46): + """series: list of (label, p50, p90, count).""" + if not series: + return "

no data

" + top = max(max(p90 for _, _, p90, _ in series), 1.0) * 1.15 + n = len(series) + span = width - pad * 2 + step = span / max(n - 1, 1) + + def pt(i, v): + return pad + i * step, height - pad - (v / top) * (height - pad * 2) + + def path(idx): + return " ".join( + f"{'M' if i == 0 else 'L'}{pt(i, s[idx])[0]:.1f},{pt(i, s[idx])[1]:.1f}" + for i, s in enumerate(series) + ) + + parts = [f""] + for g in range(5): + y = pad + g * (height - pad * 2) / 4 + val = top * (1 - g / 4) + parts.append(f"") + parts.append(f"{val:.0f}m") + parts.append(f"") + parts.append(f"") + for i, (label, p50, p90, cnt) in enumerate(series): + x, y = pt(i, p50) + parts.append( + f"" + f"{esc(label)}: median {p50:.1f} min, p90 {p90:.1f} min, {cnt} runs" + ) + if n <= 14 or i % max(1, n // 10) == 0: + parts.append( + f"{esc(label[-3:])}" + ) + # Run count sits under the label so a thin week is never mistaken + # for a real speedup. + parts.append( + f"n={cnt}" + ) + parts.append("") + return "".join(parts) + + +def stacked_bars(rows, width=860, bar=26, pad=210): + """rows: list of (name, queue_min, exec_min, runs, skipped).""" if not rows: - print(f"\nworkflow '{name}': no successful runs in the retained window") - return - print(f"\nworkflow '{name}' (successful runs only, GitHub retains a limited window):") - for day in sorted(rows)[-days:]: - v = rows[day] - print( - f" {day} runs={len(v):<4d} median={percentile(v, 0.5):6.1f} min" - f" p90={percentile(v, 0.9):6.1f} min max={max(v):6.1f} min" + return "

no data

" + top = max(q + e for _, q, e, _, _ in rows) or 1.0 + span = width - pad - 90 + height = len(rows) * (bar + 8) + 16 + parts = [f""] + for i, (name, q, e, runs, skipped) in enumerate(rows): + y = 8 + i * (bar + 8) + qw = q / top * span + ew = e / top * span + short = name if len(name) <= 30 else name[:29] + "…" + parts.append(f"{esc(short)}") + parts.append( + f"" + f"{esc(name)}: queue {q:.1f} min" + ) + parts.append( + f"" + f"{esc(name)}: execute {e:.1f} min over {runs} runs, {skipped} skipped" + ) + parts.append( + f"{q + e:.1f}m" ) + parts.append("") + return "".join(parts) + + +def quota_bar(cache): + used = min(cache["total"] / QUOTA_GB, 1.0) * 100 + cls = "over" if used >= 90 else ("warn" if used >= 75 else "ok") + return ( + f"
" + f"{cache['total']:.2f} GB / {QUOTA_GB:.0f} GB " + f"({used:.0f}%) across {cache['count']} entries
" + ) + + +CSS = """ +:root{--bg:#0f1115;--panel:#171a21;--line:#252a34;--fg:#e6e9ef;--dim:#98a1b3;--nv:#76b900;--warn:#e8b339;--bad:#e05252} +*{box-sizing:border-box} +body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif} +.wrap{max-width:960px;margin:0 auto;padding:34px 22px 70px} +h1{font-size:23px;margin:0 0 4px} +h2{font-size:15px;margin:0 0 14px;letter-spacing:.03em;text-transform:uppercase;color:var(--dim);font-weight:600} +.sub{color:var(--dim);margin:0 0 26px;font-size:13px} +.card{background:var(--panel);border:1px solid var(--line);border-radius:9px;padding:20px 22px;margin-bottom:18px} +.cause{border-left:3px solid var(--nv);padding:11px 0 11px 15px;margin-bottom:15px} +.cause:last-child{margin-bottom:0} +.cause .t{font-weight:600;margin-bottom:3px} +.cause .d{color:var(--dim)} +.cause .f{color:var(--nv);margin-top:5px;font-size:13px} +.cause.note{border-left-color:var(--warn)} +.cause.note .f{color:var(--warn)} +.stats{display:flex;gap:34px;flex-wrap:wrap;margin-bottom:6px} +.stat .v{font-size:26px;font-weight:600} +.stat .k{color:var(--dim);font-size:12px;text-transform:uppercase;letter-spacing:.05em} +.chart{width:100%;height:auto} +.grid{stroke:var(--line);stroke-width:1} +.l50{fill:none;stroke:var(--nv);stroke-width:2.2} +.l90{fill:none;stroke:#3d6ea8;stroke-width:1.6;stroke-dasharray:5 4} +.dot{fill:var(--nv)} +.ylab{fill:var(--dim);font-size:11px;text-anchor:end} +.xlab{fill:var(--dim);font-size:10px;text-anchor:middle} +.xcnt{fill:#5d6675;font-size:9px;text-anchor:middle} +.rowlab{fill:var(--fg);font-size:12px;text-anchor:end} +.rowval{fill:var(--dim);font-size:11px} +.bq{fill:var(--warn)} +.be{fill:var(--nv)} +.legend{color:var(--dim);font-size:12px;margin-top:10px} +.legend i{display:inline-block;width:10px;height:10px;border-radius:2px;margin:0 5px 0 16px;vertical-align:middle} +.legend i:first-child{margin-left:0} +.quota{position:relative;background:#22262f;border-radius:5px;height:32px;overflow:hidden} +.quotafill{height:100%} +.quotafill.ok{background:var(--nv)}.quotafill.warn{background:var(--warn)}.quotafill.over{background:var(--bad)} +.quotatxt{position:absolute;left:12px;top:7px;font-size:13px;font-weight:600;text-shadow:0 1px 3px rgba(0,0,0,.75)} +table{width:100%;border-collapse:collapse;font-size:13px} +th{text-align:left;color:var(--dim);font-weight:600;padding:7px 8px;border-bottom:1px solid var(--line)} +td{padding:7px 8px;border-bottom:1px solid var(--line)} +td.n{text-align:right;font-variant-numeric:tabular-nums} +.empty{color:var(--dim)} +code{background:#22262f;padding:1px 5px;border-radius:3px;font-size:12px} +""" + + +def render_html(repo, workflow, runs, stats, poles, pole_runs, cache, causes, wall, + generated, truncated=False): + build = sorted( + ( + (n, pct(s["queue"], 0.5), pct(s["exec"], 0.5), s["runs"], s["skipped"]) + for n, s in stats.items() + if s["runs"] and not is_gate(n) + ), + key=lambda r: -(r[1] + r[2]), + )[:16] + + cause_html = "".join( + f"
" + f"
{esc(t)}
{esc(d)}
{esc(f)}
" + for _, t, d, f in causes + ) or "

Nothing is dominating the wall clock right now.

" + + fam_rows = "".join( + f"{esc(f)}{sz:.2f} GB" + f"{sz / QUOTA_GB * 100:.0f}%{cache['counts'][f]}" + for f, sz in sorted(cache["families"].items(), key=lambda kv: -kv[1])[:8] + ) + + pole_rows = "".join( + f"{esc(n)}{c}" + f"{c / pole_runs * 100:.0f}%" + for n, c in sorted(poles.items(), key=lambda kv: -kv[1])[:8] + ) if pole_runs else "" + + weeks = weekly(runs, truncated) + span = f"{runs[-1]['start'].date()} to {runs[0]['start'].date()}" if runs else "no runs" + + return f""" + +Build health: {esc(repo)}
+

Why is my build slow?

+

{esc(repo)} · workflow {esc(workflow)} · +{len(runs)} successful runs ({esc(span)}) · per-job detail from the most recent +{pole_runs} runs · generated {esc(generated)}

+ +

Ranked causes

{cause_html}
+ +

Wall clock

+
+
{wall:.1f} min
median run
+
{pct([r['minutes'] for r in runs], 0.9):.1f} min
p90 run
+
{len(weeks)}
weeks of history
+
+{line_chart(weeks)} +
medianp90 +· hover a point for the week and run count
+ +

Where each job's time goes

+{stacked_bars(build)} +
waiting for a runner +executing +· median per job; gate jobs excluded
+ +{"

What finishes last

" + pole_rows + "
JobRuns where it was lastShare
" if pole_rows else ""} + +

Cache

+{quota_bar(cache)} +

Largest families. GitHub allows {QUOTA_GB:.0f} GB per +repository and evicts least-recently-used entries once full.

+ +{fam_rows}
FamilySizeQuotaCopies
+{"

" + f"{cache['stranded']:.2f} GB is stored on gh-readonly-queue refs. Those branches are deleted when the merge queue drains, so the entries can never be restored but still count against quota." + "

" if cache["stranded"] else ""} +
+
""" + + +# ----------------------------------------------------------------- text views + + +def cache_report(cache): + quota = cache["total"] / QUOTA_GB * 100 + print(f"cache: {cache['total']:.2f} GB / {QUOTA_GB:.0f} GB across {cache['count']} entries ({quota:.0f}% of quota)") + if quota >= 90: + print(" AT QUOTA: new entries are evicting existing ones. Builds still pass, just colder.") + elif quota >= 75: + print(" approaching quota; expect evictions soon") + if cache["families"]: + print("\nlargest families:") + for f, size in sorted(cache["families"].items(), key=lambda kv: -kv[1])[:8]: + print(f" {size:6.2f} GB {size / QUOTA_GB * 100:4.1f}% of quota x{cache['counts'][f]:<3d} {f}") + if cache["dupes"]: + print(f"\nkeys stored under multiple refs: {len(cache['dupes'])}") + if cache["stranded"]: + print(f" {cache['stranded']:.2f} GB of that is on merge-queue refs, which are unrestorable") + + +def why_report(causes, wall, runs): + print(f"\nmedian run {wall:.1f} min, p90 {pct([r['minutes'] for r in runs], 0.9):.1f} min " + f"over {len(runs)} successful runs\n") + if not causes: + print("nothing is dominating the wall clock right now") + return + for i, (_, title, detail, fix) in enumerate(causes, 1): + print(f"{i}. {title}\n {detail}\n -> {fix}\n") + + +def workflow_report(runs, workflow, weeks_shown, truncated=False): + if not runs: + print(f"\nworkflow '{workflow}': no successful runs in the retained window") + return + print(f"\nworkflow '{workflow}' (successful runs only), by week:") + for label, p50, p90, n in weekly(runs, truncated)[-weeks_shown:]: + print(f" {label} runs={n:<5d} median={p50:6.1f} min p90={p90:6.1f} min") print(" NOTE: a low median can mean rows were skipped by change detection,") print(" not that builds got faster. Compare p90 and run counts too.") def merge_time_report(repo, limit): - prs = gh(f"pulls?state=closed&per_page={limit}&sort=updated&direction=desc", repo) - lat = [] - for p in prs: - if not p.get("merged_at"): - continue - created = dt.datetime.fromisoformat(p["created_at"].replace("Z", "+00:00")) - merged = dt.datetime.fromisoformat(p["merged_at"].replace("Z", "+00:00")) - lat.append(((merged - created).total_seconds() / 3600, p["number"])) + prs = gh_paged("pulls?state=closed&sort=updated&direction=desc", repo, "items", limit) \ + if limit > 100 else gh_json( + f"pulls?state=closed&per_page={limit}&sort=updated&direction=desc", repo) + lat = [ + ((ts(p["merged_at"]) - ts(p["created_at"])).total_seconds() / 3600, p["number"]) + for p in prs + if p.get("merged_at") + ] if not lat: print("\nno merged PRs in the sampled window") return hours = [h for h, _ in lat] print(f"\nPR open -> merge, {len(lat)} merged PRs sampled:") - print(f" median {percentile(hours, 0.5):.1f} h p90 {percentile(hours, 0.9):.1f} h") - slowest = sorted(lat, reverse=True)[:5] + print(f" median {pct(hours, 0.5):.1f} h p90 {pct(hours, 0.9):.1f} h") print(" slowest:") - for h, n in slowest: + for h, n in sorted(lat, reverse=True)[:5]: print(f" #{n:<6d} {h:8.1f} h") +# ------------------------------------------------------------------------ cli + + def main(): - ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) ap.add_argument("--repo", default="NVIDIA/nvcf") - ap.add_argument("--workflow", help="report duration percentiles for this workflow name") - ap.add_argument("--merge-times", action="store_true", help="report PR open-to-merge latency") - ap.add_argument("--all", action="store_true", help="cache + bazel durations + merge times") - ap.add_argument("--days", type=int, default=10) + ap.add_argument("--workflow", default=DEFAULT_WORKFLOW, + help="workflow file name, e.g. bazel.yml (default: %(default)s)") + ap.add_argument("--dashboard", nargs="?", const="build-health.html", metavar="FILE", + help="write a self-contained HTML dashboard and open it") + ap.add_argument("--why", action="store_true", help="rank the causes of slowness as text") + ap.add_argument("--durations", action="store_true", help="duration percentiles by week") + ap.add_argument("--merge-times", action="store_true", help="PR open-to-merge latency") + ap.add_argument("--all", action="store_true") + ap.add_argument("--runs", type=int, default=60, help="runs to pull per-job detail for") + ap.add_argument("--history", type=int, default=1000, help="runs to trend over") + ap.add_argument("--weeks", type=int, default=12) ap.add_argument("--prs", type=int, default=100) + ap.add_argument("--no-open", action="store_true", help="do not launch a browser") args = ap.parse_args() - if args.all or (not args.workflow and not args.merge_times): - cache_report(args.repo) - if args.workflow: - workflow_report(args.repo, args.workflow, args.days) - elif args.all: - workflow_report(args.repo, "bazel", args.days) + wf = args.workflow if args.workflow.endswith(".yml") else f"{args.workflow}.yml" + needs_jobs = bool(args.dashboard) or args.why or args.all + needs_runs = needs_jobs or args.durations + + try: + runs = fetch_runs(args.repo, wf, args.history) if needs_runs else [] + if needs_runs and not runs: + sys.exit(f"no successful runs found for workflow '{wf}' in {args.repo}") + truncated = len(runs) >= args.history + jobs = fetch_jobs(args.repo, runs[: args.runs]) if needs_jobs else [] + cache = cache_state(args.repo) + except RuntimeError as e: + sys.exit(str(e)) + + stats = analyse_jobs(jobs) + poles, pole_runs = long_poles(jobs) + causes, wall = diagnose(runs, stats, poles, pole_runs, cache) if needs_jobs else ([], 0.0) + + if args.dashboard: + generated = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + out = os.path.abspath(args.dashboard) + with open(out, "w") as fh: + fh.write(render_html(args.repo, wf, runs, stats, poles, pole_runs, + cache, causes, wall, generated, truncated)) + print(f"wrote {out}") + if not args.no_open: + webbrowser.open(f"file://{out}") + return + + if args.why or args.all: + why_report(causes, wall, runs) + if args.durations or args.all: + workflow_report(runs, wf, args.weeks, truncated) if args.merge_times or args.all: merge_time_report(args.repo, args.prs) + if not (args.why or args.durations or args.merge_times): + cache_report(cache) + elif args.all: + print() + cache_report(cache) if __name__ == "__main__": diff --git a/tools/ci/test-ci-health b/tools/ci/test-ci-health new file mode 100755 index 000000000..a19133b24 --- /dev/null +++ b/tools/ci/test-ci-health @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for tools/ci/ci-health. + +Every case here corresponds to a defect the tool actually shipped with at +some point. The analysis functions are pure, so none of this touches the +network. + +Run: tools/ci/test-ci-health +""" + +import datetime as dt +import importlib.util +import os +import unittest + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_spec = importlib.util.spec_from_loader( + "ci_health", + importlib.machinery.SourceFileLoader("ci_health", os.path.join(_HERE, "ci-health")), +) +ci = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(ci) + + +def at(day, hour=0, minute=0): + return dt.datetime(2026, 7, day, hour, minute, tzinfo=dt.timezone.utc) + + +def iso(day, hour=0, minute=0, second=0): + return f"2026-07-{day:02d}T{hour:02d}:{minute:02d}:{second:02d}Z" + + +def job(name, created, started, completed, conclusion="success", run_id=1, runner="self"): + return { + "name": name, + "created_at": created, + "started_at": started, + "completed_at": completed, + "conclusion": conclusion, + "run_id": run_id, + "runner_name": runner, + } + + +class Percentile(unittest.TestCase): + def test_interpolates_between_samples(self): + # Index-truncation would return 10.0 here; the real p90 is 9.1. + self.assertAlmostEqual(ci.pct([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 0.9), 9.1) + + def test_median_of_even_length_is_the_midpoint(self): + self.assertAlmostEqual(ci.pct([1, 2, 3, 4], 0.5), 2.5) + + def test_degenerate_inputs(self): + self.assertEqual(ci.pct([], 0.5), 0.0) + self.assertEqual(ci.pct([7.5], 0.9), 7.5) + + +class Weekly(unittest.TestCase): + """The oldest bucket is only a slice of its week when history was cut.""" + + def runs(self): + # W28 has a single fast run (the truncated tail); W29 and W30 are whole. + return ( + [{"start": at(6, 9), "minutes": 1.0}] + + [{"start": at(13, 9), "minutes": 10.0} for _ in range(5)] + + [{"start": at(20, 9), "minutes": 20.0} for _ in range(5)] + ) + + def test_keeps_every_week_when_not_truncated(self): + weeks = ci.weekly(self.runs(), truncated=False) + self.assertEqual([w[0] for w in weeks], ["2026-W28", "2026-W29", "2026-W30"]) + self.assertEqual(weeks[0][1], 1.0) + + def test_drops_the_partial_oldest_week_when_truncated(self): + weeks = ci.weekly(self.runs(), truncated=True) + self.assertEqual([w[0] for w in weeks], ["2026-W29", "2026-W30"]) + + def test_never_drops_the_only_week(self): + one = [{"start": at(20, 9), "minutes": 4.0}] + self.assertEqual(len(ci.weekly(one, truncated=True)), 1) + + def test_reports_run_counts(self): + self.assertEqual([w[3] for w in ci.weekly(self.runs(), truncated=True)], [5, 5]) + + +class Classification(unittest.TestCase): + def test_unexpanded_matrix_expression_counts_as_skipped(self): + # A skipped matrix row keeps its raw expression as the job name. + self.assertTrue( + ci.is_skipped_matrix( + {"name": "bazel (${{ matrix.subtree.id }})", "conclusion": "success"} + ) + ) + + def test_explicit_skip_counts_as_skipped(self): + self.assertTrue(ci.is_skipped_matrix({"name": "bazel (nvca)", "conclusion": "skipped"})) + + def test_real_row_is_not_skipped(self): + self.assertFalse(ci.is_skipped_matrix({"name": "bazel (nvca)", "conclusion": "success"})) + + def test_gate_jobs_are_recognised(self): + self.assertTrue(ci.is_gate("detect changed subtrees")) + self.assertTrue(ci.is_gate("bazel required checks")) + self.assertFalse(ci.is_gate("bazel (nvca)")) + + +class AnalyseJobs(unittest.TestCase): + def test_splits_queue_from_execution(self): + stats = ci.analyse_jobs( + [job("bazel (nvca)", iso(20, 10, 0), iso(20, 10, 2), iso(20, 10, 12))] + ) + s = stats["bazel (nvca)"] + self.assertEqual(s["runs"], 1) + self.assertAlmostEqual(s["queue"][0], 2.0) + self.assertAlmostEqual(s["exec"][0], 10.0) + + def test_skipped_rows_are_counted_not_timed(self): + stats = ci.analyse_jobs( + [job("bazel (nvca)", iso(20), iso(20), iso(20), conclusion="skipped")] + ) + self.assertEqual(stats["bazel (nvca)"]["skipped"], 1) + self.assertEqual(stats["bazel (nvca)"]["runs"], 0) + + def test_missing_timestamps_are_dropped(self): + stats = ci.analyse_jobs([job("bazel (nvca)", iso(20), None, iso(20, 1))]) + self.assertEqual(stats["bazel (nvca)"]["runs"], 0) + + def test_hosted_runners_are_tallied(self): + stats = ci.analyse_jobs( + [job("gate", iso(20, 1), iso(20, 1), iso(20, 2), runner="GitHub Actions 12")] + ) + self.assertEqual(stats["gate"]["hosted"], 1) + + +class LongPoles(unittest.TestCase): + def test_picks_the_last_finishing_build_job(self): + jobs = [ + job("bazel (fast)", iso(20, 10), iso(20, 10), iso(20, 10, 30), run_id=1), + job("bazel (slow)", iso(20, 10), iso(20, 10), iso(20, 10, 50), run_id=1), + ] + tally, runs = ci.long_poles(jobs) + self.assertEqual(runs, 1) + self.assertEqual(dict(tally), {"bazel (slow)": 1}) + + def test_gate_jobs_never_count_as_the_long_pole(self): + # The required-checks gate finishes last by construction; it is not + # the reason the build is slow. + jobs = [ + job("bazel (slow)", iso(20, 10), iso(20, 10), iso(20, 10, 50), run_id=1), + job("bazel required checks", iso(20, 10), iso(20, 10), iso(20, 11), run_id=1), + ] + tally, _ = ci.long_poles(jobs) + self.assertEqual(dict(tally), {"bazel (slow)": 1}) + + def test_tallies_across_runs_independently(self): + jobs = [ + job("bazel (a)", iso(20, 10), iso(20, 10), iso(20, 10, 50), run_id=1), + job("bazel (b)", iso(20, 10), iso(20, 10), iso(20, 10, 30), run_id=1), + job("bazel (b)", iso(21, 10), iso(21, 10), iso(21, 10, 50), run_id=2), + job("bazel (a)", iso(21, 10), iso(21, 10), iso(21, 10, 30), run_id=2), + ] + tally, runs = ci.long_poles(jobs) + self.assertEqual(runs, 2) + self.assertEqual(dict(tally), {"bazel (a)": 1, "bazel (b)": 1}) + + +class Diagnose(unittest.TestCase): + def cache(self, total=1.0, stranded=0.0): + return {"total": total, "count": 3, "families": {}, "counts": {}, + "dupes": {}, "stranded": stranded} + + def test_flags_cache_pressure_at_quota(self): + runs = [{"start": at(20), "minutes": 10.0}] + causes, _ = ci.diagnose(runs, {}, {}, 0, self.cache(total=9.5, stranded=1.9)) + titles = [c[1] for c in causes] + self.assertIn("Cache pressure", titles) + + def test_quiet_when_everything_is_healthy(self): + runs = [{"start": at(20), "minutes": 10.0}] + stats = ci.analyse_jobs( + [job("bazel (nvca)", iso(20, 10), iso(20, 10), iso(20, 10, 30))] + ) + causes, wall = ci.diagnose(runs, stats, {}, 0, self.cache(total=1.0)) + self.assertEqual(causes, []) + self.assertAlmostEqual(wall, 10.0) + + def test_warns_when_change_detection_skipped_most_rows(self): + runs = [{"start": at(20), "minutes": 10.0}] + jobs = [ + job(f"bazel (s{i})", iso(20), iso(20), iso(20), conclusion="skipped") + for i in range(9) + ] + [job("bazel (real)", iso(20, 10), iso(20, 10), iso(20, 10, 30))] + causes, _ = ci.diagnose(runs, ci.analyse_jobs(jobs), {}, 0, self.cache()) + self.assertTrue(any(c[1].startswith("Note") for c in causes), + "a 90% skip rate must be called out") + + def test_ranks_the_largest_contributor_first(self): + runs = [{"start": at(20), "minutes": 20.0}] + # A job that waits 10 min for a runner and executes for 1. + jobs = [job("bazel (a)", iso(20, 10, 0), iso(20, 10, 10), iso(20, 10, 11))] + causes, _ = ci.diagnose(runs, ci.analyse_jobs(jobs), {}, 0, self.cache()) + self.assertEqual(causes[0][1], "Runner queue wait") + + +class Rendering(unittest.TestCase): + def test_job_names_with_matrix_syntax_are_escaped(self): + svg = ci.stacked_bars([("bazel ()", 1.0, 2.0, 3, 0)]) + self.assertNotIn("", svg) + self.assertIn("&", svg) + + def test_charts_handle_no_data(self): + self.assertIn("no data", ci.line_chart([])) + self.assertIn("no data", ci.stacked_bars([])) + + def test_dashboard_has_no_external_references(self): + cache = {"total": 1.0, "count": 1, "families": {"fam": 1.0}, + "counts": {"fam": 1}, "dupes": {}, "stranded": 0.0} + runs = [{"start": at(20), "minutes": 5.0}] + out = ci.render_html("o/r", "bazel.yml", runs, {}, {}, 0, cache, [], 5.0, "now") + for marker in ("http://", "https://", "src="): + self.assertNotIn(marker, out, f"dashboard must stay self-contained ({marker})") + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 32fbfd1854dc9a5760a443c67e04c697baa6f6ca Mon Sep 17 00:00:00 2001 From: balaji Date: Tue, 4 Aug 2026 09:02:07 -0700 Subject: [PATCH 3/6] perf(ci): raise the bazel matrix cap from 8 to 12 A 25-row merge-queue run needs three waves at max-parallel 8 and two at 12, and full-matrix runs are what set the p90 (20.1 min against a 6.8 min median). The cap exists to keep simultaneous actions/checkout downloads under GitHub's rate limit, which previously failed runs at "Set up job" with HTTP 429. That constraint is unchanged, so this is a measured step rather than a jump to 16: combined with the bazel-docker matrix the checkout burst goes from 12 to 16. Walk this back first if 429s reappear. Co-authored-by: Balaji Ganesan --- .github/workflows/bazel.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 1ed0b7a12..8d022850a 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -439,7 +439,14 @@ jobs: # same instant, tripping GitHub's action-download rate limit (HTTP 429 in # "Set up job"). Batching keeps the download burst under that limit; # change-aware scheduling already keeps most PRs well below the cap. - max-parallel: 8 + # + # Raised 8 -> 12 to shorten full-matrix runs: a recent 25-row merge-queue + # run needed three waves at 8 and needs two at 12. The bazel-docker matrix + # below contributes 4 more, so simultaneous checkouts peak at 16 rather + # than the previous 12. If "Set up job" starts failing with HTTP 429 this + # is the first knob to walk back; measure with `tools/ci/ci-health --why`, + # which reports queue wait separately from execution time. + max-parallel: 12 matrix: subtree: ${{ fromJSON(needs.detect.outputs.matrix) }} steps: From e8db7054976f866dd8ba622ff3a956f0262ac48a Mon Sep 17 00:00:00 2001 From: balaji Date: Tue, 4 Aug 2026 09:41:52 -0700 Subject: [PATCH 4/6] refactor(ci): rewrite ci-health in Go Review feedback: tools/AGENTS.md prefers Go for non-trivial repo tooling and says to avoid new Python, partly because some CI environments here do not guarantee a Python interpreter. This tool is squarely in that category: structured API parsing, concurrent fetches, and logic that benefits from unit tests. It should not have been Python to begin with. Behaviour is unchanged. The Go build produces the same ranked causes, the same weekly trend, and a visually identical dashboard against live data. Layout follows the existing convention: the tool lives in its own module at tools/ci-health/, with tools/ci/ci-health kept as the stable entrypoint. Two fixes carried over from review of the Python version: - The pulls endpoint returns a bare array, so paging it with a wrapper key crashed once --prs exceeded one page. The Go paging helper takes an empty key for bare-array endpoints, and a test covers both response shapes. - The cache listing read a single page, silently capping the report at 100 entries. It now pages. 37 tests, no third-party dependencies. Co-authored-by: Balaji Ganesan --- tools/ci-health/.gitignore | 2 + tools/ci-health/analysis.go | 308 +++++++++++++++ tools/ci-health/analysis_test.go | 347 ++++++++++++++++ tools/ci-health/github.go | 228 +++++++++++ tools/ci-health/github_test.go | 214 ++++++++++ tools/ci-health/go.mod | 3 + tools/ci-health/main.go | 310 +++++++++++++++ tools/ci-health/render.go | 344 ++++++++++++++++ tools/ci/ci-health | 654 +------------------------------ tools/ci/test-ci-health | 239 ----------- 10 files changed, 1772 insertions(+), 877 deletions(-) create mode 100644 tools/ci-health/.gitignore create mode 100644 tools/ci-health/analysis.go create mode 100644 tools/ci-health/analysis_test.go create mode 100644 tools/ci-health/github.go create mode 100644 tools/ci-health/github_test.go create mode 100644 tools/ci-health/go.mod create mode 100644 tools/ci-health/main.go create mode 100644 tools/ci-health/render.go delete mode 100755 tools/ci/test-ci-health diff --git a/tools/ci-health/.gitignore b/tools/ci-health/.gitignore new file mode 100644 index 000000000..4e691465f --- /dev/null +++ b/tools/ci-health/.gitignore @@ -0,0 +1,2 @@ +# Build output from `go build` in this directory. +/ci-health diff --git a/tools/ci-health/analysis.go b/tools/ci-health/analysis.go new file mode 100644 index 000000000..8fea5fc6e --- /dev/null +++ b/tools/ci-health/analysis.go @@ -0,0 +1,308 @@ +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "math" + "sort" + "strings" +) + +const ( + quotaGB = 10.0 + maxCacheEntries = 2000 +) + +// gateHints name the jobs that gate or fan out the matrix rather than doing +// build work. They are held out so they do not distort the per-subtree picture, +// and so the required-checks job is never blamed as the critical path. +var gateHints = []string{"detect changed", "required checks", "collect", "summary"} + +type JobStat struct { + Queue []float64 + Exec []float64 + Runs int + Skipped int + Hosted int +} + +type Week struct { + Label string + P50 float64 + P90 float64 + Count int +} + +type Cause struct { + Weight float64 + Title string + Detail string + Fix string + Note bool +} + +type PRLatency struct { + Number int + Hours float64 +} + +type CacheState struct { + TotalGB float64 + Count int + Families map[string]float64 + Copies map[string]int + Dupes map[string][]string + StrandedGB float64 +} + +func gb(n int64) float64 { return float64(n) / (1024 * 1024 * 1024) } + +// pctl is a linear-interpolated percentile. Plain index truncation is too +// coarse at p90: for ten ascending samples it just returns the maximum. +func pctl(vals []float64, p float64) float64 { + if len(vals) == 0 { + return 0 + } + s := append([]float64(nil), vals...) + sort.Float64s(s) + if len(s) == 1 { + return s[0] + } + k := float64(len(s)-1) * p + lo, hi := math.Floor(k), math.Ceil(k) + if lo == hi { + return s[int(k)] + } + return s[int(lo)]*(hi-k) + s[int(hi)]*(k-lo) +} + +func isGate(name string) bool { + low := strings.ToLower(name) + for _, h := range gateHints { + if strings.Contains(low, h) { + return true + } + } + return false +} + +// isSkippedMatrix reports whether a job never really ran. A row skipped by +// change detection keeps its unexpanded matrix expression as its name, which is +// the only signal GitHub gives that the row was elided rather than executed. +func isSkippedMatrix(j Job) bool { + return j.Conclusion == "skipped" || strings.Contains(j.Name, "${{") +} + +func analyseJobs(jobs []Job) map[string]*JobStat { + stats := map[string]*JobStat{} + for _, j := range jobs { + s, ok := stats[j.Name] + if !ok { + s = &JobStat{} + stats[j.Name] = s + } + if isSkippedMatrix(j) { + s.Skipped++ + continue + } + if j.CreatedAt == nil || j.StartedAt == nil || j.CompletedAt == nil { + continue + } + s.Runs++ + s.Queue = append(s.Queue, math.Max(0, j.StartedAt.Sub(*j.CreatedAt).Minutes())) + s.Exec = append(s.Exec, math.Max(0, j.CompletedAt.Sub(*j.StartedAt).Minutes())) + if strings.HasPrefix(j.RunnerName, "GitHub Actions") { + s.Hosted++ + } + } + return stats +} + +// longPoles counts how often each job is the last to finish in its run. That +// job sets the wall clock: no amount of parallelism gets below it. +func longPoles(jobs []Job) (map[string]int, int) { + type last struct { + name string + at float64 + } + byRun := map[int64]last{} + for _, j := range jobs { + if isSkippedMatrix(j) || isGate(j.Name) || j.CompletedAt == nil { + continue + } + at := float64(j.CompletedAt.UnixNano()) + if cur, ok := byRun[j.RunID]; !ok || at > cur.at { + byRun[j.RunID] = last{name: j.Name, at: at} + } + } + tally := map[string]int{} + for _, l := range byRun { + tally[l.name]++ + } + return tally, len(byRun) +} + +func summariseCaches(caches []Cache, totalBytes int64, count int) CacheState { + st := CacheState{ + TotalGB: gb(totalBytes), + Count: count, + Families: map[string]float64{}, + Copies: map[string]int{}, + Dupes: map[string][]string{}, + } + refs := map[string][]string{} + for _, c := range caches { + fam := c.Key + if i := strings.LastIndex(fam, "-"); i > 0 { + fam = fam[:i] + } + if len(fam) > 40 { + fam = fam[:40] + } + st.Families[fam] += gb(c.SizeInBytes) + st.Copies[fam]++ + refs[c.Key] = append(refs[c.Key], c.Ref) + } + for k, rs := range refs { + if len(rs) > 1 { + st.Dupes[k] = rs + } + } + // A key on a merge-queue ref is the least useful copy: the branch is deleted + // when the queue drains, so the entry can never be restored, yet it still + // counts against the quota until evicted. + for _, c := range caches { + if _, dup := st.Dupes[c.Key]; dup && strings.Contains(c.Ref, "gh-readonly-queue") { + st.StrandedGB += gb(c.SizeInBytes) + } + } + return st +} + +// weekly buckets runs by ISO week, oldest first. +// +// When the history window was truncated we hold only the tail of the oldest +// week, so its median comes from an arbitrary slice and is not comparable to +// the rest. Drop it rather than plot a misleading point. +func weekly(runs []Run, truncated bool) []Week { + buckets := map[string][]float64{} + for _, r := range runs { + y, w := r.Start.ISOWeek() + buckets[fmt.Sprintf("%d-W%02d", y, w)] = append(buckets[fmt.Sprintf("%d-W%02d", y, w)], r.Minutes) + } + labels := make([]string, 0, len(buckets)) + for k := range buckets { + labels = append(labels, k) + } + sort.Strings(labels) + if truncated && len(labels) > 1 { + labels = labels[1:] + } + weeks := make([]Week, 0, len(labels)) + for _, l := range labels { + v := buckets[l] + weeks = append(weeks, Week{Label: l, P50: pctl(v, 0.5), P90: pctl(v, 0.9), Count: len(v)}) + } + return weeks +} + +// diagnose ranks the causes of slowness by measured contribution to wall clock. +func diagnose(runs []Run, stats map[string]*JobStat, poles map[string]int, poleRuns int, cache CacheState) ([]Cause, float64) { + var causes []Cause + mins := make([]float64, 0, len(runs)) + for _, r := range runs { + mins = append(mins, r.Minutes) + } + wall := pctl(mins, 0.5) + + build := map[string]*JobStat{} + var queues []float64 + for n, s := range stats { + if isGate(n) || s.Runs == 0 { + continue + } + build[n] = s + queues = append(queues, s.Queue...) + } + + if len(queues) > 0 && wall > 0 { + q50, q90 := pctl(queues, 0.5), pctl(queues, 0.9) + share := q50 / wall * 100 + if share >= 10 || q90 >= 2 { + causes = append(causes, Cause{ + Weight: share, + Title: "Runner queue wait", + Detail: fmt.Sprintf("Jobs wait a median %.1f min (p90 %.1f min) for a runner before executing, %.0f%% of the %.1f min median run.", q50, q90, share, wall), + Fix: "Add runner capacity or reduce concurrent matrix width.", + }) + } + } + + if poleRuns > 0 && len(poles) > 0 { + var name string + var hits int + for n, c := range poles { + // Ties break on name so the report is deterministic. + if c > hits || (c == hits && n < name) { + name, hits = n, c + } + } + if s, ok := build[name]; ok && len(s.Exec) > 0 { + e50 := pctl(s.Exec, 0.5) + share := 0.0 + if wall > 0 { + share = e50 / wall * 100 + } + causes = append(causes, Cause{ + Weight: share, + Title: "Critical path: " + name, + Detail: fmt.Sprintf("Finishes last in %d/%d runs (%.0f%%), median %.1f min. Every other job waits on it.", hits, poleRuns, float64(hits)/float64(poleRuns)*100, e50), + Fix: "Nothing below this job's runtime is achievable; split or cache it.", + }) + } + } + + if quota := cache.TotalGB / quotaGB * 100; quota >= 75 { + fix := "Trim the largest cache family." + if cache.StrandedGB > 0 { + fix = fmt.Sprintf("%.2f GB sits on merge-queue refs that can never be restored.", cache.StrandedGB) + } + causes = append(causes, Cause{ + Weight: quota / 4, + Title: "Cache pressure", + Detail: fmt.Sprintf("%.2f GB of %.0f GB used (%.0f%%). GitHub evicts least-recently-used entries at quota, so jobs silently rebuild from cold.", cache.TotalGB, quotaGB, quota), + Fix: fix, + }) + } + + skipped, slots := 0, 0 + for _, s := range stats { + skipped += s.Skipped + slots += s.Skipped + s.Runs + } + if slots > 0 && float64(skipped)/float64(slots) > 0.3 { + causes = append(causes, Cause{ + Title: "Note: change detection is skipping rows", + Detail: fmt.Sprintf("%d/%d matrix slots (%.0f%%) were skipped. Medians look fast because work was avoided, not accelerated.", skipped, slots, float64(skipped)/float64(slots)*100), + Fix: "Compare p90 and run counts, not the median alone.", + Note: true, + }) + } + + sort.SliceStable(causes, func(i, j int) bool { return causes[i].Weight > causes[j].Weight }) + return causes, wall +} diff --git a/tools/ci-health/analysis_test.go b/tools/ci-health/analysis_test.go new file mode 100644 index 000000000..2ab680ed3 --- /dev/null +++ b/tools/ci-health/analysis_test.go @@ -0,0 +1,347 @@ +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "math" + "strings" + "testing" + "time" +) + +// Every case here corresponds to a defect this tool actually shipped with. The +// analysis functions are pure, so none of this touches the network. + +func at(day, hour int) time.Time { + return time.Date(2026, 7, day, hour, 0, 0, 0, time.UTC) +} + +func tp(day, hour, minute int) *time.Time { + t := time.Date(2026, 7, day, hour, minute, 0, 0, time.UTC) + return &t +} + +func mkJob(name string, created, started, completed *time.Time, conclusion string, runID int64, runner string) Job { + return Job{ + Name: name, + RunID: runID, + CreatedAt: created, + StartedAt: started, + CompletedAt: completed, + Conclusion: conclusion, + RunnerName: runner, + } +} + +func TestPercentileInterpolatesBetweenSamples(t *testing.T) { + // Index truncation would return 10; the real p90 is 9.1. + got := pctl([]float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, 0.9) + if math.Abs(got-9.1) > 1e-9 { + t.Fatalf("p90 = %v, want 9.1", got) + } +} + +func TestPercentileMedianOfEvenLength(t *testing.T) { + if got := pctl([]float64{1, 2, 3, 4}, 0.5); math.Abs(got-2.5) > 1e-9 { + t.Fatalf("median = %v, want 2.5", got) + } +} + +func TestPercentileDegenerateInputs(t *testing.T) { + if got := pctl(nil, 0.5); got != 0 { + t.Fatalf("empty = %v, want 0", got) + } + if got := pctl([]float64{7.5}, 0.9); got != 7.5 { + t.Fatalf("single = %v, want 7.5", got) + } +} + +func TestPercentileDoesNotMutateInput(t *testing.T) { + in := []float64{3, 1, 2} + pctl(in, 0.5) + if in[0] != 3 || in[1] != 1 || in[2] != 2 { + t.Fatalf("input was reordered: %v", in) + } +} + +// weeklyFixture has a truncated tail in W28, then two whole weeks. +func weeklyFixture() []Run { + runs := []Run{{Start: at(6, 9), Minutes: 1}} + for i := 0; i < 5; i++ { + runs = append(runs, Run{Start: at(13, 9), Minutes: 10}) + } + for i := 0; i < 5; i++ { + runs = append(runs, Run{Start: at(20, 9), Minutes: 20}) + } + return runs +} + +func TestWeeklyKeepsEveryWeekWhenNotTruncated(t *testing.T) { + weeks := weekly(weeklyFixture(), false) + if len(weeks) != 3 { + t.Fatalf("got %d weeks, want 3", len(weeks)) + } + if weeks[0].Label != "2026-W28" || weeks[0].P50 != 1 { + t.Fatalf("first week = %+v", weeks[0]) + } +} + +func TestWeeklyDropsPartialOldestWeekWhenTruncated(t *testing.T) { + weeks := weekly(weeklyFixture(), true) + if len(weeks) != 2 { + t.Fatalf("got %d weeks, want 2", len(weeks)) + } + if weeks[0].Label != "2026-W29" { + t.Fatalf("first week = %q, want 2026-W29", weeks[0].Label) + } +} + +func TestWeeklyNeverDropsTheOnlyWeek(t *testing.T) { + if got := weekly([]Run{{Start: at(20, 9), Minutes: 4}}, true); len(got) != 1 { + t.Fatalf("got %d weeks, want 1", len(got)) + } +} + +func TestWeeklyReportsRunCounts(t *testing.T) { + weeks := weekly(weeklyFixture(), true) + for _, w := range weeks { + if w.Count != 5 { + t.Fatalf("week %s count = %d, want 5", w.Label, w.Count) + } + } +} + +func TestSkippedMatrixDetection(t *testing.T) { + cases := []struct { + name string + job Job + wantSkip bool + }{ + // A row skipped by change detection keeps its raw matrix expression. + {"unexpanded expression", Job{Name: "bazel (${{ matrix.subtree.id }})", Conclusion: "success"}, true}, + {"explicit skip", Job{Name: "bazel (nvca)", Conclusion: "skipped"}, true}, + {"real row", Job{Name: "bazel (nvca)", Conclusion: "success"}, false}, + } + for _, c := range cases { + if got := isSkippedMatrix(c.job); got != c.wantSkip { + t.Errorf("%s: isSkippedMatrix = %v, want %v", c.name, got, c.wantSkip) + } + } +} + +func TestGateJobDetection(t *testing.T) { + for name, want := range map[string]bool{ + "detect changed subtrees": true, + "bazel required checks": true, + "bazel (nvca)": false, + } { + if got := isGate(name); got != want { + t.Errorf("isGate(%q) = %v, want %v", name, got, want) + } + } +} + +func TestAnalyseJobsSplitsQueueFromExecution(t *testing.T) { + stats := analyseJobs([]Job{ + mkJob("bazel (nvca)", tp(20, 10, 0), tp(20, 10, 2), tp(20, 10, 12), "success", 1, "self"), + }) + s := stats["bazel (nvca)"] + if s.Runs != 1 { + t.Fatalf("runs = %d, want 1", s.Runs) + } + if math.Abs(s.Queue[0]-2) > 1e-9 { + t.Errorf("queue = %v, want 2", s.Queue[0]) + } + if math.Abs(s.Exec[0]-10) > 1e-9 { + t.Errorf("exec = %v, want 10", s.Exec[0]) + } +} + +func TestAnalyseJobsCountsSkippedWithoutTiming(t *testing.T) { + stats := analyseJobs([]Job{ + mkJob("bazel (nvca)", tp(20, 0, 0), tp(20, 0, 0), tp(20, 0, 0), "skipped", 1, "self"), + }) + s := stats["bazel (nvca)"] + if s.Skipped != 1 || s.Runs != 0 { + t.Fatalf("skipped = %d, runs = %d; want 1, 0", s.Skipped, s.Runs) + } +} + +func TestAnalyseJobsDropsMissingTimestamps(t *testing.T) { + stats := analyseJobs([]Job{ + mkJob("bazel (nvca)", tp(20, 0, 0), nil, tp(20, 1, 0), "success", 1, "self"), + }) + if s := stats["bazel (nvca)"]; s.Runs != 0 { + t.Fatalf("runs = %d, want 0", s.Runs) + } +} + +func TestAnalyseJobsTalliesHostedRunners(t *testing.T) { + stats := analyseJobs([]Job{ + mkJob("gate", tp(20, 1, 0), tp(20, 1, 0), tp(20, 2, 0), "success", 1, "GitHub Actions 12"), + }) + if s := stats["gate"]; s.Hosted != 1 { + t.Fatalf("hosted = %d, want 1", s.Hosted) + } +} + +func TestLongPolesPicksLastFinishingBuildJob(t *testing.T) { + tally, runs := longPoles([]Job{ + mkJob("bazel (fast)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 30), "success", 1, "self"), + mkJob("bazel (slow)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 50), "success", 1, "self"), + }) + if runs != 1 || tally["bazel (slow)"] != 1 || len(tally) != 1 { + t.Fatalf("tally = %v over %d runs", tally, runs) + } +} + +func TestLongPolesExcludesGateJobs(t *testing.T) { + // The required-checks gate finishes last by construction. It is not the + // reason the build is slow. + tally, _ := longPoles([]Job{ + mkJob("bazel (slow)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 50), "success", 1, "self"), + mkJob("bazel required checks", tp(20, 10, 0), tp(20, 10, 0), tp(20, 11, 0), "success", 1, "self"), + }) + if tally["bazel (slow)"] != 1 || len(tally) != 1 { + t.Fatalf("tally = %v", tally) + } +} + +func TestLongPolesTalliesRunsIndependently(t *testing.T) { + tally, runs := longPoles([]Job{ + mkJob("bazel (a)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 50), "success", 1, "self"), + mkJob("bazel (b)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 30), "success", 1, "self"), + mkJob("bazel (b)", tp(21, 10, 0), tp(21, 10, 0), tp(21, 10, 50), "success", 2, "self"), + mkJob("bazel (a)", tp(21, 10, 0), tp(21, 10, 0), tp(21, 10, 30), "success", 2, "self"), + }) + if runs != 2 || tally["bazel (a)"] != 1 || tally["bazel (b)"] != 1 { + t.Fatalf("tally = %v over %d runs", tally, runs) + } +} + +func healthyCache() CacheState { + return CacheState{TotalGB: 1, Count: 3, Families: map[string]float64{}, Copies: map[string]int{}, Dupes: map[string][]string{}} +} + +func TestDiagnoseFlagsCachePressure(t *testing.T) { + c := healthyCache() + c.TotalGB, c.StrandedGB = 9.5, 1.9 + causes, _ := diagnose([]Run{{Start: at(20, 0), Minutes: 10}}, nil, nil, 0, c) + if !hasCause(causes, "Cache pressure") { + t.Fatalf("causes = %+v", causes) + } +} + +func TestDiagnoseQuietWhenHealthy(t *testing.T) { + stats := analyseJobs([]Job{ + mkJob("bazel (nvca)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 30), "success", 1, "self"), + }) + causes, wall := diagnose([]Run{{Start: at(20, 0), Minutes: 10}}, stats, nil, 0, healthyCache()) + if len(causes) != 0 { + t.Fatalf("expected no causes, got %+v", causes) + } + if math.Abs(wall-10) > 1e-9 { + t.Fatalf("wall = %v, want 10", wall) + } +} + +func TestDiagnoseWarnsOnHighSkipRate(t *testing.T) { + var jobs []Job + for i := 0; i < 9; i++ { + jobs = append(jobs, mkJob("bazel (s)", tp(20, 0, 0), tp(20, 0, 0), tp(20, 0, 0), "skipped", 1, "self")) + } + jobs = append(jobs, mkJob("bazel (real)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 30), "success", 1, "self")) + causes, _ := diagnose([]Run{{Start: at(20, 0), Minutes: 10}}, analyseJobs(jobs), nil, 0, healthyCache()) + found := false + for _, c := range causes { + if c.Note { + found = true + } + } + if !found { + t.Fatalf("a 90%% skip rate must be called out; causes = %+v", causes) + } +} + +func TestDiagnoseRanksLargestContributorFirst(t *testing.T) { + // A job that waits 10 min for a runner and executes for 1. + stats := analyseJobs([]Job{ + mkJob("bazel (a)", tp(20, 10, 0), tp(20, 10, 10), tp(20, 10, 11), "success", 1, "self"), + }) + causes, _ := diagnose([]Run{{Start: at(20, 0), Minutes: 20}}, stats, nil, 0, healthyCache()) + if len(causes) == 0 || causes[0].Title != "Runner queue wait" { + t.Fatalf("first cause = %+v", causes) + } +} + +func hasCause(causes []Cause, title string) bool { + for _, c := range causes { + if c.Title == title { + return true + } + } + return false +} + +func TestSummariseCachesFindsStrandedMergeQueueEntries(t *testing.T) { + caches := []Cache{ + {Key: "bazel-root-abc", Ref: "refs/heads/main", SizeInBytes: 1 << 30}, + {Key: "bazel-root-abc", Ref: "refs/heads/gh-readonly-queue/main/pr-1", SizeInBytes: 2 << 30}, + {Key: "other-def", Ref: "refs/heads/main", SizeInBytes: 1 << 30}, + } + st := summariseCaches(caches, 4<<30, 3) + if len(st.Dupes) != 1 { + t.Fatalf("dupes = %v, want 1", st.Dupes) + } + if math.Abs(st.StrandedGB-2) > 1e-9 { + t.Fatalf("stranded = %v GB, want 2", st.StrandedGB) + } + if math.Abs(st.Families["bazel-root"]-3) > 1e-9 { + t.Fatalf("family total = %v, want 3", st.Families["bazel-root"]) + } +} + +func TestRenderEscapesMatrixSyntaxInJobNames(t *testing.T) { + svg := stackedBars([]barRow{{Name: "bazel ()", Queue: 1, Exec: 2, Runs: 3}}) + if strings.Contains(svg, "") { + t.Fatal("job name was not escaped") + } + if !strings.Contains(svg, "&") { + t.Fatal("expected an escaped ampersand") + } +} + +func TestChartsHandleNoData(t *testing.T) { + if !strings.Contains(lineChart(nil), "no data") { + t.Error("lineChart(nil) should say so") + } + if !strings.Contains(stackedBars(nil), "no data") { + t.Error("stackedBars(nil) should say so") + } +} + +func TestDashboardHasNoExternalReferences(t *testing.T) { + cache := healthyCache() + cache.Families["fam"] = 1 + cache.Copies["fam"] = 1 + out := renderHTML("o/r", "bazel.yml", []Run{{Start: at(20, 0), Minutes: 5}}, + nil, nil, 0, cache, nil, 5, "now", false) + for _, marker := range []string{"http://", "https://", "src="} { + if strings.Contains(out, marker) { + t.Errorf("dashboard must stay self-contained, found %q", marker) + } + } +} diff --git a/tools/ci-health/github.go b/tools/ci-health/github.go new file mode 100644 index 000000000..6fe681f8e --- /dev/null +++ b/tools/ci-health/github.go @@ -0,0 +1,228 @@ +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os/exec" + "strings" + "sync" + "time" +) + +// fetch is the single point where this tool talks to GitHub. Tests replace it, +// which is why every other function here is exercised without a network. +var fetch = func(path, repo string) ([]byte, error) { + out, err := exec.Command("gh", "api", "repos/"+repo+"/"+path).Output() + if err != nil { + var ee *exec.ExitError + if ok := asExitError(err, &ee); ok { + return nil, fmt.Errorf("gh api %s: %s", path, strings.TrimSpace(string(ee.Stderr))) + } + return nil, fmt.Errorf("gh api %s: %w", path, err) + } + return out, nil +} + +func asExitError(err error, target **exec.ExitError) bool { + if ee, ok := err.(*exec.ExitError); ok { + *target = ee + return true + } + return false +} + +func getJSON(path, repo string, v any) error { + raw, err := fetch(path, repo) + if err != nil { + return err + } + return json.Unmarshal(raw, v) +} + +// paged walks a list endpoint until limit items or the data runs out. +// +// key names the array field for endpoints that wrap their results in an object +// (actions/caches, actions/runs). Pass "" for endpoints that return a bare +// array, such as pulls; asking for a key there is a decode error, not an empty +// result. +func paged(path, repo, key string, limit int) ([]json.RawMessage, error) { + var items []json.RawMessage + sep := "?" + if strings.Contains(path, "?") { + sep = "&" + } + for page := 1; len(items) < limit; page++ { + raw, err := fetch(fmt.Sprintf("%s%sper_page=100&page=%d", path, sep, page), repo) + if err != nil { + return nil, err + } + var batch []json.RawMessage + if key == "" { + if err := json.Unmarshal(raw, &batch); err != nil { + return nil, fmt.Errorf("decode %s: %w", path, err) + } + } else { + var obj map[string]json.RawMessage + if err := json.Unmarshal(raw, &obj); err != nil { + return nil, fmt.Errorf("decode %s: %w", path, err) + } + if body, ok := obj[key]; ok { + if err := json.Unmarshal(body, &batch); err != nil { + return nil, fmt.Errorf("decode %s.%s: %w", path, key, err) + } + } + } + if len(batch) == 0 { + break + } + items = append(items, batch...) + if len(batch) < 100 { + break + } + } + if len(items) > limit { + items = items[:limit] + } + return items, nil +} + +type apiRun struct { + ID int64 `json:"id"` + RunStartedAt *time.Time `json:"run_started_at"` + UpdatedAt *time.Time `json:"updated_at"` + HeadBranch string `json:"head_branch"` +} + +// Job is one row of a workflow run. started_at and completed_at are absent for +// jobs that never ran, so they stay pointers. +type Job struct { + Name string `json:"name"` + RunID int64 `json:"run_id"` + CreatedAt *time.Time `json:"created_at"` + StartedAt *time.Time `json:"started_at"` + CompletedAt *time.Time `json:"completed_at"` + Conclusion string `json:"conclusion"` + RunnerName string `json:"runner_name"` +} + +type Cache struct { + Key string `json:"key"` + Ref string `json:"ref"` + SizeInBytes int64 `json:"size_in_bytes"` +} + +type apiPR struct { + Number int `json:"number"` + CreatedAt *time.Time `json:"created_at"` + MergedAt *time.Time `json:"merged_at"` +} + +// Run is a workflow run reduced to what the report needs. +type Run struct { + ID int64 + Start time.Time + Minutes float64 + Branch string +} + +func fetchRuns(repo, workflow string, limit int) ([]Run, error) { + raws, err := paged("actions/workflows/"+workflow+"/runs?status=success", repo, "workflow_runs", limit) + if err != nil { + return nil, err + } + var runs []Run + for _, raw := range raws { + var r apiRun + if err := json.Unmarshal(raw, &r); err != nil { + continue + } + if r.RunStartedAt == nil || r.UpdatedAt == nil || r.UpdatedAt.Before(*r.RunStartedAt) { + continue + } + runs = append(runs, Run{ + ID: r.ID, + Start: *r.RunStartedAt, + Minutes: r.UpdatedAt.Sub(*r.RunStartedAt).Minutes(), + Branch: r.HeadBranch, + }) + } + return runs, nil +} + +// fetchJobs reads jobs for each run concurrently. One call per run is the only +// way GitHub exposes job timings, so this is the expensive part of a report. +// A run whose jobs cannot be read is skipped rather than failing the report. +func fetchJobs(repo string, runs []Run, workers int) []Job { + type result struct{ jobs []Job } + sem := make(chan struct{}, workers) + out := make([]result, len(runs)) + var wg sync.WaitGroup + for i, run := range runs { + wg.Add(1) + go func(i int, id int64) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + var body struct { + Jobs []Job `json:"jobs"` + } + if err := getJSON(fmt.Sprintf("actions/runs/%d/jobs?per_page=100", id), repo, &body); err != nil { + return + } + out[i] = result{jobs: body.Jobs} + }(i, run.ID) + } + wg.Wait() + var jobs []Job + for _, r := range out { + jobs = append(jobs, r.jobs...) + } + return jobs +} + +func fetchCaches(repo string) ([]Cache, error) { + raws, err := paged("actions/caches", repo, "actions_caches", maxCacheEntries) + if err != nil { + return nil, err + } + caches := make([]Cache, 0, len(raws)) + for _, raw := range raws { + var c Cache + if err := json.Unmarshal(raw, &c); err == nil { + caches = append(caches, c) + } + } + return caches, nil +} + +func fetchMergeLatencies(repo string, limit int) ([]PRLatency, error) { + // This endpoint returns a bare array, hence the empty key. + raws, err := paged("pulls?state=closed&sort=updated&direction=desc", repo, "", limit) + if err != nil { + return nil, err + } + var out []PRLatency + for _, raw := range raws { + var p apiPR + if err := json.Unmarshal(raw, &p); err != nil || p.MergedAt == nil || p.CreatedAt == nil { + continue + } + out = append(out, PRLatency{Number: p.Number, Hours: p.MergedAt.Sub(*p.CreatedAt).Hours()}) + } + return out, nil +} diff --git a/tools/ci-health/github_test.go b/tools/ci-health/github_test.go new file mode 100644 index 000000000..3f4596699 --- /dev/null +++ b/tools/ci-health/github_test.go @@ -0,0 +1,214 @@ +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +// stubFetch replaces the network layer with canned pages and records the paths +// requested, so pagination is asserted rather than assumed. +func stubFetch(t *testing.T, pages []string) *[]string { + t.Helper() + var calls []string + prev := fetch + fetch = func(path, repo string) ([]byte, error) { + calls = append(calls, path) + idx := 0 + if i := strings.Index(path, "&page="); i >= 0 { + fmt.Sscanf(path[i+len("&page="):], "%d", &idx) + idx-- + } + if idx < len(pages) { + return []byte(pages[idx]), nil + } + if strings.HasPrefix(strings.TrimSpace(pages[0]), "[") { + return []byte("[]"), nil + } + return []byte("{}"), nil + } + t.Cleanup(func() { fetch = prev }) + return &calls +} + +func arrayPage(n, from int) string { + items := make([]string, 0, n) + for i := 0; i < n; i++ { + items = append(items, fmt.Sprintf(`{"number":%d}`, from+i)) + } + return "[" + strings.Join(items, ",") + "]" +} + +func wrappedPage(key string, n, from int) string { + return fmt.Sprintf(`{%q:%s}`, key, arrayPage(n, from)) +} + +// The pulls endpoint returns a bare array. Asking for a wrapper key there used +// to crash the merge-time report once --prs exceeded one page. +func TestPagedBareArrayEndpoint(t *testing.T) { + stubFetch(t, []string{arrayPage(100, 0), arrayPage(1, 100)}) + got, err := paged("pulls?state=closed", "o/r", "", 150) + if err != nil { + t.Fatal(err) + } + if len(got) != 101 { + t.Fatalf("got %d items, want 101", len(got)) + } +} + +func TestPagedWrappedObjectEndpoint(t *testing.T) { + stubFetch(t, []string{wrappedPage("workflow_runs", 100, 0), wrappedPage("workflow_runs", 1, 100)}) + got, err := paged("actions/workflows/x/runs", "o/r", "workflow_runs", 150) + if err != nil { + t.Fatal(err) + } + if len(got) != 101 { + t.Fatalf("got %d items, want 101", len(got)) + } +} + +func TestPagedStopsAtLimit(t *testing.T) { + calls := stubFetch(t, []string{arrayPage(100, 0), arrayPage(100, 100), arrayPage(100, 200)}) + got, err := paged("pulls", "o/r", "", 150) + if err != nil { + t.Fatal(err) + } + if len(got) != 150 { + t.Fatalf("got %d items, want 150", len(got)) + } + if len(*calls) != 2 { + t.Fatalf("made %d calls, want 2; must not page past the limit", len(*calls)) + } +} + +func TestPagedStopsOnShortPage(t *testing.T) { + calls := stubFetch(t, []string{arrayPage(1, 0)}) + got, err := paged("pulls", "o/r", "", 500) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("got %d items, want 1", len(got)) + } + if len(*calls) != 1 { + t.Fatalf("made %d calls, want 1; a short page means the data ran out", len(*calls)) + } +} + +func TestPagedAppendsToExistingQuery(t *testing.T) { + calls := stubFetch(t, []string{arrayPage(1, 0)}) + if _, err := paged("pulls?state=closed", "o/r", "", 10); err != nil { + t.Fatal(err) + } + if want := "pulls?state=closed&per_page=100&page=1"; (*calls)[0] != want { + t.Fatalf("path = %q, want %q", (*calls)[0], want) + } +} + +func TestPagedStartsQueryWhenPathHasNone(t *testing.T) { + calls := stubFetch(t, []string{arrayPage(1, 0)}) + if _, err := paged("actions/caches", "o/r", "", 10); err != nil { + t.Fatal(err) + } + if want := "actions/caches?per_page=100&page=1"; (*calls)[0] != want { + t.Fatalf("path = %q, want %q", (*calls)[0], want) + } +} + +func TestPagedMissingKeyYieldsNothing(t *testing.T) { + stubFetch(t, []string{`{"something_else":[{"number":1}]}`}) + got, err := paged("actions/caches", "o/r", "actions_caches", 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("got %d items, want 0", len(got)) + } +} + +func TestPagedSurfacesDecodeErrors(t *testing.T) { + stubFetch(t, []string{`not json`}) + if _, err := paged("pulls", "o/r", "", 10); err == nil { + t.Fatal("expected a decode error") + } +} + +func TestFetchRunsSkipsRunsWithoutTimestamps(t *testing.T) { + page := `{"workflow_runs":[ + {"id":1,"run_started_at":"2026-07-20T10:00:00Z","updated_at":"2026-07-20T10:06:00Z","head_branch":"main"}, + {"id":2,"run_started_at":null,"updated_at":"2026-07-20T10:06:00Z"}, + {"id":3,"run_started_at":"2026-07-20T10:00:00Z","updated_at":"2026-07-20T09:00:00Z"} + ]}` + stubFetch(t, []string{page}) + runs, err := fetchRuns("o/r", "bazel.yml", 100) + if err != nil { + t.Fatal(err) + } + if len(runs) != 1 { + t.Fatalf("got %d runs, want 1 (null and end-before-start must be dropped)", len(runs)) + } + if runs[0].Minutes != 6 { + t.Fatalf("minutes = %v, want 6", runs[0].Minutes) + } +} + +func TestFetchMergeLatenciesIgnoresUnmergedPRs(t *testing.T) { + page := `[ + {"number":1,"created_at":"2026-07-20T00:00:00Z","merged_at":"2026-07-20T05:00:00Z"}, + {"number":2,"created_at":"2026-07-20T00:00:00Z","merged_at":null} + ]` + stubFetch(t, []string{page}) + lat, err := fetchMergeLatencies("o/r", 100) + if err != nil { + t.Fatal(err) + } + if len(lat) != 1 || lat[0].Number != 1 || lat[0].Hours != 5 { + t.Fatalf("latencies = %+v", lat) + } +} + +func TestFetchJobsToleratesFailures(t *testing.T) { + prev := fetch + fetch = func(path, repo string) ([]byte, error) { + if strings.Contains(path, "/2/") { + return nil, fmt.Errorf("boom") + } + return []byte(`{"jobs":[{"name":"bazel (a)","run_id":1,"conclusion":"success"}]}`), nil + } + t.Cleanup(func() { fetch = prev }) + + jobs := fetchJobs("o/r", []Run{{ID: 1}, {ID: 2}, {ID: 3}}, 2) + if len(jobs) != 2 { + t.Fatalf("got %d jobs, want 2; a failed run must be skipped, not fatal", len(jobs)) + } +} + +func TestJobDecodesNullTimestamps(t *testing.T) { + var j Job + body := `{"name":"x","run_id":1,"created_at":"2026-07-20T10:00:00Z","started_at":null,"completed_at":null,"conclusion":"skipped","runner_name":null}` + if err := json.Unmarshal([]byte(body), &j); err != nil { + t.Fatal(err) + } + if j.StartedAt != nil || j.CompletedAt != nil { + t.Fatal("null timestamps must decode to nil, not the zero time") + } + if j.CreatedAt == nil { + t.Fatal("created_at should have decoded") + } +} diff --git a/tools/ci-health/go.mod b/tools/ci-health/go.mod new file mode 100644 index 000000000..357ef372a --- /dev/null +++ b/tools/ci-health/go.mod @@ -0,0 +1,3 @@ +module ci-health + +go 1.26 diff --git a/tools/ci-health/main.go b/tools/ci-health/main.go new file mode 100644 index 000000000..855a589ea --- /dev/null +++ b/tools/ci-health/main.go @@ -0,0 +1,310 @@ +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command ci-health answers "why is my build slow" for a GitHub Actions +// repository. +// +// A workflow's wall-clock time is not one number, it is a stack: +// +// wall clock = queue wait + critical-path job + gate jobs +// +// Only the middle term is what people mean by "the build". A run that spends +// four minutes waiting for a runner looks identical, in the Actions UI, to a run +// that spends four minutes compiling. This tool separates them, finds which +// matrix row is the long pole, and reports cache pressure, which is the usual +// reason a job that used to be fast no longer is. +// +// Usage: +// +// ci-health --dashboard # visual report, opens in a browser +// ci-health --why # same findings, as text +// ci-health # cache and quota only +// ci-health --durations # duration percentiles by week +// ci-health --merge-times # PR open to merge latency +// ci-health --all +// ci-health --workflow release-tags.yml # default: bazel.yml +// ci-health --repo OWNER/NAME # default: NVIDIA/nvcf +// +// Trends read every retained run of the workflow, which is cheap because the +// runs endpoint paginates 100 at a time. Per-job analysis costs one API call per +// run, so it is limited to the most recent --runs runs. +// +// Requires the gh CLI, authenticated. +package main + +import ( + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "time" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +type options struct { + repo string + workflow string + dashboard string + why bool + durations bool + mergeTimes bool + all bool + runs int + history int + weeks int + prs int + noOpen bool +} + +func parseFlags() *options { + o := &options{} + flag.StringVar(&o.repo, "repo", "NVIDIA/nvcf", "repository as OWNER/NAME") + flag.StringVar(&o.workflow, "workflow", "bazel.yml", "workflow file name") + flag.StringVar(&o.dashboard, "dashboard", "", "write a self-contained HTML dashboard to this path (default build-health.html when the flag is given without a value)") + flag.BoolVar(&o.why, "why", false, "rank the causes of slowness as text") + flag.BoolVar(&o.durations, "durations", false, "duration percentiles by week") + flag.BoolVar(&o.mergeTimes, "merge-times", false, "PR open-to-merge latency") + flag.BoolVar(&o.all, "all", false, "cache, durations and merge times") + flag.IntVar(&o.runs, "runs", 60, "runs to pull per-job detail for") + flag.IntVar(&o.history, "history", 1000, "runs to trend over") + flag.IntVar(&o.weeks, "weeks", 12, "weeks of trend to print") + flag.IntVar(&o.prs, "prs", 100, "merged PRs to sample") + flag.BoolVar(&o.noOpen, "no-open", false, "do not launch a browser") + + // Allow a bare --dashboard with no value, which is the common case. + for i, a := range os.Args { + if a == "--dashboard" || a == "-dashboard" { + if i+1 >= len(os.Args) || len(os.Args[i+1]) > 0 && os.Args[i+1][0] == '-' { + os.Args = append(os.Args[:i+1:i+1], append([]string{"build-health.html"}, os.Args[i+1:]...)...) + } + break + } + } + flag.Parse() + return o +} + +func run() error { + o := parseFlags() + + workflow := o.workflow + if filepath.Ext(workflow) != ".yml" && filepath.Ext(workflow) != ".yaml" { + workflow += ".yml" + } + + needJobs := o.dashboard != "" || o.why || o.all + needRuns := needJobs || o.durations + + var ( + runsList []Run + jobs []Job + truncated bool + ) + if needRuns { + var err error + runsList, err = fetchRuns(o.repo, workflow, o.history) + if err != nil { + return err + } + if len(runsList) == 0 { + return fmt.Errorf("no successful runs found for workflow %q in %s", workflow, o.repo) + } + truncated = len(runsList) >= o.history + } + if needJobs { + sample := runsList + if len(sample) > o.runs { + sample = sample[:o.runs] + } + jobs = fetchJobs(o.repo, sample, 8) + } + + usage := struct { + Size int64 `json:"active_caches_size_in_bytes"` + Count int `json:"active_caches_count"` + }{} + if err := getJSON("actions/cache/usage", o.repo, &usage); err != nil { + return err + } + caches, err := fetchCaches(o.repo) + if err != nil { + return err + } + cache := summariseCaches(caches, usage.Size, usage.Count) + + stats := analyseJobs(jobs) + poles, poleRuns := longPoles(jobs) + var causes []Cause + var wall float64 + if needJobs { + causes, wall = diagnose(runsList, stats, poles, poleRuns, cache) + } + + if o.dashboard != "" { + out, err := filepath.Abs(o.dashboard) + if err != nil { + return err + } + generated := time.Now().UTC().Format("2006-01-02 15:04 UTC") + body := renderHTML(o.repo, workflow, runsList, stats, poles, poleRuns, cache, causes, wall, generated, truncated) + if err := os.WriteFile(out, []byte(body), 0o644); err != nil { + return err + } + fmt.Printf("wrote %s\n", out) + if !o.noOpen { + openBrowser(out) + } + return nil + } + + if o.why || o.all { + printWhy(causes, wall, runsList) + } + if o.durations || o.all { + printDurations(runsList, workflow, o.weeks, truncated) + } + if o.mergeTimes || o.all { + lat, err := fetchMergeLatencies(o.repo, o.prs) + if err != nil { + return err + } + printMergeTimes(lat) + } + if !o.why && !o.durations && !o.mergeTimes { + printCache(cache) + } else if o.all { + fmt.Println() + printCache(cache) + } + return nil +} + +func openBrowser(path string) { + url := "file://" + path + var cmd *exec.Cmd + switch runtime.GOOS { + case "darwin": + cmd = exec.Command("open", url) + case "windows": + cmd = exec.Command("rundll32", "url.dll,FileProtocolHandler", url) + default: + cmd = exec.Command("xdg-open", url) + } + // Best effort: a headless environment has no browser, and that is not an + // error worth failing the report over. + _ = cmd.Start() +} + +func printCache(c CacheState) { + quota := c.TotalGB / quotaGB * 100 + fmt.Printf("cache: %.2f GB / %.0f GB across %d entries (%.0f%% of quota)\n", c.TotalGB, quotaGB, c.Count, quota) + switch { + case quota >= 90: + fmt.Println(" AT QUOTA: new entries are evicting existing ones. Builds still pass, just colder.") + case quota >= 75: + fmt.Println(" approaching quota; expect evictions soon") + } + if len(c.Families) > 0 { + type fr struct { + n string + s float64 + } + var fams []fr + for n, s := range c.Families { + fams = append(fams, fr{n, s}) + } + sort.Slice(fams, func(i, j int) bool { + if fams[i].s == fams[j].s { + return fams[i].n < fams[j].n + } + return fams[i].s > fams[j].s + }) + fmt.Println("\nlargest families:") + for i, f := range fams { + if i >= 8 { + break + } + fmt.Printf(" %6.2f GB %4.1f%% of quota x%-3d %s\n", f.s, f.s/quotaGB*100, c.Copies[f.n], f.n) + } + } + if len(c.Dupes) > 0 { + fmt.Printf("\nkeys stored under multiple refs: %d\n", len(c.Dupes)) + if c.StrandedGB > 0 { + fmt.Printf(" %.2f GB of that is on merge-queue refs, which are unrestorable\n", c.StrandedGB) + } + } +} + +func printWhy(causes []Cause, wall float64, runs []Run) { + mins := make([]float64, 0, len(runs)) + for _, r := range runs { + mins = append(mins, r.Minutes) + } + fmt.Printf("\nmedian run %.1f min, p90 %.1f min over %d successful runs\n\n", wall, pctl(mins, 0.9), len(runs)) + if len(causes) == 0 { + fmt.Println("nothing is dominating the wall clock right now") + return + } + for i, c := range causes { + fmt.Printf("%d. %s\n %s\n -> %s\n\n", i+1, c.Title, c.Detail, c.Fix) + } +} + +func printDurations(runs []Run, workflow string, show int, truncated bool) { + weeks := weekly(runs, truncated) + if len(weeks) == 0 { + fmt.Printf("\nworkflow %q: no successful runs in the retained window\n", workflow) + return + } + if len(weeks) > show { + weeks = weeks[len(weeks)-show:] + } + fmt.Printf("\nworkflow %q (successful runs only), by week:\n", workflow) + for _, w := range weeks { + fmt.Printf(" %s runs=%-5d median=%6.1f min p90=%6.1f min\n", w.Label, w.Count, w.P50, w.P90) + } + fmt.Println(" NOTE: a low median can mean rows were skipped by change detection,") + fmt.Println(" not that builds got faster. Compare p90 and run counts too.") +} + +func printMergeTimes(lat []PRLatency) { + if len(lat) == 0 { + fmt.Println("\nno merged PRs in the sampled window") + return + } + hours := make([]float64, 0, len(lat)) + for _, l := range lat { + hours = append(hours, l.Hours) + } + fmt.Printf("\nPR open -> merge, %d merged PRs sampled:\n", len(lat)) + fmt.Printf(" median %.1f h p90 %.1f h\n", pctl(hours, 0.5), pctl(hours, 0.9)) + sort.Slice(lat, func(i, j int) bool { return lat[i].Hours > lat[j].Hours }) + fmt.Println(" slowest:") + for i, l := range lat { + if i >= 5 { + break + } + fmt.Printf(" #%-6d %8.1f h\n", l.Number, l.Hours) + } +} diff --git a/tools/ci-health/render.go b/tools/ci-health/render.go new file mode 100644 index 000000000..931cf2d56 --- /dev/null +++ b/tools/ci-health/render.go @@ -0,0 +1,344 @@ +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "html" + "sort" + "strings" +) + +// The dashboard is deliberately one self-contained file: inline SVG, no CDN and +// no JavaScript dependency. It renders offline and adds nothing to the +// repository's dependency surface. + +const dashboardCSS = ` +:root{--bg:#0f1115;--panel:#171a21;--line:#252a34;--fg:#e6e9ef;--dim:#98a1b3;--nv:#76b900;--warn:#e8b339;--bad:#e05252} +*{box-sizing:border-box} +body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif} +.wrap{max-width:960px;margin:0 auto;padding:34px 22px 70px} +h1{font-size:23px;margin:0 0 4px} +h2{font-size:15px;margin:0 0 14px;letter-spacing:.03em;text-transform:uppercase;color:var(--dim);font-weight:600} +.sub{color:var(--dim);margin:0 0 26px;font-size:13px} +.card{background:var(--panel);border:1px solid var(--line);border-radius:9px;padding:20px 22px;margin-bottom:18px} +.cause{border-left:3px solid var(--nv);padding:11px 0 11px 15px;margin-bottom:15px} +.cause:last-child{margin-bottom:0} +.cause .t{font-weight:600;margin-bottom:3px} +.cause .d{color:var(--dim)} +.cause .f{color:var(--nv);margin-top:5px;font-size:13px} +.cause.note{border-left-color:var(--warn)} +.cause.note .f{color:var(--warn)} +.stats{display:flex;gap:34px;flex-wrap:wrap;margin-bottom:6px} +.stat .v{font-size:26px;font-weight:600} +.stat .k{color:var(--dim);font-size:12px;text-transform:uppercase;letter-spacing:.05em} +.chart{width:100%;height:auto} +.grid{stroke:var(--line);stroke-width:1} +.l50{fill:none;stroke:var(--nv);stroke-width:2.2} +.l90{fill:none;stroke:#3d6ea8;stroke-width:1.6;stroke-dasharray:5 4} +.dot{fill:var(--nv)} +.ylab{fill:var(--dim);font-size:11px;text-anchor:end} +.xlab{fill:var(--dim);font-size:10px;text-anchor:middle} +.xcnt{fill:#5d6675;font-size:9px;text-anchor:middle} +.rowlab{fill:var(--fg);font-size:12px;text-anchor:end} +.rowval{fill:var(--dim);font-size:11px} +.bq{fill:var(--warn)} +.be{fill:var(--nv)} +.legend{color:var(--dim);font-size:12px;margin-top:10px} +.legend i{display:inline-block;width:10px;height:10px;border-radius:2px;margin:0 5px 0 16px;vertical-align:middle} +.legend i:first-child{margin-left:0} +.quota{position:relative;background:#22262f;border-radius:5px;height:32px;overflow:hidden} +.quotafill{height:100%} +.quotafill.ok{background:var(--nv)}.quotafill.warn{background:var(--warn)}.quotafill.over{background:var(--bad)} +.quotatxt{position:absolute;left:12px;top:7px;font-size:13px;font-weight:600;text-shadow:0 1px 3px rgba(0,0,0,.75)} +table{width:100%;border-collapse:collapse;font-size:13px} +th{text-align:left;color:var(--dim);font-weight:600;padding:7px 8px;border-bottom:1px solid var(--line)} +td{padding:7px 8px;border-bottom:1px solid var(--line)} +td.n{text-align:right;font-variant-numeric:tabular-nums} +.empty{color:var(--dim)} +code{background:#22262f;padding:1px 5px;border-radius:3px;font-size:12px} +` + +func esc(s string) string { return html.EscapeString(s) } + +func lineChart(weeks []Week) string { + if len(weeks) == 0 { + return "

no data

" + } + const w, h, pad = 860.0, 272.0, 46.0 + top := 1.0 + for _, k := range weeks { + if k.P90 > top { + top = k.P90 + } + } + top *= 1.15 + n := len(weeks) + step := (w - pad*2) / math1(n-1) + x := func(i int) float64 { return pad + float64(i)*step } + y := func(v float64) float64 { return h - pad - (v/top)*(h-pad*2) } + + var b strings.Builder + fmt.Fprintf(&b, "", w, h) + for g := 0; g < 5; g++ { + gy := pad + float64(g)*(h-pad*2)/4 + fmt.Fprintf(&b, "", pad, gy, w-pad, gy) + fmt.Fprintf(&b, "%.0fm", pad-8, gy+4, top*(1-float64(g)/4)) + } + path := func(pick func(Week) float64) string { + var p strings.Builder + for i, k := range weeks { + cmd := "L" + if i == 0 { + cmd = "M" + } + fmt.Fprintf(&p, "%s%.1f,%.1f ", cmd, x(i), y(pick(k))) + } + return strings.TrimSpace(p.String()) + } + fmt.Fprintf(&b, "", path(func(k Week) float64 { return k.P90 })) + fmt.Fprintf(&b, "", path(func(k Week) float64 { return k.P50 })) + for i, k := range weeks { + fmt.Fprintf(&b, "%s: median %.1f min, p90 %.1f min, %d runs", + x(i), y(k.P50), esc(k.Label), k.P50, k.P90, k.Count) + label := k.Label + if len(label) > 3 { + label = label[len(label)-3:] + } + fmt.Fprintf(&b, "%s", x(i), h-pad+18, esc(label)) + // The run count sits under the label so a thin week is never mistaken + // for a real speedup. + fmt.Fprintf(&b, "n=%d", x(i), h-pad+31, k.Count) + } + b.WriteString("") + return b.String() +} + +func math1(n int) float64 { + if n < 1 { + return 1 + } + return float64(n) +} + +type barRow struct { + Name string + Queue float64 + Exec float64 + Runs int + Skipped int +} + +func stackedBars(rows []barRow) string { + if len(rows) == 0 { + return "

no data

" + } + const w, bar, pad = 860.0, 26.0, 210.0 + top := 0.0 + for _, r := range rows { + if r.Queue+r.Exec > top { + top = r.Queue + r.Exec + } + } + if top == 0 { + top = 1 + } + span := w - pad - 90 + h := float64(len(rows))*(bar+8) + 16 + + var b strings.Builder + fmt.Fprintf(&b, "", w, h) + for i, r := range rows { + y := 8 + float64(i)*(bar+8) + qw := r.Queue / top * span + ew := r.Exec / top * span + short := r.Name + if len([]rune(short)) > 30 { + short = string([]rune(short)[:29]) + "…" + } + fmt.Fprintf(&b, "%s", pad-10, y+bar*0.68, esc(short)) + fmt.Fprintf(&b, "%s: queue %.1f min", + pad, y, qw, bar, esc(r.Name), r.Queue) + fmt.Fprintf(&b, "%s: execute %.1f min over %d runs, %d skipped", + pad+qw, y, ew, bar, esc(r.Name), r.Exec, r.Runs, r.Skipped) + fmt.Fprintf(&b, "%.1fm", pad+qw+ew+8, y+bar*0.68, r.Queue+r.Exec) + } + b.WriteString("") + return b.String() +} + +func quotaBar(c CacheState) string { + used := c.TotalGB / quotaGB * 100 + if used > 100 { + used = 100 + } + cls := "ok" + if used >= 90 { + cls = "over" + } else if used >= 75 { + cls = "warn" + } + return fmt.Sprintf("
"+ + "%.2f GB / %.0f GB (%.0f%%) across %d entries
", + cls, used, c.TotalGB, quotaGB, used, c.Count) +} + +func buildRows(stats map[string]*JobStat) []barRow { + var rows []barRow + for n, s := range stats { + if s.Runs == 0 || isGate(n) { + continue + } + rows = append(rows, barRow{Name: n, Queue: pctl(s.Queue, 0.5), Exec: pctl(s.Exec, 0.5), Runs: s.Runs, Skipped: s.Skipped}) + } + sort.Slice(rows, func(i, j int) bool { + a, b := rows[i].Queue+rows[i].Exec, rows[j].Queue+rows[j].Exec + if a == b { + return rows[i].Name < rows[j].Name + } + return a > b + }) + if len(rows) > 16 { + rows = rows[:16] + } + return rows +} + +func renderHTML(repo, workflow string, runs []Run, stats map[string]*JobStat, + poles map[string]int, poleRuns int, cache CacheState, causes []Cause, + wall float64, generated string, truncated bool) string { + + var causeHTML strings.Builder + for _, c := range causes { + cls := "cause" + if c.Note { + cls += " note" + } + fmt.Fprintf(&causeHTML, "
%s
%s
%s
", + cls, esc(c.Title), esc(c.Detail), esc(c.Fix)) + } + if len(causes) == 0 { + causeHTML.WriteString("

Nothing is dominating the wall clock right now.

") + } + + type famRow struct { + name string + size float64 + } + var fams []famRow + for n, s := range cache.Families { + fams = append(fams, famRow{n, s}) + } + sort.Slice(fams, func(i, j int) bool { + if fams[i].size == fams[j].size { + return fams[i].name < fams[j].name + } + return fams[i].size > fams[j].size + }) + var famHTML strings.Builder + for i, f := range fams { + if i >= 8 { + break + } + fmt.Fprintf(&famHTML, "%s%.2f GB%.0f%%%d", + esc(f.name), f.size, f.size/quotaGB*100, cache.Copies[f.name]) + } + + type poleRow struct { + name string + n int + } + var pl []poleRow + for n, c := range poles { + pl = append(pl, poleRow{n, c}) + } + sort.Slice(pl, func(i, j int) bool { + if pl[i].n == pl[j].n { + return pl[i].name < pl[j].name + } + return pl[i].n > pl[j].n + }) + var poleHTML strings.Builder + if poleRuns > 0 { + poleHTML.WriteString("

What finishes last

") + for i, p := range pl { + if i >= 8 { + break + } + fmt.Fprintf(&poleHTML, "", + esc(p.name), p.n, float64(p.n)/float64(poleRuns)*100) + } + poleHTML.WriteString("
JobRuns where it was lastShare
%s%d%.0f%%
") + } + + mins := make([]float64, 0, len(runs)) + for _, r := range runs { + mins = append(mins, r.Minutes) + } + weeks := weekly(runs, truncated) + span := "no runs" + if len(runs) > 0 { + span = fmt.Sprintf("%s to %s", runs[len(runs)-1].Start.Format("2006-01-02"), runs[0].Start.Format("2006-01-02")) + } + + stranded := "" + if cache.StrandedGB > 0 { + stranded = fmt.Sprintf("

%.2f GB is stored on gh-readonly-queue refs. "+ + "Those branches are deleted when the merge queue drains, so the entries can never be restored but still count against quota.

", cache.StrandedGB) + } + + return fmt.Sprintf(` + +Build health: %s
+

Why is my build slow?

+

%s · workflow %s · %d successful runs (%s) · +per-job detail from the most recent %d runs · generated %s

+ +

Ranked causes

%s
+ +

Wall clock

+
+
%.1f min
median run
+
%.1f min
p90 run
+
%d
weeks of history
+
+%s +
medianp90 +· hover a point for the week and run count
+ +

Where each job's time goes

+%s +
waiting for a runner +executing · median per job; gate jobs excluded
+ +%s + +

Cache

+%s +

Largest families. GitHub allows %.0f GB per repository and +evicts least-recently-used entries once full.

+%s
FamilySizeQuotaCopies
+%s
+
`, + esc(repo), dashboardCSS, + esc(repo), esc(workflow), len(runs), esc(span), poleRuns, esc(generated), + causeHTML.String(), + wall, pctl(mins, 0.9), len(weeks), + lineChart(weeks), + stackedBars(buildRows(stats)), + poleHTML.String(), + quotaBar(cache), quotaGB, famHTML.String(), stranded) +} diff --git a/tools/ci/ci-health b/tools/ci/ci-health index 78447bf07..861e4078c 100755 --- a/tools/ci/ci-health +++ b/tools/ci/ci-health @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env bash # SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -13,641 +13,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Answer "why is my build slow" for a GitHub Actions repository. - -A workflow's wall-clock time is not one number, it is a stack: - - wall clock = queue wait + critical-path job + gate jobs - -Only the middle term is what people mean by "the build". A run that spends -four minutes waiting for a runner looks identical, in the Actions UI, to a -run that spends four minutes compiling. This tool separates them, finds -which matrix row is the long pole, and reports cache pressure, which is the -usual reason a job that used to be fast no longer is. - -Usage: - tools/ci/ci-health --dashboard # visual report, opens in a browser - tools/ci/ci-health --why # same findings, as text - tools/ci/ci-health # cache and quota only - tools/ci/ci-health --durations # duration percentiles by week - tools/ci/ci-health --merge-times # PR open -> merge latency - tools/ci/ci-health --all - tools/ci/ci-health --workflow release-tags.yml # default: bazel.yml - tools/ci/ci-health --repo OWNER/NAME # default: NVIDIA/nvcf - -Sampling: trends read every retained run of the workflow (cheap, paginated). -Per-job analysis reads jobs for the most recent --runs runs (default 60), -since that costs one API call each. - -Requires the `gh` CLI, authenticated. -""" - -import argparse -import datetime as dt -import html -import json -import math -import os -import subprocess -import sys -import webbrowser -from collections import defaultdict -from concurrent.futures import ThreadPoolExecutor - -QUOTA_GB = 10.0 -DEFAULT_WORKFLOW = "bazel.yml" - -# Jobs that gate or fan out the matrix rather than doing build work. They are -# reported separately so they do not distort the per-subtree picture. -GATE_HINTS = ("detect changed", "required checks", "collect", "summary") - - -def gh_json(path, repo): - cmd = ["gh", "api", f"repos/{repo}/{path}"] - out = subprocess.run(cmd, capture_output=True, text=True) - if out.returncode != 0: - raise RuntimeError(f"gh api {path} failed: {out.stderr.strip()}") - return json.loads(out.stdout) - - -def gh_paged(path, repo, key, limit): - """Page through a list endpoint until `limit` items or the data runs out. - - `key` names the array field for endpoints that wrap their results in an - object; pass None for endpoints that return a bare array. - """ - items, page = [], 1 - sep = "&" if "?" in path else "?" - while len(items) < limit: - body = gh_json(f"{path}{sep}per_page=100&page={page}", repo) - batch = body if key is None else body.get(key, []) - if not batch: - break - items.extend(batch) - if len(batch) < 100: - break - page += 1 - return items[:limit] - - -def ts(value): - if not value: - return None - try: - return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError: - return None - - -def gb(n): - return n / 1024 ** 3 - - -def pct(vals, p): - """Linear-interpolated percentile. Plain indexing is too coarse at p90.""" - if not vals: - return 0.0 - s = sorted(vals) - if len(s) == 1: - return s[0] - k = (len(s) - 1) * p - lo, hi = math.floor(k), math.ceil(k) - if lo == hi: - return s[int(k)] - return s[lo] * (hi - k) + s[hi] * (k - lo) - - -def is_gate(name): - low = name.lower() - return any(h in low for h in GATE_HINTS) - - -def is_skipped_matrix(job): - # An unexpanded matrix expression means change detection skipped the row. - return job["conclusion"] == "skipped" or "${{" in job["name"] - - -# ----------------------------------------------------------------- collection - - -def fetch_runs(repo, workflow, limit): - path = f"actions/workflows/{workflow}/runs?status=success" - runs = gh_paged(path, repo, "workflow_runs", limit) - out = [] - for r in runs: - start, end = ts(r.get("run_started_at")), ts(r.get("updated_at")) - if not start or not end or end < start: - continue - out.append( - { - "id": r["id"], - "start": start, - "minutes": (end - start).total_seconds() / 60, - "branch": r.get("head_branch") or "", - } - ) - return out - - -def fetch_jobs(repo, runs, workers=8): - """Fetch jobs for each run concurrently. Failures are dropped, not fatal.""" - - def one(run): - try: - return gh_json(f"actions/runs/{run['id']}/jobs?per_page=100", repo).get("jobs", []) - except RuntimeError: - return [] - - with ThreadPoolExecutor(max_workers=workers) as pool: - return [j for batch in pool.map(one, runs) for j in batch] - - -# ------------------------------------------------------------------- analysis - - -def analyse_jobs(jobs): - """Split each job into queue wait and execution, grouped by job name.""" - stats = defaultdict(lambda: {"queue": [], "exec": [], "runs": 0, "skipped": 0, "hosted": 0}) - for j in jobs: - name = j["name"] - s = stats[name] - if is_skipped_matrix(j): - s["skipped"] += 1 - continue - created, started, done = ts(j.get("created_at")), ts(j.get("started_at")), ts(j.get("completed_at")) - if not (created and started and done): - continue - s["runs"] += 1 - s["queue"].append(max(0.0, (started - created).total_seconds() / 60)) - s["exec"].append(max(0.0, (done - started).total_seconds() / 60)) - if (j.get("runner_name") or "").startswith("GitHub Actions"): - s["hosted"] += 1 - return stats - - -def long_poles(jobs): - """How often each job is the last one to finish in its run.""" - by_run = defaultdict(list) - for j in jobs: - if is_skipped_matrix(j) or is_gate(j["name"]): - continue - done = ts(j.get("completed_at")) - if done: - by_run[j["run_id"]].append((done, j["name"])) - tally = defaultdict(int) - for entries in by_run.values(): - if entries: - tally[max(entries)[1]] += 1 - return tally, len(by_run) - - -def cache_state(repo): - usage = gh_json("actions/cache/usage", repo) - caches = gh_json("actions/caches?per_page=100", repo).get("actions_caches", []) - families, counts = defaultdict(float), defaultdict(int) - for c in caches: - fam = c["key"].rsplit("-", 1)[0][:40] - families[fam] += gb(c["size_in_bytes"]) - counts[fam] += 1 - refs = defaultdict(list) - for c in caches: - refs[c["key"]].append(c["ref"]) - dupes = {k: v for k, v in refs.items() if len(v) > 1} - stranded = sum( - gb(c["size_in_bytes"]) - for c in caches - if c["key"] in dupes and "gh-readonly-queue" in c["ref"] - ) - return { - "total": gb(usage["active_caches_size_in_bytes"]), - "count": usage["active_caches_count"], - "families": families, - "counts": counts, - "dupes": dupes, - "stranded": stranded, - } - - -def diagnose(runs, stats, poles, pole_runs, cache): - """Rank the causes of slowness, largest measured contribution first.""" - causes = [] - wall = pct([r["minutes"] for r in runs], 0.5) if runs else 0.0 - - build = {n: s for n, s in stats.items() if not is_gate(n) and s["runs"]} - queues = [q for s in build.values() for q in s["queue"]] - if queues and wall: - q50, q90 = pct(queues, 0.5), pct(queues, 0.9) - share = q50 / wall * 100 - if share >= 10 or q90 >= 2: - causes.append( - ( - share, - "Runner queue wait", - f"Jobs wait a median {q50:.1f} min (p90 {q90:.1f} min) for a runner " - f"before executing, {share:.0f}% of the {wall:.1f} min median run.", - "Add runner capacity or reduce concurrent matrix width.", - ) - ) - - if poles and pole_runs: - name, hits = max(poles.items(), key=lambda kv: kv[1]) - s = build.get(name) - if s and s["exec"]: - e50 = pct(s["exec"], 0.5) - share = e50 / wall * 100 if wall else 0.0 - causes.append( - ( - share, - f"Critical path: {name}", - f"Finishes last in {hits}/{pole_runs} runs ({hits / pole_runs * 100:.0f}%), " - f"median {e50:.1f} min. Every other job waits on it.", - "Nothing below this job's runtime is achievable; split or cache it.", - ) - ) - - quota = cache["total"] / QUOTA_GB * 100 - if quota >= 75: - causes.append( - ( - quota / 4, - "Cache pressure", - f"{cache['total']:.2f} GB of {QUOTA_GB:.0f} GB used ({quota:.0f}%). " - f"GitHub evicts least-recently-used entries at quota, so jobs " - f"silently rebuild from cold.", - f"{cache['stranded']:.2f} GB sits on merge-queue refs that can never be restored." - if cache["stranded"] - else "Trim the largest cache family.", - ) - ) - - skipped = sum(s["skipped"] for s in stats.values()) - total_slots = skipped + sum(s["runs"] for s in stats.values()) - if total_slots and skipped / total_slots > 0.3: - causes.append( - ( - 0.0, - "Note: change detection is skipping rows", - f"{skipped}/{total_slots} matrix slots ({skipped / total_slots * 100:.0f}%) were " - f"skipped. Medians look fast because work was avoided, not accelerated.", - "Compare p90 and run counts, not the median alone.", - ) - ) - - causes.sort(key=lambda c: -c[0]) - return causes, wall - - -def weekly(runs, truncated=False): - """Median/p90 per ISO week, oldest first. - - When the history window was truncated we only hold the tail of the oldest - week, so its median is computed from an arbitrary slice of that week and is - not comparable to the others. Drop it rather than plot a misleading point. - """ - buckets = defaultdict(list) - for r in runs: - y, w, _ = r["start"].isocalendar() - buckets[f"{y}-W{w:02d}"].append(r["minutes"]) - weeks = sorted(buckets) - if truncated and len(weeks) > 1: - weeks = weeks[1:] - return [(k, pct(buckets[k], 0.5), pct(buckets[k], 0.9), len(buckets[k])) for k in weeks] - - -# ------------------------------------------------------------------------ svg - - -def esc(s): - return html.escape(str(s), quote=True) - - -def line_chart(series, width=860, height=272, pad=46): - """series: list of (label, p50, p90, count).""" - if not series: - return "

no data

" - top = max(max(p90 for _, _, p90, _ in series), 1.0) * 1.15 - n = len(series) - span = width - pad * 2 - step = span / max(n - 1, 1) - - def pt(i, v): - return pad + i * step, height - pad - (v / top) * (height - pad * 2) - - def path(idx): - return " ".join( - f"{'M' if i == 0 else 'L'}{pt(i, s[idx])[0]:.1f},{pt(i, s[idx])[1]:.1f}" - for i, s in enumerate(series) - ) - - parts = [f""] - for g in range(5): - y = pad + g * (height - pad * 2) / 4 - val = top * (1 - g / 4) - parts.append(f"") - parts.append(f"{val:.0f}m") - parts.append(f"") - parts.append(f"") - for i, (label, p50, p90, cnt) in enumerate(series): - x, y = pt(i, p50) - parts.append( - f"" - f"{esc(label)}: median {p50:.1f} min, p90 {p90:.1f} min, {cnt} runs" - ) - if n <= 14 or i % max(1, n // 10) == 0: - parts.append( - f"{esc(label[-3:])}" - ) - # Run count sits under the label so a thin week is never mistaken - # for a real speedup. - parts.append( - f"n={cnt}" - ) - parts.append("") - return "".join(parts) - - -def stacked_bars(rows, width=860, bar=26, pad=210): - """rows: list of (name, queue_min, exec_min, runs, skipped).""" - if not rows: - return "

no data

" - top = max(q + e for _, q, e, _, _ in rows) or 1.0 - span = width - pad - 90 - height = len(rows) * (bar + 8) + 16 - parts = [f""] - for i, (name, q, e, runs, skipped) in enumerate(rows): - y = 8 + i * (bar + 8) - qw = q / top * span - ew = e / top * span - short = name if len(name) <= 30 else name[:29] + "…" - parts.append(f"{esc(short)}") - parts.append( - f"" - f"{esc(name)}: queue {q:.1f} min" - ) - parts.append( - f"" - f"{esc(name)}: execute {e:.1f} min over {runs} runs, {skipped} skipped" - ) - parts.append( - f"{q + e:.1f}m" - ) - parts.append("") - return "".join(parts) - - -def quota_bar(cache): - used = min(cache["total"] / QUOTA_GB, 1.0) * 100 - cls = "over" if used >= 90 else ("warn" if used >= 75 else "ok") - return ( - f"
" - f"{cache['total']:.2f} GB / {QUOTA_GB:.0f} GB " - f"({used:.0f}%) across {cache['count']} entries
" - ) - - -CSS = """ -:root{--bg:#0f1115;--panel:#171a21;--line:#252a34;--fg:#e6e9ef;--dim:#98a1b3;--nv:#76b900;--warn:#e8b339;--bad:#e05252} -*{box-sizing:border-box} -body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif} -.wrap{max-width:960px;margin:0 auto;padding:34px 22px 70px} -h1{font-size:23px;margin:0 0 4px} -h2{font-size:15px;margin:0 0 14px;letter-spacing:.03em;text-transform:uppercase;color:var(--dim);font-weight:600} -.sub{color:var(--dim);margin:0 0 26px;font-size:13px} -.card{background:var(--panel);border:1px solid var(--line);border-radius:9px;padding:20px 22px;margin-bottom:18px} -.cause{border-left:3px solid var(--nv);padding:11px 0 11px 15px;margin-bottom:15px} -.cause:last-child{margin-bottom:0} -.cause .t{font-weight:600;margin-bottom:3px} -.cause .d{color:var(--dim)} -.cause .f{color:var(--nv);margin-top:5px;font-size:13px} -.cause.note{border-left-color:var(--warn)} -.cause.note .f{color:var(--warn)} -.stats{display:flex;gap:34px;flex-wrap:wrap;margin-bottom:6px} -.stat .v{font-size:26px;font-weight:600} -.stat .k{color:var(--dim);font-size:12px;text-transform:uppercase;letter-spacing:.05em} -.chart{width:100%;height:auto} -.grid{stroke:var(--line);stroke-width:1} -.l50{fill:none;stroke:var(--nv);stroke-width:2.2} -.l90{fill:none;stroke:#3d6ea8;stroke-width:1.6;stroke-dasharray:5 4} -.dot{fill:var(--nv)} -.ylab{fill:var(--dim);font-size:11px;text-anchor:end} -.xlab{fill:var(--dim);font-size:10px;text-anchor:middle} -.xcnt{fill:#5d6675;font-size:9px;text-anchor:middle} -.rowlab{fill:var(--fg);font-size:12px;text-anchor:end} -.rowval{fill:var(--dim);font-size:11px} -.bq{fill:var(--warn)} -.be{fill:var(--nv)} -.legend{color:var(--dim);font-size:12px;margin-top:10px} -.legend i{display:inline-block;width:10px;height:10px;border-radius:2px;margin:0 5px 0 16px;vertical-align:middle} -.legend i:first-child{margin-left:0} -.quota{position:relative;background:#22262f;border-radius:5px;height:32px;overflow:hidden} -.quotafill{height:100%} -.quotafill.ok{background:var(--nv)}.quotafill.warn{background:var(--warn)}.quotafill.over{background:var(--bad)} -.quotatxt{position:absolute;left:12px;top:7px;font-size:13px;font-weight:600;text-shadow:0 1px 3px rgba(0,0,0,.75)} -table{width:100%;border-collapse:collapse;font-size:13px} -th{text-align:left;color:var(--dim);font-weight:600;padding:7px 8px;border-bottom:1px solid var(--line)} -td{padding:7px 8px;border-bottom:1px solid var(--line)} -td.n{text-align:right;font-variant-numeric:tabular-nums} -.empty{color:var(--dim)} -code{background:#22262f;padding:1px 5px;border-radius:3px;font-size:12px} -""" - - -def render_html(repo, workflow, runs, stats, poles, pole_runs, cache, causes, wall, - generated, truncated=False): - build = sorted( - ( - (n, pct(s["queue"], 0.5), pct(s["exec"], 0.5), s["runs"], s["skipped"]) - for n, s in stats.items() - if s["runs"] and not is_gate(n) - ), - key=lambda r: -(r[1] + r[2]), - )[:16] - - cause_html = "".join( - f"
" - f"
{esc(t)}
{esc(d)}
{esc(f)}
" - for _, t, d, f in causes - ) or "

Nothing is dominating the wall clock right now.

" - - fam_rows = "".join( - f"{esc(f)}{sz:.2f} GB" - f"{sz / QUOTA_GB * 100:.0f}%{cache['counts'][f]}" - for f, sz in sorted(cache["families"].items(), key=lambda kv: -kv[1])[:8] - ) - - pole_rows = "".join( - f"{esc(n)}{c}" - f"{c / pole_runs * 100:.0f}%" - for n, c in sorted(poles.items(), key=lambda kv: -kv[1])[:8] - ) if pole_runs else "" - - weeks = weekly(runs, truncated) - span = f"{runs[-1]['start'].date()} to {runs[0]['start'].date()}" if runs else "no runs" - - return f""" - -Build health: {esc(repo)}
-

Why is my build slow?

-

{esc(repo)} · workflow {esc(workflow)} · -{len(runs)} successful runs ({esc(span)}) · per-job detail from the most recent -{pole_runs} runs · generated {esc(generated)}

- -

Ranked causes

{cause_html}
- -

Wall clock

-
-
{wall:.1f} min
median run
-
{pct([r['minutes'] for r in runs], 0.9):.1f} min
p90 run
-
{len(weeks)}
weeks of history
-
-{line_chart(weeks)} -
medianp90 -· hover a point for the week and run count
- -

Where each job's time goes

-{stacked_bars(build)} -
waiting for a runner -executing -· median per job; gate jobs excluded
- -{"

What finishes last

" + pole_rows + "
JobRuns where it was lastShare
" if pole_rows else ""} - -

Cache

-{quota_bar(cache)} -

Largest families. GitHub allows {QUOTA_GB:.0f} GB per -repository and evicts least-recently-used entries once full.

- -{fam_rows}
FamilySizeQuotaCopies
-{"

" + f"{cache['stranded']:.2f} GB is stored on gh-readonly-queue refs. Those branches are deleted when the merge queue drains, so the entries can never be restored but still count against quota." + "

" if cache["stranded"] else ""} -
-
""" - - -# ----------------------------------------------------------------- text views - - -def cache_report(cache): - quota = cache["total"] / QUOTA_GB * 100 - print(f"cache: {cache['total']:.2f} GB / {QUOTA_GB:.0f} GB across {cache['count']} entries ({quota:.0f}% of quota)") - if quota >= 90: - print(" AT QUOTA: new entries are evicting existing ones. Builds still pass, just colder.") - elif quota >= 75: - print(" approaching quota; expect evictions soon") - if cache["families"]: - print("\nlargest families:") - for f, size in sorted(cache["families"].items(), key=lambda kv: -kv[1])[:8]: - print(f" {size:6.2f} GB {size / QUOTA_GB * 100:4.1f}% of quota x{cache['counts'][f]:<3d} {f}") - if cache["dupes"]: - print(f"\nkeys stored under multiple refs: {len(cache['dupes'])}") - if cache["stranded"]: - print(f" {cache['stranded']:.2f} GB of that is on merge-queue refs, which are unrestorable") - - -def why_report(causes, wall, runs): - print(f"\nmedian run {wall:.1f} min, p90 {pct([r['minutes'] for r in runs], 0.9):.1f} min " - f"over {len(runs)} successful runs\n") - if not causes: - print("nothing is dominating the wall clock right now") - return - for i, (_, title, detail, fix) in enumerate(causes, 1): - print(f"{i}. {title}\n {detail}\n -> {fix}\n") - - -def workflow_report(runs, workflow, weeks_shown, truncated=False): - if not runs: - print(f"\nworkflow '{workflow}': no successful runs in the retained window") - return - print(f"\nworkflow '{workflow}' (successful runs only), by week:") - for label, p50, p90, n in weekly(runs, truncated)[-weeks_shown:]: - print(f" {label} runs={n:<5d} median={p50:6.1f} min p90={p90:6.1f} min") - print(" NOTE: a low median can mean rows were skipped by change detection,") - print(" not that builds got faster. Compare p90 and run counts too.") - - -def merge_time_report(repo, limit): - prs = gh_paged("pulls?state=closed&sort=updated&direction=desc", repo, "items", limit) \ - if limit > 100 else gh_json( - f"pulls?state=closed&per_page={limit}&sort=updated&direction=desc", repo) - lat = [ - ((ts(p["merged_at"]) - ts(p["created_at"])).total_seconds() / 3600, p["number"]) - for p in prs - if p.get("merged_at") - ] - if not lat: - print("\nno merged PRs in the sampled window") - return - hours = [h for h, _ in lat] - print(f"\nPR open -> merge, {len(lat)} merged PRs sampled:") - print(f" median {pct(hours, 0.5):.1f} h p90 {pct(hours, 0.9):.1f} h") - print(" slowest:") - for h, n in sorted(lat, reverse=True)[:5]: - print(f" #{n:<6d} {h:8.1f} h") - - -# ------------------------------------------------------------------------ cli - - -def main(): - ap = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - ap.add_argument("--repo", default="NVIDIA/nvcf") - ap.add_argument("--workflow", default=DEFAULT_WORKFLOW, - help="workflow file name, e.g. bazel.yml (default: %(default)s)") - ap.add_argument("--dashboard", nargs="?", const="build-health.html", metavar="FILE", - help="write a self-contained HTML dashboard and open it") - ap.add_argument("--why", action="store_true", help="rank the causes of slowness as text") - ap.add_argument("--durations", action="store_true", help="duration percentiles by week") - ap.add_argument("--merge-times", action="store_true", help="PR open-to-merge latency") - ap.add_argument("--all", action="store_true") - ap.add_argument("--runs", type=int, default=60, help="runs to pull per-job detail for") - ap.add_argument("--history", type=int, default=1000, help="runs to trend over") - ap.add_argument("--weeks", type=int, default=12) - ap.add_argument("--prs", type=int, default=100) - ap.add_argument("--no-open", action="store_true", help="do not launch a browser") - args = ap.parse_args() - - wf = args.workflow if args.workflow.endswith(".yml") else f"{args.workflow}.yml" - needs_jobs = bool(args.dashboard) or args.why or args.all - needs_runs = needs_jobs or args.durations - - try: - runs = fetch_runs(args.repo, wf, args.history) if needs_runs else [] - if needs_runs and not runs: - sys.exit(f"no successful runs found for workflow '{wf}' in {args.repo}") - truncated = len(runs) >= args.history - jobs = fetch_jobs(args.repo, runs[: args.runs]) if needs_jobs else [] - cache = cache_state(args.repo) - except RuntimeError as e: - sys.exit(str(e)) - - stats = analyse_jobs(jobs) - poles, pole_runs = long_poles(jobs) - causes, wall = diagnose(runs, stats, poles, pole_runs, cache) if needs_jobs else ([], 0.0) - - if args.dashboard: - generated = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M UTC") - out = os.path.abspath(args.dashboard) - with open(out, "w") as fh: - fh.write(render_html(args.repo, wf, runs, stats, poles, pole_runs, - cache, causes, wall, generated, truncated)) - print(f"wrote {out}") - if not args.no_open: - webbrowser.open(f"file://{out}") - return - - if args.why or args.all: - why_report(causes, wall, runs) - if args.durations or args.all: - workflow_report(runs, wf, args.weeks, truncated) - if args.merge_times or args.all: - merge_time_report(args.repo, args.prs) - if not (args.why or args.durations or args.merge_times): - cache_report(cache) - elif args.all: - print() - cache_report(cache) - +# +# Stable entrypoint for the ci-health Go tool, which answers "why is my build +# slow" for a GitHub Actions repository. +# +# tools/ci/ci-health --dashboard # visual report, opens in a browser +# tools/ci/ci-health --why # same findings, as text +# tools/ci/ci-health # cache and quota only +# tools/ci/ci-health --durations # duration percentiles by week +# tools/ci/ci-health --merge-times # PR open to merge latency +# tools/ci/ci-health --help +# +# Requires Go and an authenticated `gh` CLI. +set -euo pipefail -if __name__ == "__main__": - main() +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec go run -C "${here}/../ci-health" . "$@" diff --git a/tools/ci/test-ci-health b/tools/ci/test-ci-health deleted file mode 100755 index a19133b24..000000000 --- a/tools/ci/test-ci-health +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for tools/ci/ci-health. - -Every case here corresponds to a defect the tool actually shipped with at -some point. The analysis functions are pure, so none of this touches the -network. - -Run: tools/ci/test-ci-health -""" - -import datetime as dt -import importlib.util -import os -import unittest - -_HERE = os.path.dirname(os.path.abspath(__file__)) -_spec = importlib.util.spec_from_loader( - "ci_health", - importlib.machinery.SourceFileLoader("ci_health", os.path.join(_HERE, "ci-health")), -) -ci = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(ci) - - -def at(day, hour=0, minute=0): - return dt.datetime(2026, 7, day, hour, minute, tzinfo=dt.timezone.utc) - - -def iso(day, hour=0, minute=0, second=0): - return f"2026-07-{day:02d}T{hour:02d}:{minute:02d}:{second:02d}Z" - - -def job(name, created, started, completed, conclusion="success", run_id=1, runner="self"): - return { - "name": name, - "created_at": created, - "started_at": started, - "completed_at": completed, - "conclusion": conclusion, - "run_id": run_id, - "runner_name": runner, - } - - -class Percentile(unittest.TestCase): - def test_interpolates_between_samples(self): - # Index-truncation would return 10.0 here; the real p90 is 9.1. - self.assertAlmostEqual(ci.pct([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 0.9), 9.1) - - def test_median_of_even_length_is_the_midpoint(self): - self.assertAlmostEqual(ci.pct([1, 2, 3, 4], 0.5), 2.5) - - def test_degenerate_inputs(self): - self.assertEqual(ci.pct([], 0.5), 0.0) - self.assertEqual(ci.pct([7.5], 0.9), 7.5) - - -class Weekly(unittest.TestCase): - """The oldest bucket is only a slice of its week when history was cut.""" - - def runs(self): - # W28 has a single fast run (the truncated tail); W29 and W30 are whole. - return ( - [{"start": at(6, 9), "minutes": 1.0}] - + [{"start": at(13, 9), "minutes": 10.0} for _ in range(5)] - + [{"start": at(20, 9), "minutes": 20.0} for _ in range(5)] - ) - - def test_keeps_every_week_when_not_truncated(self): - weeks = ci.weekly(self.runs(), truncated=False) - self.assertEqual([w[0] for w in weeks], ["2026-W28", "2026-W29", "2026-W30"]) - self.assertEqual(weeks[0][1], 1.0) - - def test_drops_the_partial_oldest_week_when_truncated(self): - weeks = ci.weekly(self.runs(), truncated=True) - self.assertEqual([w[0] for w in weeks], ["2026-W29", "2026-W30"]) - - def test_never_drops_the_only_week(self): - one = [{"start": at(20, 9), "minutes": 4.0}] - self.assertEqual(len(ci.weekly(one, truncated=True)), 1) - - def test_reports_run_counts(self): - self.assertEqual([w[3] for w in ci.weekly(self.runs(), truncated=True)], [5, 5]) - - -class Classification(unittest.TestCase): - def test_unexpanded_matrix_expression_counts_as_skipped(self): - # A skipped matrix row keeps its raw expression as the job name. - self.assertTrue( - ci.is_skipped_matrix( - {"name": "bazel (${{ matrix.subtree.id }})", "conclusion": "success"} - ) - ) - - def test_explicit_skip_counts_as_skipped(self): - self.assertTrue(ci.is_skipped_matrix({"name": "bazel (nvca)", "conclusion": "skipped"})) - - def test_real_row_is_not_skipped(self): - self.assertFalse(ci.is_skipped_matrix({"name": "bazel (nvca)", "conclusion": "success"})) - - def test_gate_jobs_are_recognised(self): - self.assertTrue(ci.is_gate("detect changed subtrees")) - self.assertTrue(ci.is_gate("bazel required checks")) - self.assertFalse(ci.is_gate("bazel (nvca)")) - - -class AnalyseJobs(unittest.TestCase): - def test_splits_queue_from_execution(self): - stats = ci.analyse_jobs( - [job("bazel (nvca)", iso(20, 10, 0), iso(20, 10, 2), iso(20, 10, 12))] - ) - s = stats["bazel (nvca)"] - self.assertEqual(s["runs"], 1) - self.assertAlmostEqual(s["queue"][0], 2.0) - self.assertAlmostEqual(s["exec"][0], 10.0) - - def test_skipped_rows_are_counted_not_timed(self): - stats = ci.analyse_jobs( - [job("bazel (nvca)", iso(20), iso(20), iso(20), conclusion="skipped")] - ) - self.assertEqual(stats["bazel (nvca)"]["skipped"], 1) - self.assertEqual(stats["bazel (nvca)"]["runs"], 0) - - def test_missing_timestamps_are_dropped(self): - stats = ci.analyse_jobs([job("bazel (nvca)", iso(20), None, iso(20, 1))]) - self.assertEqual(stats["bazel (nvca)"]["runs"], 0) - - def test_hosted_runners_are_tallied(self): - stats = ci.analyse_jobs( - [job("gate", iso(20, 1), iso(20, 1), iso(20, 2), runner="GitHub Actions 12")] - ) - self.assertEqual(stats["gate"]["hosted"], 1) - - -class LongPoles(unittest.TestCase): - def test_picks_the_last_finishing_build_job(self): - jobs = [ - job("bazel (fast)", iso(20, 10), iso(20, 10), iso(20, 10, 30), run_id=1), - job("bazel (slow)", iso(20, 10), iso(20, 10), iso(20, 10, 50), run_id=1), - ] - tally, runs = ci.long_poles(jobs) - self.assertEqual(runs, 1) - self.assertEqual(dict(tally), {"bazel (slow)": 1}) - - def test_gate_jobs_never_count_as_the_long_pole(self): - # The required-checks gate finishes last by construction; it is not - # the reason the build is slow. - jobs = [ - job("bazel (slow)", iso(20, 10), iso(20, 10), iso(20, 10, 50), run_id=1), - job("bazel required checks", iso(20, 10), iso(20, 10), iso(20, 11), run_id=1), - ] - tally, _ = ci.long_poles(jobs) - self.assertEqual(dict(tally), {"bazel (slow)": 1}) - - def test_tallies_across_runs_independently(self): - jobs = [ - job("bazel (a)", iso(20, 10), iso(20, 10), iso(20, 10, 50), run_id=1), - job("bazel (b)", iso(20, 10), iso(20, 10), iso(20, 10, 30), run_id=1), - job("bazel (b)", iso(21, 10), iso(21, 10), iso(21, 10, 50), run_id=2), - job("bazel (a)", iso(21, 10), iso(21, 10), iso(21, 10, 30), run_id=2), - ] - tally, runs = ci.long_poles(jobs) - self.assertEqual(runs, 2) - self.assertEqual(dict(tally), {"bazel (a)": 1, "bazel (b)": 1}) - - -class Diagnose(unittest.TestCase): - def cache(self, total=1.0, stranded=0.0): - return {"total": total, "count": 3, "families": {}, "counts": {}, - "dupes": {}, "stranded": stranded} - - def test_flags_cache_pressure_at_quota(self): - runs = [{"start": at(20), "minutes": 10.0}] - causes, _ = ci.diagnose(runs, {}, {}, 0, self.cache(total=9.5, stranded=1.9)) - titles = [c[1] for c in causes] - self.assertIn("Cache pressure", titles) - - def test_quiet_when_everything_is_healthy(self): - runs = [{"start": at(20), "minutes": 10.0}] - stats = ci.analyse_jobs( - [job("bazel (nvca)", iso(20, 10), iso(20, 10), iso(20, 10, 30))] - ) - causes, wall = ci.diagnose(runs, stats, {}, 0, self.cache(total=1.0)) - self.assertEqual(causes, []) - self.assertAlmostEqual(wall, 10.0) - - def test_warns_when_change_detection_skipped_most_rows(self): - runs = [{"start": at(20), "minutes": 10.0}] - jobs = [ - job(f"bazel (s{i})", iso(20), iso(20), iso(20), conclusion="skipped") - for i in range(9) - ] + [job("bazel (real)", iso(20, 10), iso(20, 10), iso(20, 10, 30))] - causes, _ = ci.diagnose(runs, ci.analyse_jobs(jobs), {}, 0, self.cache()) - self.assertTrue(any(c[1].startswith("Note") for c in causes), - "a 90% skip rate must be called out") - - def test_ranks_the_largest_contributor_first(self): - runs = [{"start": at(20), "minutes": 20.0}] - # A job that waits 10 min for a runner and executes for 1. - jobs = [job("bazel (a)", iso(20, 10, 0), iso(20, 10, 10), iso(20, 10, 11))] - causes, _ = ci.diagnose(runs, ci.analyse_jobs(jobs), {}, 0, self.cache()) - self.assertEqual(causes[0][1], "Runner queue wait") - - -class Rendering(unittest.TestCase): - def test_job_names_with_matrix_syntax_are_escaped(self): - svg = ci.stacked_bars([("bazel ()", 1.0, 2.0, 3, 0)]) - self.assertNotIn("", svg) - self.assertIn("&", svg) - - def test_charts_handle_no_data(self): - self.assertIn("no data", ci.line_chart([])) - self.assertIn("no data", ci.stacked_bars([])) - - def test_dashboard_has_no_external_references(self): - cache = {"total": 1.0, "count": 1, "families": {"fam": 1.0}, - "counts": {"fam": 1}, "dupes": {}, "stranded": 0.0} - runs = [{"start": at(20), "minutes": 5.0}] - out = ci.render_html("o/r", "bazel.yml", runs, {}, {}, 0, cache, [], 5.0, "now") - for marker in ("http://", "https://", "src="): - self.assertNotIn(marker, out, f"dashboard must stay self-contained ({marker})") - - -if __name__ == "__main__": - unittest.main(verbosity=2) From b21a78c49fdc10b3263f5e015ff003ec11bbcc91 Mon Sep 17 00:00:00 2001 From: balaji Date: Tue, 4 Aug 2026 21:27:41 -0700 Subject: [PATCH 5/6] fix(ci): address review findings in ci-health Four issues from review, all real, all verified rather than taken on trust. fetchJobs requested per_page=100 but never asked for page 2, so any run with more than 100 jobs reported partial timings. Wide matrix runs are exactly the ones whose numbers matter, so this silently understated the busiest runs. It now pages until total_count is reached. Negative counts panicked instead of erroring: --runs, --history, --weeks and --prs all reach slice bounds, and `--runs=-1` died with "slice bounds out of range [:-1]". Reproduced before fixing. They are now rejected with a message naming the flag. --durations and --merge-times read no cache data but still fetched it, so they failed whenever cache access failed, for reports that never used the result. The cache calls are now made only for the modes that read them. diagnose divided job medians drawn from the --runs sample by a wall-clock median drawn from the full --history window. Mixing two windows can misstate each cause's share and reorder them. Diagnosis now runs on the sampled window; the full history still feeds the trend charts, where it belongs. Tests cover pagination across two pages, the single-page stop, and rejection of every negative count flag. 40 tests pass. Verified live afterwards: --runs=-1 now prints a clean error, --durations completes without touching the cache, and --why reports over the window it actually sampled. Co-authored-by: Balaji Ganesan --- tools/ci-health/analysis_test.go | 22 ++++++++++++++ tools/ci-health/github.go | 24 +++++++++++---- tools/ci-health/github_test.go | 52 ++++++++++++++++++++++++++++++++ tools/ci-health/main.go | 51 ++++++++++++++++++++----------- 4 files changed, 126 insertions(+), 23 deletions(-) diff --git a/tools/ci-health/analysis_test.go b/tools/ci-health/analysis_test.go index 2ab680ed3..22cbab9bb 100644 --- a/tools/ci-health/analysis_test.go +++ b/tools/ci-health/analysis_test.go @@ -16,7 +16,10 @@ package main import ( + "flag" + "io" "math" + "os" "strings" "testing" "time" @@ -345,3 +348,22 @@ func TestDashboardHasNoExternalReferences(t *testing.T) { } } } + +// --runs, --history, --weeks and --prs all reach slice bounds, where a negative +// value panics instead of erroring. +func TestNegativeCountFlagsAreRejected(t *testing.T) { + for _, name := range []string{"runs", "history", "weeks", "prs"} { + t.Run(name, func(t *testing.T) { + flag.CommandLine = flag.NewFlagSet("ci-health", flag.ContinueOnError) + flag.CommandLine.SetOutput(io.Discard) + os.Args = []string{"ci-health", "--" + name + "=-1", "--repo", "o/r"} + err := run() + if err == nil { + t.Fatalf("--%s=-1 was accepted; it panics when used as a slice bound", name) + } + if !strings.Contains(err.Error(), "--"+name+" must be at least 1") { + t.Fatalf("error = %q, want it to name --%s", err, name) + } + }) + } +} diff --git a/tools/ci-health/github.go b/tools/ci-health/github.go index 6fe681f8e..5e35423a3 100644 --- a/tools/ci-health/github.go +++ b/tools/ci-health/github.go @@ -178,13 +178,25 @@ func fetchJobs(repo string, runs []Run, workers int) []Job { defer wg.Done() sem <- struct{}{} defer func() { <-sem }() - var body struct { - Jobs []Job `json:"jobs"` - } - if err := getJSON(fmt.Sprintf("actions/runs/%d/jobs?per_page=100", id), repo, &body); err != nil { - return + // Paginate. A run with more than 100 jobs would otherwise report + // partial timings, and the wide matrix rows are exactly the runs + // whose numbers matter most. + var all []Job + for page := 1; ; page++ { + var body struct { + Total int `json:"total_count"` + Jobs []Job `json:"jobs"` + } + path := fmt.Sprintf("actions/runs/%d/jobs?per_page=100&page=%d", id, page) + if err := getJSON(path, repo, &body); err != nil { + return + } + all = append(all, body.Jobs...) + if len(body.Jobs) < 100 || len(all) >= body.Total { + break + } } - out[i] = result{jobs: body.Jobs} + out[i] = result{jobs: all} }(i, run.ID) } wg.Wait() diff --git a/tools/ci-health/github_test.go b/tools/ci-health/github_test.go index 3f4596699..7d1ec8167 100644 --- a/tools/ci-health/github_test.go +++ b/tools/ci-health/github_test.go @@ -212,3 +212,55 @@ func TestJobDecodesNullTimestamps(t *testing.T) { t.Fatal("created_at should have decoded") } } + +// A run with more than 100 jobs used to report only the first page, and the +// wide matrix runs are exactly the ones whose timings matter most. +func TestFetchJobsPaginatesRunsWithManyJobs(t *testing.T) { + var calls []string + prev := fetch + fetch = func(path, repo string) ([]byte, error) { + calls = append(calls, path) + page := 1 + if i := strings.Index(path, "&page="); i >= 0 { + fmt.Sscanf(path[i+len("&page="):], "%d", &page) + } + n := 100 + if page == 2 { + n = 40 + } + if page > 2 { + n = 0 + } + jobs := make([]string, 0, n) + for i := 0; i < n; i++ { + jobs = append(jobs, `{"name":"bazel (x)","run_id":1,"conclusion":"success"}`) + } + return []byte(fmt.Sprintf(`{"total_count":140,"jobs":[%s]}`, strings.Join(jobs, ","))), nil + } + t.Cleanup(func() { fetch = prev }) + + got := fetchJobs("o/r", []Run{{ID: 1}}, 1) + if len(got) != 140 { + t.Fatalf("got %d jobs, want 140 (page 1 + page 2)", len(got)) + } + if len(calls) != 2 { + t.Fatalf("made %d requests, want 2; must stop once total_count is reached", len(calls)) + } +} + +func TestFetchJobsStopsOnSinglePage(t *testing.T) { + var calls int + prev := fetch + fetch = func(path, repo string) ([]byte, error) { + calls++ + return []byte(`{"total_count":1,"jobs":[{"name":"bazel (x)","run_id":1,"conclusion":"success"}]}`), nil + } + t.Cleanup(func() { fetch = prev }) + + if got := fetchJobs("o/r", []Run{{ID: 1}}, 1); len(got) != 1 { + t.Fatalf("got %d jobs, want 1", len(got)) + } + if calls != 1 { + t.Fatalf("made %d requests, want 1; a short page means no more pages", calls) + } +} diff --git a/tools/ci-health/main.go b/tools/ci-health/main.go index 855a589ea..4999bd914 100644 --- a/tools/ci-health/main.go +++ b/tools/ci-health/main.go @@ -108,6 +108,16 @@ func parseFlags() *options { func run() error { o := parseFlags() + // These reach slice bounds, so a negative value panics rather than erroring. + for _, c := range []struct { + name string + v int + }{{"runs", o.runs}, {"history", o.history}, {"weeks", o.weeks}, {"prs", o.prs}} { + if c.v < 1 { + return fmt.Errorf("--%s must be at least 1, got %d", c.name, c.v) + } + } + workflow := o.workflow if filepath.Ext(workflow) != ".yml" && filepath.Ext(workflow) != ".yaml" { workflow += ".yml" @@ -132,33 +142,40 @@ func run() error { } truncated = len(runsList) >= o.history } + var sampled []Run if needJobs { - sample := runsList - if len(sample) > o.runs { - sample = sample[:o.runs] + sampled = runsList + if len(sampled) > o.runs { + sampled = sampled[:o.runs] } - jobs = fetchJobs(o.repo, sample, 8) + jobs = fetchJobs(o.repo, sampled, 8) } - usage := struct { - Size int64 `json:"active_caches_size_in_bytes"` - Count int `json:"active_caches_count"` - }{} - if err := getJSON("actions/cache/usage", o.repo, &usage); err != nil { - return err - } - caches, err := fetchCaches(o.repo) - if err != nil { - return err + // --durations and --merge-times read no cache data, so they should neither + // pay for these calls nor fail when cache access does. + needCache := needJobs || (!o.why && !o.durations && !o.mergeTimes) + var cache CacheState + if needCache { + usage := struct { + Size int64 `json:"active_caches_size_in_bytes"` + Count int `json:"active_caches_count"` + }{} + if err := getJSON("actions/cache/usage", o.repo, &usage); err != nil { + return err + } + caches, err := fetchCaches(o.repo) + if err != nil { + return err + } + cache = summariseCaches(caches, usage.Size, usage.Count) } - cache := summariseCaches(caches, usage.Size, usage.Count) stats := analyseJobs(jobs) poles, poleRuns := longPoles(jobs) var causes []Cause var wall float64 if needJobs { - causes, wall = diagnose(runsList, stats, poles, poleRuns, cache) + causes, wall = diagnose(sampled, stats, poles, poleRuns, cache) } if o.dashboard != "" { @@ -179,7 +196,7 @@ func run() error { } if o.why || o.all { - printWhy(causes, wall, runsList) + printWhy(causes, wall, sampled) } if o.durations || o.all { printDurations(runsList, workflow, o.weeks, truncated) From 1e6aeb8b9978986c986c18ce4f1450d121933267 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Wed, 12 Aug 2026 11:31:56 -0700 Subject: [PATCH 6/6] fix(ci-health): preserve the gh error and check the stub page scans Two review findings from !656. The gh api failure path formatted stderr into a new error and dropped the *exec.ExitError, so a caller could read the message but not match on the exit status. Wrap the original with %w and keep stderr in the text. Both pagination stubs ignored the result of fmt.Sscanf. In the first a failed scan leaves the index at 0, the following decrement makes it -1, and indexing panics, which would surface a stub bug as a crash in the code under test. In the second it leaves the page at 1, so every request serves page one and a real pagination bug would pass. Return a wrapped error instead. Not changed: the suggestion to drop the oldest bucket when it is the only one. tools/ci-health/analysis_test.go already asserts the opposite in TestWeeklyNeverDropsTheOnlyWeek, so the current guard is deliberate rather than an oversight, and flipping it would trade a partial-week percentile for an empty report. Co-authored-by: Balaji Ganesan --- tools/ci-health/github.go | 4 +++- tools/ci-health/github_test.go | 13 +++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tools/ci-health/github.go b/tools/ci-health/github.go index 5e35423a3..6b2c663ae 100644 --- a/tools/ci-health/github.go +++ b/tools/ci-health/github.go @@ -31,7 +31,9 @@ var fetch = func(path, repo string) ([]byte, error) { if err != nil { var ee *exec.ExitError if ok := asExitError(err, &ee); ok { - return nil, fmt.Errorf("gh api %s: %s", path, strings.TrimSpace(string(ee.Stderr))) + // Keep the ExitError in the chain: the stderr text is what a human + // reads, but the exit status is what a caller can match on. + return nil, fmt.Errorf("gh api %s: %w: %s", path, err, strings.TrimSpace(string(ee.Stderr))) } return nil, fmt.Errorf("gh api %s: %w", path, err) } diff --git a/tools/ci-health/github_test.go b/tools/ci-health/github_test.go index 7d1ec8167..276c5e068 100644 --- a/tools/ci-health/github_test.go +++ b/tools/ci-health/github_test.go @@ -32,7 +32,12 @@ func stubFetch(t *testing.T, pages []string) *[]string { calls = append(calls, path) idx := 0 if i := strings.Index(path, "&page="); i >= 0 { - fmt.Sscanf(path[i+len("&page="):], "%d", &idx) + // An unchecked scan leaves idx at 0, and the decrement below then + // indexes pages[-1] and panics, reporting a stub bug as a crash in + // the code under test. + if _, err := fmt.Sscanf(path[i+len("&page="):], "%d", &idx); err != nil { + return nil, fmt.Errorf("stub: parse page from %q: %w", path, err) + } idx-- } if idx < len(pages) { @@ -222,7 +227,11 @@ func TestFetchJobsPaginatesRunsWithManyJobs(t *testing.T) { calls = append(calls, path) page := 1 if i := strings.Index(path, "&page="); i >= 0 { - fmt.Sscanf(path[i+len("&page="):], "%d", &page) + // An unchecked scan leaves page at 1 and silently serves page one + // for every request, so a pagination bug would pass this test. + if _, err := fmt.Sscanf(path[i+len("&page="):], "%d", &page); err != nil { + return nil, fmt.Errorf("stub: parse page from %q: %w", path, err) + } } n := 100 if page == 2 {