diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 1ed0b7a12..8d022850a 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -439,7 +439,14 @@ jobs: # same instant, tripping GitHub's action-download rate limit (HTTP 429 in # "Set up job"). Batching keeps the download burst under that limit; # change-aware scheduling already keeps most PRs well below the cap. - max-parallel: 8 + # + # Raised 8 -> 12 to shorten full-matrix runs: a recent 25-row merge-queue + # run needed three waves at 8 and needs two at 12. The bazel-docker matrix + # below contributes 4 more, so simultaneous checkouts peak at 16 rather + # than the previous 12. If "Set up job" starts failing with HTTP 429 this + # is the first knob to walk back; measure with `tools/ci/ci-health --why`, + # which reports queue wait separately from execution time. + max-parallel: 12 matrix: subtree: ${{ fromJSON(needs.detect.outputs.matrix) }} steps: diff --git a/tools/ci-health/.gitignore b/tools/ci-health/.gitignore new file mode 100644 index 000000000..4e691465f --- /dev/null +++ b/tools/ci-health/.gitignore @@ -0,0 +1,2 @@ +# Build output from `go build` in this directory. +/ci-health diff --git a/tools/ci-health/analysis.go b/tools/ci-health/analysis.go new file mode 100644 index 000000000..8fea5fc6e --- /dev/null +++ b/tools/ci-health/analysis.go @@ -0,0 +1,308 @@ +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "math" + "sort" + "strings" +) + +const ( + quotaGB = 10.0 + maxCacheEntries = 2000 +) + +// gateHints name the jobs that gate or fan out the matrix rather than doing +// build work. They are held out so they do not distort the per-subtree picture, +// and so the required-checks job is never blamed as the critical path. +var gateHints = []string{"detect changed", "required checks", "collect", "summary"} + +type JobStat struct { + Queue []float64 + Exec []float64 + Runs int + Skipped int + Hosted int +} + +type Week struct { + Label string + P50 float64 + P90 float64 + Count int +} + +type Cause struct { + Weight float64 + Title string + Detail string + Fix string + Note bool +} + +type PRLatency struct { + Number int + Hours float64 +} + +type CacheState struct { + TotalGB float64 + Count int + Families map[string]float64 + Copies map[string]int + Dupes map[string][]string + StrandedGB float64 +} + +func gb(n int64) float64 { return float64(n) / (1024 * 1024 * 1024) } + +// pctl is a linear-interpolated percentile. Plain index truncation is too +// coarse at p90: for ten ascending samples it just returns the maximum. +func pctl(vals []float64, p float64) float64 { + if len(vals) == 0 { + return 0 + } + s := append([]float64(nil), vals...) + sort.Float64s(s) + if len(s) == 1 { + return s[0] + } + k := float64(len(s)-1) * p + lo, hi := math.Floor(k), math.Ceil(k) + if lo == hi { + return s[int(k)] + } + return s[int(lo)]*(hi-k) + s[int(hi)]*(k-lo) +} + +func isGate(name string) bool { + low := strings.ToLower(name) + for _, h := range gateHints { + if strings.Contains(low, h) { + return true + } + } + return false +} + +// isSkippedMatrix reports whether a job never really ran. A row skipped by +// change detection keeps its unexpanded matrix expression as its name, which is +// the only signal GitHub gives that the row was elided rather than executed. +func isSkippedMatrix(j Job) bool { + return j.Conclusion == "skipped" || strings.Contains(j.Name, "${{") +} + +func analyseJobs(jobs []Job) map[string]*JobStat { + stats := map[string]*JobStat{} + for _, j := range jobs { + s, ok := stats[j.Name] + if !ok { + s = &JobStat{} + stats[j.Name] = s + } + if isSkippedMatrix(j) { + s.Skipped++ + continue + } + if j.CreatedAt == nil || j.StartedAt == nil || j.CompletedAt == nil { + continue + } + s.Runs++ + s.Queue = append(s.Queue, math.Max(0, j.StartedAt.Sub(*j.CreatedAt).Minutes())) + s.Exec = append(s.Exec, math.Max(0, j.CompletedAt.Sub(*j.StartedAt).Minutes())) + if strings.HasPrefix(j.RunnerName, "GitHub Actions") { + s.Hosted++ + } + } + return stats +} + +// longPoles counts how often each job is the last to finish in its run. That +// job sets the wall clock: no amount of parallelism gets below it. +func longPoles(jobs []Job) (map[string]int, int) { + type last struct { + name string + at float64 + } + byRun := map[int64]last{} + for _, j := range jobs { + if isSkippedMatrix(j) || isGate(j.Name) || j.CompletedAt == nil { + continue + } + at := float64(j.CompletedAt.UnixNano()) + if cur, ok := byRun[j.RunID]; !ok || at > cur.at { + byRun[j.RunID] = last{name: j.Name, at: at} + } + } + tally := map[string]int{} + for _, l := range byRun { + tally[l.name]++ + } + return tally, len(byRun) +} + +func summariseCaches(caches []Cache, totalBytes int64, count int) CacheState { + st := CacheState{ + TotalGB: gb(totalBytes), + Count: count, + Families: map[string]float64{}, + Copies: map[string]int{}, + Dupes: map[string][]string{}, + } + refs := map[string][]string{} + for _, c := range caches { + fam := c.Key + if i := strings.LastIndex(fam, "-"); i > 0 { + fam = fam[:i] + } + if len(fam) > 40 { + fam = fam[:40] + } + st.Families[fam] += gb(c.SizeInBytes) + st.Copies[fam]++ + refs[c.Key] = append(refs[c.Key], c.Ref) + } + for k, rs := range refs { + if len(rs) > 1 { + st.Dupes[k] = rs + } + } + // A key on a merge-queue ref is the least useful copy: the branch is deleted + // when the queue drains, so the entry can never be restored, yet it still + // counts against the quota until evicted. + for _, c := range caches { + if _, dup := st.Dupes[c.Key]; dup && strings.Contains(c.Ref, "gh-readonly-queue") { + st.StrandedGB += gb(c.SizeInBytes) + } + } + return st +} + +// weekly buckets runs by ISO week, oldest first. +// +// When the history window was truncated we hold only the tail of the oldest +// week, so its median comes from an arbitrary slice and is not comparable to +// the rest. Drop it rather than plot a misleading point. +func weekly(runs []Run, truncated bool) []Week { + buckets := map[string][]float64{} + for _, r := range runs { + y, w := r.Start.ISOWeek() + buckets[fmt.Sprintf("%d-W%02d", y, w)] = append(buckets[fmt.Sprintf("%d-W%02d", y, w)], r.Minutes) + } + labels := make([]string, 0, len(buckets)) + for k := range buckets { + labels = append(labels, k) + } + sort.Strings(labels) + if truncated && len(labels) > 1 { + labels = labels[1:] + } + weeks := make([]Week, 0, len(labels)) + for _, l := range labels { + v := buckets[l] + weeks = append(weeks, Week{Label: l, P50: pctl(v, 0.5), P90: pctl(v, 0.9), Count: len(v)}) + } + return weeks +} + +// diagnose ranks the causes of slowness by measured contribution to wall clock. +func diagnose(runs []Run, stats map[string]*JobStat, poles map[string]int, poleRuns int, cache CacheState) ([]Cause, float64) { + var causes []Cause + mins := make([]float64, 0, len(runs)) + for _, r := range runs { + mins = append(mins, r.Minutes) + } + wall := pctl(mins, 0.5) + + build := map[string]*JobStat{} + var queues []float64 + for n, s := range stats { + if isGate(n) || s.Runs == 0 { + continue + } + build[n] = s + queues = append(queues, s.Queue...) + } + + if len(queues) > 0 && wall > 0 { + q50, q90 := pctl(queues, 0.5), pctl(queues, 0.9) + share := q50 / wall * 100 + if share >= 10 || q90 >= 2 { + causes = append(causes, Cause{ + Weight: share, + Title: "Runner queue wait", + Detail: fmt.Sprintf("Jobs wait a median %.1f min (p90 %.1f min) for a runner before executing, %.0f%% of the %.1f min median run.", q50, q90, share, wall), + Fix: "Add runner capacity or reduce concurrent matrix width.", + }) + } + } + + if poleRuns > 0 && len(poles) > 0 { + var name string + var hits int + for n, c := range poles { + // Ties break on name so the report is deterministic. + if c > hits || (c == hits && n < name) { + name, hits = n, c + } + } + if s, ok := build[name]; ok && len(s.Exec) > 0 { + e50 := pctl(s.Exec, 0.5) + share := 0.0 + if wall > 0 { + share = e50 / wall * 100 + } + causes = append(causes, Cause{ + Weight: share, + Title: "Critical path: " + name, + Detail: fmt.Sprintf("Finishes last in %d/%d runs (%.0f%%), median %.1f min. Every other job waits on it.", hits, poleRuns, float64(hits)/float64(poleRuns)*100, e50), + Fix: "Nothing below this job's runtime is achievable; split or cache it.", + }) + } + } + + if quota := cache.TotalGB / quotaGB * 100; quota >= 75 { + fix := "Trim the largest cache family." + if cache.StrandedGB > 0 { + fix = fmt.Sprintf("%.2f GB sits on merge-queue refs that can never be restored.", cache.StrandedGB) + } + causes = append(causes, Cause{ + Weight: quota / 4, + Title: "Cache pressure", + Detail: fmt.Sprintf("%.2f GB of %.0f GB used (%.0f%%). GitHub evicts least-recently-used entries at quota, so jobs silently rebuild from cold.", cache.TotalGB, quotaGB, quota), + Fix: fix, + }) + } + + skipped, slots := 0, 0 + for _, s := range stats { + skipped += s.Skipped + slots += s.Skipped + s.Runs + } + if slots > 0 && float64(skipped)/float64(slots) > 0.3 { + causes = append(causes, Cause{ + Title: "Note: change detection is skipping rows", + Detail: fmt.Sprintf("%d/%d matrix slots (%.0f%%) were skipped. Medians look fast because work was avoided, not accelerated.", skipped, slots, float64(skipped)/float64(slots)*100), + Fix: "Compare p90 and run counts, not the median alone.", + Note: true, + }) + } + + sort.SliceStable(causes, func(i, j int) bool { return causes[i].Weight > causes[j].Weight }) + return causes, wall +} diff --git a/tools/ci-health/analysis_test.go b/tools/ci-health/analysis_test.go new file mode 100644 index 000000000..22cbab9bb --- /dev/null +++ b/tools/ci-health/analysis_test.go @@ -0,0 +1,369 @@ +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "io" + "math" + "os" + "strings" + "testing" + "time" +) + +// Every case here corresponds to a defect this tool actually shipped with. The +// analysis functions are pure, so none of this touches the network. + +func at(day, hour int) time.Time { + return time.Date(2026, 7, day, hour, 0, 0, 0, time.UTC) +} + +func tp(day, hour, minute int) *time.Time { + t := time.Date(2026, 7, day, hour, minute, 0, 0, time.UTC) + return &t +} + +func mkJob(name string, created, started, completed *time.Time, conclusion string, runID int64, runner string) Job { + return Job{ + Name: name, + RunID: runID, + CreatedAt: created, + StartedAt: started, + CompletedAt: completed, + Conclusion: conclusion, + RunnerName: runner, + } +} + +func TestPercentileInterpolatesBetweenSamples(t *testing.T) { + // Index truncation would return 10; the real p90 is 9.1. + got := pctl([]float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, 0.9) + if math.Abs(got-9.1) > 1e-9 { + t.Fatalf("p90 = %v, want 9.1", got) + } +} + +func TestPercentileMedianOfEvenLength(t *testing.T) { + if got := pctl([]float64{1, 2, 3, 4}, 0.5); math.Abs(got-2.5) > 1e-9 { + t.Fatalf("median = %v, want 2.5", got) + } +} + +func TestPercentileDegenerateInputs(t *testing.T) { + if got := pctl(nil, 0.5); got != 0 { + t.Fatalf("empty = %v, want 0", got) + } + if got := pctl([]float64{7.5}, 0.9); got != 7.5 { + t.Fatalf("single = %v, want 7.5", got) + } +} + +func TestPercentileDoesNotMutateInput(t *testing.T) { + in := []float64{3, 1, 2} + pctl(in, 0.5) + if in[0] != 3 || in[1] != 1 || in[2] != 2 { + t.Fatalf("input was reordered: %v", in) + } +} + +// weeklyFixture has a truncated tail in W28, then two whole weeks. +func weeklyFixture() []Run { + runs := []Run{{Start: at(6, 9), Minutes: 1}} + for i := 0; i < 5; i++ { + runs = append(runs, Run{Start: at(13, 9), Minutes: 10}) + } + for i := 0; i < 5; i++ { + runs = append(runs, Run{Start: at(20, 9), Minutes: 20}) + } + return runs +} + +func TestWeeklyKeepsEveryWeekWhenNotTruncated(t *testing.T) { + weeks := weekly(weeklyFixture(), false) + if len(weeks) != 3 { + t.Fatalf("got %d weeks, want 3", len(weeks)) + } + if weeks[0].Label != "2026-W28" || weeks[0].P50 != 1 { + t.Fatalf("first week = %+v", weeks[0]) + } +} + +func TestWeeklyDropsPartialOldestWeekWhenTruncated(t *testing.T) { + weeks := weekly(weeklyFixture(), true) + if len(weeks) != 2 { + t.Fatalf("got %d weeks, want 2", len(weeks)) + } + if weeks[0].Label != "2026-W29" { + t.Fatalf("first week = %q, want 2026-W29", weeks[0].Label) + } +} + +func TestWeeklyNeverDropsTheOnlyWeek(t *testing.T) { + if got := weekly([]Run{{Start: at(20, 9), Minutes: 4}}, true); len(got) != 1 { + t.Fatalf("got %d weeks, want 1", len(got)) + } +} + +func TestWeeklyReportsRunCounts(t *testing.T) { + weeks := weekly(weeklyFixture(), true) + for _, w := range weeks { + if w.Count != 5 { + t.Fatalf("week %s count = %d, want 5", w.Label, w.Count) + } + } +} + +func TestSkippedMatrixDetection(t *testing.T) { + cases := []struct { + name string + job Job + wantSkip bool + }{ + // A row skipped by change detection keeps its raw matrix expression. + {"unexpanded expression", Job{Name: "bazel (${{ matrix.subtree.id }})", Conclusion: "success"}, true}, + {"explicit skip", Job{Name: "bazel (nvca)", Conclusion: "skipped"}, true}, + {"real row", Job{Name: "bazel (nvca)", Conclusion: "success"}, false}, + } + for _, c := range cases { + if got := isSkippedMatrix(c.job); got != c.wantSkip { + t.Errorf("%s: isSkippedMatrix = %v, want %v", c.name, got, c.wantSkip) + } + } +} + +func TestGateJobDetection(t *testing.T) { + for name, want := range map[string]bool{ + "detect changed subtrees": true, + "bazel required checks": true, + "bazel (nvca)": false, + } { + if got := isGate(name); got != want { + t.Errorf("isGate(%q) = %v, want %v", name, got, want) + } + } +} + +func TestAnalyseJobsSplitsQueueFromExecution(t *testing.T) { + stats := analyseJobs([]Job{ + mkJob("bazel (nvca)", tp(20, 10, 0), tp(20, 10, 2), tp(20, 10, 12), "success", 1, "self"), + }) + s := stats["bazel (nvca)"] + if s.Runs != 1 { + t.Fatalf("runs = %d, want 1", s.Runs) + } + if math.Abs(s.Queue[0]-2) > 1e-9 { + t.Errorf("queue = %v, want 2", s.Queue[0]) + } + if math.Abs(s.Exec[0]-10) > 1e-9 { + t.Errorf("exec = %v, want 10", s.Exec[0]) + } +} + +func TestAnalyseJobsCountsSkippedWithoutTiming(t *testing.T) { + stats := analyseJobs([]Job{ + mkJob("bazel (nvca)", tp(20, 0, 0), tp(20, 0, 0), tp(20, 0, 0), "skipped", 1, "self"), + }) + s := stats["bazel (nvca)"] + if s.Skipped != 1 || s.Runs != 0 { + t.Fatalf("skipped = %d, runs = %d; want 1, 0", s.Skipped, s.Runs) + } +} + +func TestAnalyseJobsDropsMissingTimestamps(t *testing.T) { + stats := analyseJobs([]Job{ + mkJob("bazel (nvca)", tp(20, 0, 0), nil, tp(20, 1, 0), "success", 1, "self"), + }) + if s := stats["bazel (nvca)"]; s.Runs != 0 { + t.Fatalf("runs = %d, want 0", s.Runs) + } +} + +func TestAnalyseJobsTalliesHostedRunners(t *testing.T) { + stats := analyseJobs([]Job{ + mkJob("gate", tp(20, 1, 0), tp(20, 1, 0), tp(20, 2, 0), "success", 1, "GitHub Actions 12"), + }) + if s := stats["gate"]; s.Hosted != 1 { + t.Fatalf("hosted = %d, want 1", s.Hosted) + } +} + +func TestLongPolesPicksLastFinishingBuildJob(t *testing.T) { + tally, runs := longPoles([]Job{ + mkJob("bazel (fast)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 30), "success", 1, "self"), + mkJob("bazel (slow)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 50), "success", 1, "self"), + }) + if runs != 1 || tally["bazel (slow)"] != 1 || len(tally) != 1 { + t.Fatalf("tally = %v over %d runs", tally, runs) + } +} + +func TestLongPolesExcludesGateJobs(t *testing.T) { + // The required-checks gate finishes last by construction. It is not the + // reason the build is slow. + tally, _ := longPoles([]Job{ + mkJob("bazel (slow)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 50), "success", 1, "self"), + mkJob("bazel required checks", tp(20, 10, 0), tp(20, 10, 0), tp(20, 11, 0), "success", 1, "self"), + }) + if tally["bazel (slow)"] != 1 || len(tally) != 1 { + t.Fatalf("tally = %v", tally) + } +} + +func TestLongPolesTalliesRunsIndependently(t *testing.T) { + tally, runs := longPoles([]Job{ + mkJob("bazel (a)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 50), "success", 1, "self"), + mkJob("bazel (b)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 30), "success", 1, "self"), + mkJob("bazel (b)", tp(21, 10, 0), tp(21, 10, 0), tp(21, 10, 50), "success", 2, "self"), + mkJob("bazel (a)", tp(21, 10, 0), tp(21, 10, 0), tp(21, 10, 30), "success", 2, "self"), + }) + if runs != 2 || tally["bazel (a)"] != 1 || tally["bazel (b)"] != 1 { + t.Fatalf("tally = %v over %d runs", tally, runs) + } +} + +func healthyCache() CacheState { + return CacheState{TotalGB: 1, Count: 3, Families: map[string]float64{}, Copies: map[string]int{}, Dupes: map[string][]string{}} +} + +func TestDiagnoseFlagsCachePressure(t *testing.T) { + c := healthyCache() + c.TotalGB, c.StrandedGB = 9.5, 1.9 + causes, _ := diagnose([]Run{{Start: at(20, 0), Minutes: 10}}, nil, nil, 0, c) + if !hasCause(causes, "Cache pressure") { + t.Fatalf("causes = %+v", causes) + } +} + +func TestDiagnoseQuietWhenHealthy(t *testing.T) { + stats := analyseJobs([]Job{ + mkJob("bazel (nvca)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 30), "success", 1, "self"), + }) + causes, wall := diagnose([]Run{{Start: at(20, 0), Minutes: 10}}, stats, nil, 0, healthyCache()) + if len(causes) != 0 { + t.Fatalf("expected no causes, got %+v", causes) + } + if math.Abs(wall-10) > 1e-9 { + t.Fatalf("wall = %v, want 10", wall) + } +} + +func TestDiagnoseWarnsOnHighSkipRate(t *testing.T) { + var jobs []Job + for i := 0; i < 9; i++ { + jobs = append(jobs, mkJob("bazel (s)", tp(20, 0, 0), tp(20, 0, 0), tp(20, 0, 0), "skipped", 1, "self")) + } + jobs = append(jobs, mkJob("bazel (real)", tp(20, 10, 0), tp(20, 10, 0), tp(20, 10, 30), "success", 1, "self")) + causes, _ := diagnose([]Run{{Start: at(20, 0), Minutes: 10}}, analyseJobs(jobs), nil, 0, healthyCache()) + found := false + for _, c := range causes { + if c.Note { + found = true + } + } + if !found { + t.Fatalf("a 90%% skip rate must be called out; causes = %+v", causes) + } +} + +func TestDiagnoseRanksLargestContributorFirst(t *testing.T) { + // A job that waits 10 min for a runner and executes for 1. + stats := analyseJobs([]Job{ + mkJob("bazel (a)", tp(20, 10, 0), tp(20, 10, 10), tp(20, 10, 11), "success", 1, "self"), + }) + causes, _ := diagnose([]Run{{Start: at(20, 0), Minutes: 20}}, stats, nil, 0, healthyCache()) + if len(causes) == 0 || causes[0].Title != "Runner queue wait" { + t.Fatalf("first cause = %+v", causes) + } +} + +func hasCause(causes []Cause, title string) bool { + for _, c := range causes { + if c.Title == title { + return true + } + } + return false +} + +func TestSummariseCachesFindsStrandedMergeQueueEntries(t *testing.T) { + caches := []Cache{ + {Key: "bazel-root-abc", Ref: "refs/heads/main", SizeInBytes: 1 << 30}, + {Key: "bazel-root-abc", Ref: "refs/heads/gh-readonly-queue/main/pr-1", SizeInBytes: 2 << 30}, + {Key: "other-def", Ref: "refs/heads/main", SizeInBytes: 1 << 30}, + } + st := summariseCaches(caches, 4<<30, 3) + if len(st.Dupes) != 1 { + t.Fatalf("dupes = %v, want 1", st.Dupes) + } + if math.Abs(st.StrandedGB-2) > 1e-9 { + t.Fatalf("stranded = %v GB, want 2", st.StrandedGB) + } + if math.Abs(st.Families["bazel-root"]-3) > 1e-9 { + t.Fatalf("family total = %v, want 3", st.Families["bazel-root"]) + } +} + +func TestRenderEscapesMatrixSyntaxInJobNames(t *testing.T) { + svg := stackedBars([]barRow{{Name: "bazel ()", Queue: 1, Exec: 2, Runs: 3}}) + if strings.Contains(svg, "") { + t.Fatal("job name was not escaped") + } + if !strings.Contains(svg, "&") { + t.Fatal("expected an escaped ampersand") + } +} + +func TestChartsHandleNoData(t *testing.T) { + if !strings.Contains(lineChart(nil), "no data") { + t.Error("lineChart(nil) should say so") + } + if !strings.Contains(stackedBars(nil), "no data") { + t.Error("stackedBars(nil) should say so") + } +} + +func TestDashboardHasNoExternalReferences(t *testing.T) { + cache := healthyCache() + cache.Families["fam"] = 1 + cache.Copies["fam"] = 1 + out := renderHTML("o/r", "bazel.yml", []Run{{Start: at(20, 0), Minutes: 5}}, + nil, nil, 0, cache, nil, 5, "now", false) + for _, marker := range []string{"http://", "https://", "src="} { + if strings.Contains(out, marker) { + t.Errorf("dashboard must stay self-contained, found %q", marker) + } + } +} + +// --runs, --history, --weeks and --prs all reach slice bounds, where a negative +// value panics instead of erroring. +func TestNegativeCountFlagsAreRejected(t *testing.T) { + for _, name := range []string{"runs", "history", "weeks", "prs"} { + t.Run(name, func(t *testing.T) { + flag.CommandLine = flag.NewFlagSet("ci-health", flag.ContinueOnError) + flag.CommandLine.SetOutput(io.Discard) + os.Args = []string{"ci-health", "--" + name + "=-1", "--repo", "o/r"} + err := run() + if err == nil { + t.Fatalf("--%s=-1 was accepted; it panics when used as a slice bound", name) + } + if !strings.Contains(err.Error(), "--"+name+" must be at least 1") { + t.Fatalf("error = %q, want it to name --%s", err, name) + } + }) + } +} diff --git a/tools/ci-health/github.go b/tools/ci-health/github.go new file mode 100644 index 000000000..6b2c663ae --- /dev/null +++ b/tools/ci-health/github.go @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os/exec" + "strings" + "sync" + "time" +) + +// fetch is the single point where this tool talks to GitHub. Tests replace it, +// which is why every other function here is exercised without a network. +var fetch = func(path, repo string) ([]byte, error) { + out, err := exec.Command("gh", "api", "repos/"+repo+"/"+path).Output() + if err != nil { + var ee *exec.ExitError + if ok := asExitError(err, &ee); ok { + // Keep the ExitError in the chain: the stderr text is what a human + // reads, but the exit status is what a caller can match on. + return nil, fmt.Errorf("gh api %s: %w: %s", path, err, strings.TrimSpace(string(ee.Stderr))) + } + return nil, fmt.Errorf("gh api %s: %w", path, err) + } + return out, nil +} + +func asExitError(err error, target **exec.ExitError) bool { + if ee, ok := err.(*exec.ExitError); ok { + *target = ee + return true + } + return false +} + +func getJSON(path, repo string, v any) error { + raw, err := fetch(path, repo) + if err != nil { + return err + } + return json.Unmarshal(raw, v) +} + +// paged walks a list endpoint until limit items or the data runs out. +// +// key names the array field for endpoints that wrap their results in an object +// (actions/caches, actions/runs). Pass "" for endpoints that return a bare +// array, such as pulls; asking for a key there is a decode error, not an empty +// result. +func paged(path, repo, key string, limit int) ([]json.RawMessage, error) { + var items []json.RawMessage + sep := "?" + if strings.Contains(path, "?") { + sep = "&" + } + for page := 1; len(items) < limit; page++ { + raw, err := fetch(fmt.Sprintf("%s%sper_page=100&page=%d", path, sep, page), repo) + if err != nil { + return nil, err + } + var batch []json.RawMessage + if key == "" { + if err := json.Unmarshal(raw, &batch); err != nil { + return nil, fmt.Errorf("decode %s: %w", path, err) + } + } else { + var obj map[string]json.RawMessage + if err := json.Unmarshal(raw, &obj); err != nil { + return nil, fmt.Errorf("decode %s: %w", path, err) + } + if body, ok := obj[key]; ok { + if err := json.Unmarshal(body, &batch); err != nil { + return nil, fmt.Errorf("decode %s.%s: %w", path, key, err) + } + } + } + if len(batch) == 0 { + break + } + items = append(items, batch...) + if len(batch) < 100 { + break + } + } + if len(items) > limit { + items = items[:limit] + } + return items, nil +} + +type apiRun struct { + ID int64 `json:"id"` + RunStartedAt *time.Time `json:"run_started_at"` + UpdatedAt *time.Time `json:"updated_at"` + HeadBranch string `json:"head_branch"` +} + +// Job is one row of a workflow run. started_at and completed_at are absent for +// jobs that never ran, so they stay pointers. +type Job struct { + Name string `json:"name"` + RunID int64 `json:"run_id"` + CreatedAt *time.Time `json:"created_at"` + StartedAt *time.Time `json:"started_at"` + CompletedAt *time.Time `json:"completed_at"` + Conclusion string `json:"conclusion"` + RunnerName string `json:"runner_name"` +} + +type Cache struct { + Key string `json:"key"` + Ref string `json:"ref"` + SizeInBytes int64 `json:"size_in_bytes"` +} + +type apiPR struct { + Number int `json:"number"` + CreatedAt *time.Time `json:"created_at"` + MergedAt *time.Time `json:"merged_at"` +} + +// Run is a workflow run reduced to what the report needs. +type Run struct { + ID int64 + Start time.Time + Minutes float64 + Branch string +} + +func fetchRuns(repo, workflow string, limit int) ([]Run, error) { + raws, err := paged("actions/workflows/"+workflow+"/runs?status=success", repo, "workflow_runs", limit) + if err != nil { + return nil, err + } + var runs []Run + for _, raw := range raws { + var r apiRun + if err := json.Unmarshal(raw, &r); err != nil { + continue + } + if r.RunStartedAt == nil || r.UpdatedAt == nil || r.UpdatedAt.Before(*r.RunStartedAt) { + continue + } + runs = append(runs, Run{ + ID: r.ID, + Start: *r.RunStartedAt, + Minutes: r.UpdatedAt.Sub(*r.RunStartedAt).Minutes(), + Branch: r.HeadBranch, + }) + } + return runs, nil +} + +// fetchJobs reads jobs for each run concurrently. One call per run is the only +// way GitHub exposes job timings, so this is the expensive part of a report. +// A run whose jobs cannot be read is skipped rather than failing the report. +func fetchJobs(repo string, runs []Run, workers int) []Job { + type result struct{ jobs []Job } + sem := make(chan struct{}, workers) + out := make([]result, len(runs)) + var wg sync.WaitGroup + for i, run := range runs { + wg.Add(1) + go func(i int, id int64) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + // Paginate. A run with more than 100 jobs would otherwise report + // partial timings, and the wide matrix rows are exactly the runs + // whose numbers matter most. + var all []Job + for page := 1; ; page++ { + var body struct { + Total int `json:"total_count"` + Jobs []Job `json:"jobs"` + } + path := fmt.Sprintf("actions/runs/%d/jobs?per_page=100&page=%d", id, page) + if err := getJSON(path, repo, &body); err != nil { + return + } + all = append(all, body.Jobs...) + if len(body.Jobs) < 100 || len(all) >= body.Total { + break + } + } + out[i] = result{jobs: all} + }(i, run.ID) + } + wg.Wait() + var jobs []Job + for _, r := range out { + jobs = append(jobs, r.jobs...) + } + return jobs +} + +func fetchCaches(repo string) ([]Cache, error) { + raws, err := paged("actions/caches", repo, "actions_caches", maxCacheEntries) + if err != nil { + return nil, err + } + caches := make([]Cache, 0, len(raws)) + for _, raw := range raws { + var c Cache + if err := json.Unmarshal(raw, &c); err == nil { + caches = append(caches, c) + } + } + return caches, nil +} + +func fetchMergeLatencies(repo string, limit int) ([]PRLatency, error) { + // This endpoint returns a bare array, hence the empty key. + raws, err := paged("pulls?state=closed&sort=updated&direction=desc", repo, "", limit) + if err != nil { + return nil, err + } + var out []PRLatency + for _, raw := range raws { + var p apiPR + if err := json.Unmarshal(raw, &p); err != nil || p.MergedAt == nil || p.CreatedAt == nil { + continue + } + out = append(out, PRLatency{Number: p.Number, Hours: p.MergedAt.Sub(*p.CreatedAt).Hours()}) + } + return out, nil +} diff --git a/tools/ci-health/github_test.go b/tools/ci-health/github_test.go new file mode 100644 index 000000000..276c5e068 --- /dev/null +++ b/tools/ci-health/github_test.go @@ -0,0 +1,275 @@ +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +// stubFetch replaces the network layer with canned pages and records the paths +// requested, so pagination is asserted rather than assumed. +func stubFetch(t *testing.T, pages []string) *[]string { + t.Helper() + var calls []string + prev := fetch + fetch = func(path, repo string) ([]byte, error) { + calls = append(calls, path) + idx := 0 + if i := strings.Index(path, "&page="); i >= 0 { + // An unchecked scan leaves idx at 0, and the decrement below then + // indexes pages[-1] and panics, reporting a stub bug as a crash in + // the code under test. + if _, err := fmt.Sscanf(path[i+len("&page="):], "%d", &idx); err != nil { + return nil, fmt.Errorf("stub: parse page from %q: %w", path, err) + } + idx-- + } + if idx < len(pages) { + return []byte(pages[idx]), nil + } + if strings.HasPrefix(strings.TrimSpace(pages[0]), "[") { + return []byte("[]"), nil + } + return []byte("{}"), nil + } + t.Cleanup(func() { fetch = prev }) + return &calls +} + +func arrayPage(n, from int) string { + items := make([]string, 0, n) + for i := 0; i < n; i++ { + items = append(items, fmt.Sprintf(`{"number":%d}`, from+i)) + } + return "[" + strings.Join(items, ",") + "]" +} + +func wrappedPage(key string, n, from int) string { + return fmt.Sprintf(`{%q:%s}`, key, arrayPage(n, from)) +} + +// The pulls endpoint returns a bare array. Asking for a wrapper key there used +// to crash the merge-time report once --prs exceeded one page. +func TestPagedBareArrayEndpoint(t *testing.T) { + stubFetch(t, []string{arrayPage(100, 0), arrayPage(1, 100)}) + got, err := paged("pulls?state=closed", "o/r", "", 150) + if err != nil { + t.Fatal(err) + } + if len(got) != 101 { + t.Fatalf("got %d items, want 101", len(got)) + } +} + +func TestPagedWrappedObjectEndpoint(t *testing.T) { + stubFetch(t, []string{wrappedPage("workflow_runs", 100, 0), wrappedPage("workflow_runs", 1, 100)}) + got, err := paged("actions/workflows/x/runs", "o/r", "workflow_runs", 150) + if err != nil { + t.Fatal(err) + } + if len(got) != 101 { + t.Fatalf("got %d items, want 101", len(got)) + } +} + +func TestPagedStopsAtLimit(t *testing.T) { + calls := stubFetch(t, []string{arrayPage(100, 0), arrayPage(100, 100), arrayPage(100, 200)}) + got, err := paged("pulls", "o/r", "", 150) + if err != nil { + t.Fatal(err) + } + if len(got) != 150 { + t.Fatalf("got %d items, want 150", len(got)) + } + if len(*calls) != 2 { + t.Fatalf("made %d calls, want 2; must not page past the limit", len(*calls)) + } +} + +func TestPagedStopsOnShortPage(t *testing.T) { + calls := stubFetch(t, []string{arrayPage(1, 0)}) + got, err := paged("pulls", "o/r", "", 500) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("got %d items, want 1", len(got)) + } + if len(*calls) != 1 { + t.Fatalf("made %d calls, want 1; a short page means the data ran out", len(*calls)) + } +} + +func TestPagedAppendsToExistingQuery(t *testing.T) { + calls := stubFetch(t, []string{arrayPage(1, 0)}) + if _, err := paged("pulls?state=closed", "o/r", "", 10); err != nil { + t.Fatal(err) + } + if want := "pulls?state=closed&per_page=100&page=1"; (*calls)[0] != want { + t.Fatalf("path = %q, want %q", (*calls)[0], want) + } +} + +func TestPagedStartsQueryWhenPathHasNone(t *testing.T) { + calls := stubFetch(t, []string{arrayPage(1, 0)}) + if _, err := paged("actions/caches", "o/r", "", 10); err != nil { + t.Fatal(err) + } + if want := "actions/caches?per_page=100&page=1"; (*calls)[0] != want { + t.Fatalf("path = %q, want %q", (*calls)[0], want) + } +} + +func TestPagedMissingKeyYieldsNothing(t *testing.T) { + stubFetch(t, []string{`{"something_else":[{"number":1}]}`}) + got, err := paged("actions/caches", "o/r", "actions_caches", 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("got %d items, want 0", len(got)) + } +} + +func TestPagedSurfacesDecodeErrors(t *testing.T) { + stubFetch(t, []string{`not json`}) + if _, err := paged("pulls", "o/r", "", 10); err == nil { + t.Fatal("expected a decode error") + } +} + +func TestFetchRunsSkipsRunsWithoutTimestamps(t *testing.T) { + page := `{"workflow_runs":[ + {"id":1,"run_started_at":"2026-07-20T10:00:00Z","updated_at":"2026-07-20T10:06:00Z","head_branch":"main"}, + {"id":2,"run_started_at":null,"updated_at":"2026-07-20T10:06:00Z"}, + {"id":3,"run_started_at":"2026-07-20T10:00:00Z","updated_at":"2026-07-20T09:00:00Z"} + ]}` + stubFetch(t, []string{page}) + runs, err := fetchRuns("o/r", "bazel.yml", 100) + if err != nil { + t.Fatal(err) + } + if len(runs) != 1 { + t.Fatalf("got %d runs, want 1 (null and end-before-start must be dropped)", len(runs)) + } + if runs[0].Minutes != 6 { + t.Fatalf("minutes = %v, want 6", runs[0].Minutes) + } +} + +func TestFetchMergeLatenciesIgnoresUnmergedPRs(t *testing.T) { + page := `[ + {"number":1,"created_at":"2026-07-20T00:00:00Z","merged_at":"2026-07-20T05:00:00Z"}, + {"number":2,"created_at":"2026-07-20T00:00:00Z","merged_at":null} + ]` + stubFetch(t, []string{page}) + lat, err := fetchMergeLatencies("o/r", 100) + if err != nil { + t.Fatal(err) + } + if len(lat) != 1 || lat[0].Number != 1 || lat[0].Hours != 5 { + t.Fatalf("latencies = %+v", lat) + } +} + +func TestFetchJobsToleratesFailures(t *testing.T) { + prev := fetch + fetch = func(path, repo string) ([]byte, error) { + if strings.Contains(path, "/2/") { + return nil, fmt.Errorf("boom") + } + return []byte(`{"jobs":[{"name":"bazel (a)","run_id":1,"conclusion":"success"}]}`), nil + } + t.Cleanup(func() { fetch = prev }) + + jobs := fetchJobs("o/r", []Run{{ID: 1}, {ID: 2}, {ID: 3}}, 2) + if len(jobs) != 2 { + t.Fatalf("got %d jobs, want 2; a failed run must be skipped, not fatal", len(jobs)) + } +} + +func TestJobDecodesNullTimestamps(t *testing.T) { + var j Job + body := `{"name":"x","run_id":1,"created_at":"2026-07-20T10:00:00Z","started_at":null,"completed_at":null,"conclusion":"skipped","runner_name":null}` + if err := json.Unmarshal([]byte(body), &j); err != nil { + t.Fatal(err) + } + if j.StartedAt != nil || j.CompletedAt != nil { + t.Fatal("null timestamps must decode to nil, not the zero time") + } + if j.CreatedAt == nil { + t.Fatal("created_at should have decoded") + } +} + +// A run with more than 100 jobs used to report only the first page, and the +// wide matrix runs are exactly the ones whose timings matter most. +func TestFetchJobsPaginatesRunsWithManyJobs(t *testing.T) { + var calls []string + prev := fetch + fetch = func(path, repo string) ([]byte, error) { + calls = append(calls, path) + page := 1 + if i := strings.Index(path, "&page="); i >= 0 { + // An unchecked scan leaves page at 1 and silently serves page one + // for every request, so a pagination bug would pass this test. + if _, err := fmt.Sscanf(path[i+len("&page="):], "%d", &page); err != nil { + return nil, fmt.Errorf("stub: parse page from %q: %w", path, err) + } + } + n := 100 + if page == 2 { + n = 40 + } + if page > 2 { + n = 0 + } + jobs := make([]string, 0, n) + for i := 0; i < n; i++ { + jobs = append(jobs, `{"name":"bazel (x)","run_id":1,"conclusion":"success"}`) + } + return []byte(fmt.Sprintf(`{"total_count":140,"jobs":[%s]}`, strings.Join(jobs, ","))), nil + } + t.Cleanup(func() { fetch = prev }) + + got := fetchJobs("o/r", []Run{{ID: 1}}, 1) + if len(got) != 140 { + t.Fatalf("got %d jobs, want 140 (page 1 + page 2)", len(got)) + } + if len(calls) != 2 { + t.Fatalf("made %d requests, want 2; must stop once total_count is reached", len(calls)) + } +} + +func TestFetchJobsStopsOnSinglePage(t *testing.T) { + var calls int + prev := fetch + fetch = func(path, repo string) ([]byte, error) { + calls++ + return []byte(`{"total_count":1,"jobs":[{"name":"bazel (x)","run_id":1,"conclusion":"success"}]}`), nil + } + t.Cleanup(func() { fetch = prev }) + + if got := fetchJobs("o/r", []Run{{ID: 1}}, 1); len(got) != 1 { + t.Fatalf("got %d jobs, want 1", len(got)) + } + if calls != 1 { + t.Fatalf("made %d requests, want 1; a short page means no more pages", calls) + } +} diff --git a/tools/ci-health/go.mod b/tools/ci-health/go.mod new file mode 100644 index 000000000..357ef372a --- /dev/null +++ b/tools/ci-health/go.mod @@ -0,0 +1,3 @@ +module ci-health + +go 1.26 diff --git a/tools/ci-health/main.go b/tools/ci-health/main.go new file mode 100644 index 000000000..4999bd914 --- /dev/null +++ b/tools/ci-health/main.go @@ -0,0 +1,327 @@ +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command ci-health answers "why is my build slow" for a GitHub Actions +// repository. +// +// A workflow's wall-clock time is not one number, it is a stack: +// +// wall clock = queue wait + critical-path job + gate jobs +// +// Only the middle term is what people mean by "the build". A run that spends +// four minutes waiting for a runner looks identical, in the Actions UI, to a run +// that spends four minutes compiling. This tool separates them, finds which +// matrix row is the long pole, and reports cache pressure, which is the usual +// reason a job that used to be fast no longer is. +// +// Usage: +// +// ci-health --dashboard # visual report, opens in a browser +// ci-health --why # same findings, as text +// ci-health # cache and quota only +// ci-health --durations # duration percentiles by week +// ci-health --merge-times # PR open to merge latency +// ci-health --all +// ci-health --workflow release-tags.yml # default: bazel.yml +// ci-health --repo OWNER/NAME # default: NVIDIA/nvcf +// +// Trends read every retained run of the workflow, which is cheap because the +// runs endpoint paginates 100 at a time. Per-job analysis costs one API call per +// run, so it is limited to the most recent --runs runs. +// +// Requires the gh CLI, authenticated. +package main + +import ( + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "time" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +type options struct { + repo string + workflow string + dashboard string + why bool + durations bool + mergeTimes bool + all bool + runs int + history int + weeks int + prs int + noOpen bool +} + +func parseFlags() *options { + o := &options{} + flag.StringVar(&o.repo, "repo", "NVIDIA/nvcf", "repository as OWNER/NAME") + flag.StringVar(&o.workflow, "workflow", "bazel.yml", "workflow file name") + flag.StringVar(&o.dashboard, "dashboard", "", "write a self-contained HTML dashboard to this path (default build-health.html when the flag is given without a value)") + flag.BoolVar(&o.why, "why", false, "rank the causes of slowness as text") + flag.BoolVar(&o.durations, "durations", false, "duration percentiles by week") + flag.BoolVar(&o.mergeTimes, "merge-times", false, "PR open-to-merge latency") + flag.BoolVar(&o.all, "all", false, "cache, durations and merge times") + flag.IntVar(&o.runs, "runs", 60, "runs to pull per-job detail for") + flag.IntVar(&o.history, "history", 1000, "runs to trend over") + flag.IntVar(&o.weeks, "weeks", 12, "weeks of trend to print") + flag.IntVar(&o.prs, "prs", 100, "merged PRs to sample") + flag.BoolVar(&o.noOpen, "no-open", false, "do not launch a browser") + + // Allow a bare --dashboard with no value, which is the common case. + for i, a := range os.Args { + if a == "--dashboard" || a == "-dashboard" { + if i+1 >= len(os.Args) || len(os.Args[i+1]) > 0 && os.Args[i+1][0] == '-' { + os.Args = append(os.Args[:i+1:i+1], append([]string{"build-health.html"}, os.Args[i+1:]...)...) + } + break + } + } + flag.Parse() + return o +} + +func run() error { + o := parseFlags() + + // These reach slice bounds, so a negative value panics rather than erroring. + for _, c := range []struct { + name string + v int + }{{"runs", o.runs}, {"history", o.history}, {"weeks", o.weeks}, {"prs", o.prs}} { + if c.v < 1 { + return fmt.Errorf("--%s must be at least 1, got %d", c.name, c.v) + } + } + + workflow := o.workflow + if filepath.Ext(workflow) != ".yml" && filepath.Ext(workflow) != ".yaml" { + workflow += ".yml" + } + + needJobs := o.dashboard != "" || o.why || o.all + needRuns := needJobs || o.durations + + var ( + runsList []Run + jobs []Job + truncated bool + ) + if needRuns { + var err error + runsList, err = fetchRuns(o.repo, workflow, o.history) + if err != nil { + return err + } + if len(runsList) == 0 { + return fmt.Errorf("no successful runs found for workflow %q in %s", workflow, o.repo) + } + truncated = len(runsList) >= o.history + } + var sampled []Run + if needJobs { + sampled = runsList + if len(sampled) > o.runs { + sampled = sampled[:o.runs] + } + jobs = fetchJobs(o.repo, sampled, 8) + } + + // --durations and --merge-times read no cache data, so they should neither + // pay for these calls nor fail when cache access does. + needCache := needJobs || (!o.why && !o.durations && !o.mergeTimes) + var cache CacheState + if needCache { + usage := struct { + Size int64 `json:"active_caches_size_in_bytes"` + Count int `json:"active_caches_count"` + }{} + if err := getJSON("actions/cache/usage", o.repo, &usage); err != nil { + return err + } + caches, err := fetchCaches(o.repo) + if err != nil { + return err + } + cache = summariseCaches(caches, usage.Size, usage.Count) + } + + stats := analyseJobs(jobs) + poles, poleRuns := longPoles(jobs) + var causes []Cause + var wall float64 + if needJobs { + causes, wall = diagnose(sampled, stats, poles, poleRuns, cache) + } + + if o.dashboard != "" { + out, err := filepath.Abs(o.dashboard) + if err != nil { + return err + } + generated := time.Now().UTC().Format("2006-01-02 15:04 UTC") + body := renderHTML(o.repo, workflow, runsList, stats, poles, poleRuns, cache, causes, wall, generated, truncated) + if err := os.WriteFile(out, []byte(body), 0o644); err != nil { + return err + } + fmt.Printf("wrote %s\n", out) + if !o.noOpen { + openBrowser(out) + } + return nil + } + + if o.why || o.all { + printWhy(causes, wall, sampled) + } + if o.durations || o.all { + printDurations(runsList, workflow, o.weeks, truncated) + } + if o.mergeTimes || o.all { + lat, err := fetchMergeLatencies(o.repo, o.prs) + if err != nil { + return err + } + printMergeTimes(lat) + } + if !o.why && !o.durations && !o.mergeTimes { + printCache(cache) + } else if o.all { + fmt.Println() + printCache(cache) + } + return nil +} + +func openBrowser(path string) { + url := "file://" + path + var cmd *exec.Cmd + switch runtime.GOOS { + case "darwin": + cmd = exec.Command("open", url) + case "windows": + cmd = exec.Command("rundll32", "url.dll,FileProtocolHandler", url) + default: + cmd = exec.Command("xdg-open", url) + } + // Best effort: a headless environment has no browser, and that is not an + // error worth failing the report over. + _ = cmd.Start() +} + +func printCache(c CacheState) { + quota := c.TotalGB / quotaGB * 100 + fmt.Printf("cache: %.2f GB / %.0f GB across %d entries (%.0f%% of quota)\n", c.TotalGB, quotaGB, c.Count, quota) + switch { + case quota >= 90: + fmt.Println(" AT QUOTA: new entries are evicting existing ones. Builds still pass, just colder.") + case quota >= 75: + fmt.Println(" approaching quota; expect evictions soon") + } + if len(c.Families) > 0 { + type fr struct { + n string + s float64 + } + var fams []fr + for n, s := range c.Families { + fams = append(fams, fr{n, s}) + } + sort.Slice(fams, func(i, j int) bool { + if fams[i].s == fams[j].s { + return fams[i].n < fams[j].n + } + return fams[i].s > fams[j].s + }) + fmt.Println("\nlargest families:") + for i, f := range fams { + if i >= 8 { + break + } + fmt.Printf(" %6.2f GB %4.1f%% of quota x%-3d %s\n", f.s, f.s/quotaGB*100, c.Copies[f.n], f.n) + } + } + if len(c.Dupes) > 0 { + fmt.Printf("\nkeys stored under multiple refs: %d\n", len(c.Dupes)) + if c.StrandedGB > 0 { + fmt.Printf(" %.2f GB of that is on merge-queue refs, which are unrestorable\n", c.StrandedGB) + } + } +} + +func printWhy(causes []Cause, wall float64, runs []Run) { + mins := make([]float64, 0, len(runs)) + for _, r := range runs { + mins = append(mins, r.Minutes) + } + fmt.Printf("\nmedian run %.1f min, p90 %.1f min over %d successful runs\n\n", wall, pctl(mins, 0.9), len(runs)) + if len(causes) == 0 { + fmt.Println("nothing is dominating the wall clock right now") + return + } + for i, c := range causes { + fmt.Printf("%d. %s\n %s\n -> %s\n\n", i+1, c.Title, c.Detail, c.Fix) + } +} + +func printDurations(runs []Run, workflow string, show int, truncated bool) { + weeks := weekly(runs, truncated) + if len(weeks) == 0 { + fmt.Printf("\nworkflow %q: no successful runs in the retained window\n", workflow) + return + } + if len(weeks) > show { + weeks = weeks[len(weeks)-show:] + } + fmt.Printf("\nworkflow %q (successful runs only), by week:\n", workflow) + for _, w := range weeks { + fmt.Printf(" %s runs=%-5d median=%6.1f min p90=%6.1f min\n", w.Label, w.Count, w.P50, w.P90) + } + fmt.Println(" NOTE: a low median can mean rows were skipped by change detection,") + fmt.Println(" not that builds got faster. Compare p90 and run counts too.") +} + +func printMergeTimes(lat []PRLatency) { + if len(lat) == 0 { + fmt.Println("\nno merged PRs in the sampled window") + return + } + hours := make([]float64, 0, len(lat)) + for _, l := range lat { + hours = append(hours, l.Hours) + } + fmt.Printf("\nPR open -> merge, %d merged PRs sampled:\n", len(lat)) + fmt.Printf(" median %.1f h p90 %.1f h\n", pctl(hours, 0.5), pctl(hours, 0.9)) + sort.Slice(lat, func(i, j int) bool { return lat[i].Hours > lat[j].Hours }) + fmt.Println(" slowest:") + for i, l := range lat { + if i >= 5 { + break + } + fmt.Printf(" #%-6d %8.1f h\n", l.Number, l.Hours) + } +} diff --git a/tools/ci-health/render.go b/tools/ci-health/render.go new file mode 100644 index 000000000..931cf2d56 --- /dev/null +++ b/tools/ci-health/render.go @@ -0,0 +1,344 @@ +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "html" + "sort" + "strings" +) + +// The dashboard is deliberately one self-contained file: inline SVG, no CDN and +// no JavaScript dependency. It renders offline and adds nothing to the +// repository's dependency surface. + +const dashboardCSS = ` +:root{--bg:#0f1115;--panel:#171a21;--line:#252a34;--fg:#e6e9ef;--dim:#98a1b3;--nv:#76b900;--warn:#e8b339;--bad:#e05252} +*{box-sizing:border-box} +body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif} +.wrap{max-width:960px;margin:0 auto;padding:34px 22px 70px} +h1{font-size:23px;margin:0 0 4px} +h2{font-size:15px;margin:0 0 14px;letter-spacing:.03em;text-transform:uppercase;color:var(--dim);font-weight:600} +.sub{color:var(--dim);margin:0 0 26px;font-size:13px} +.card{background:var(--panel);border:1px solid var(--line);border-radius:9px;padding:20px 22px;margin-bottom:18px} +.cause{border-left:3px solid var(--nv);padding:11px 0 11px 15px;margin-bottom:15px} +.cause:last-child{margin-bottom:0} +.cause .t{font-weight:600;margin-bottom:3px} +.cause .d{color:var(--dim)} +.cause .f{color:var(--nv);margin-top:5px;font-size:13px} +.cause.note{border-left-color:var(--warn)} +.cause.note .f{color:var(--warn)} +.stats{display:flex;gap:34px;flex-wrap:wrap;margin-bottom:6px} +.stat .v{font-size:26px;font-weight:600} +.stat .k{color:var(--dim);font-size:12px;text-transform:uppercase;letter-spacing:.05em} +.chart{width:100%;height:auto} +.grid{stroke:var(--line);stroke-width:1} +.l50{fill:none;stroke:var(--nv);stroke-width:2.2} +.l90{fill:none;stroke:#3d6ea8;stroke-width:1.6;stroke-dasharray:5 4} +.dot{fill:var(--nv)} +.ylab{fill:var(--dim);font-size:11px;text-anchor:end} +.xlab{fill:var(--dim);font-size:10px;text-anchor:middle} +.xcnt{fill:#5d6675;font-size:9px;text-anchor:middle} +.rowlab{fill:var(--fg);font-size:12px;text-anchor:end} +.rowval{fill:var(--dim);font-size:11px} +.bq{fill:var(--warn)} +.be{fill:var(--nv)} +.legend{color:var(--dim);font-size:12px;margin-top:10px} +.legend i{display:inline-block;width:10px;height:10px;border-radius:2px;margin:0 5px 0 16px;vertical-align:middle} +.legend i:first-child{margin-left:0} +.quota{position:relative;background:#22262f;border-radius:5px;height:32px;overflow:hidden} +.quotafill{height:100%} +.quotafill.ok{background:var(--nv)}.quotafill.warn{background:var(--warn)}.quotafill.over{background:var(--bad)} +.quotatxt{position:absolute;left:12px;top:7px;font-size:13px;font-weight:600;text-shadow:0 1px 3px rgba(0,0,0,.75)} +table{width:100%;border-collapse:collapse;font-size:13px} +th{text-align:left;color:var(--dim);font-weight:600;padding:7px 8px;border-bottom:1px solid var(--line)} +td{padding:7px 8px;border-bottom:1px solid var(--line)} +td.n{text-align:right;font-variant-numeric:tabular-nums} +.empty{color:var(--dim)} +code{background:#22262f;padding:1px 5px;border-radius:3px;font-size:12px} +` + +func esc(s string) string { return html.EscapeString(s) } + +func lineChart(weeks []Week) string { + if len(weeks) == 0 { + return "

no data

" + } + const w, h, pad = 860.0, 272.0, 46.0 + top := 1.0 + for _, k := range weeks { + if k.P90 > top { + top = k.P90 + } + } + top *= 1.15 + n := len(weeks) + step := (w - pad*2) / math1(n-1) + x := func(i int) float64 { return pad + float64(i)*step } + y := func(v float64) float64 { return h - pad - (v/top)*(h-pad*2) } + + var b strings.Builder + fmt.Fprintf(&b, "", w, h) + for g := 0; g < 5; g++ { + gy := pad + float64(g)*(h-pad*2)/4 + fmt.Fprintf(&b, "", pad, gy, w-pad, gy) + fmt.Fprintf(&b, "%.0fm", pad-8, gy+4, top*(1-float64(g)/4)) + } + path := func(pick func(Week) float64) string { + var p strings.Builder + for i, k := range weeks { + cmd := "L" + if i == 0 { + cmd = "M" + } + fmt.Fprintf(&p, "%s%.1f,%.1f ", cmd, x(i), y(pick(k))) + } + return strings.TrimSpace(p.String()) + } + fmt.Fprintf(&b, "", path(func(k Week) float64 { return k.P90 })) + fmt.Fprintf(&b, "", path(func(k Week) float64 { return k.P50 })) + for i, k := range weeks { + fmt.Fprintf(&b, "%s: median %.1f min, p90 %.1f min, %d runs", + x(i), y(k.P50), esc(k.Label), k.P50, k.P90, k.Count) + label := k.Label + if len(label) > 3 { + label = label[len(label)-3:] + } + fmt.Fprintf(&b, "%s", x(i), h-pad+18, esc(label)) + // The run count sits under the label so a thin week is never mistaken + // for a real speedup. + fmt.Fprintf(&b, "n=%d", x(i), h-pad+31, k.Count) + } + b.WriteString("") + return b.String() +} + +func math1(n int) float64 { + if n < 1 { + return 1 + } + return float64(n) +} + +type barRow struct { + Name string + Queue float64 + Exec float64 + Runs int + Skipped int +} + +func stackedBars(rows []barRow) string { + if len(rows) == 0 { + return "

no data

" + } + const w, bar, pad = 860.0, 26.0, 210.0 + top := 0.0 + for _, r := range rows { + if r.Queue+r.Exec > top { + top = r.Queue + r.Exec + } + } + if top == 0 { + top = 1 + } + span := w - pad - 90 + h := float64(len(rows))*(bar+8) + 16 + + var b strings.Builder + fmt.Fprintf(&b, "", w, h) + for i, r := range rows { + y := 8 + float64(i)*(bar+8) + qw := r.Queue / top * span + ew := r.Exec / top * span + short := r.Name + if len([]rune(short)) > 30 { + short = string([]rune(short)[:29]) + "…" + } + fmt.Fprintf(&b, "%s", pad-10, y+bar*0.68, esc(short)) + fmt.Fprintf(&b, "%s: queue %.1f min", + pad, y, qw, bar, esc(r.Name), r.Queue) + fmt.Fprintf(&b, "%s: execute %.1f min over %d runs, %d skipped", + pad+qw, y, ew, bar, esc(r.Name), r.Exec, r.Runs, r.Skipped) + fmt.Fprintf(&b, "%.1fm", pad+qw+ew+8, y+bar*0.68, r.Queue+r.Exec) + } + b.WriteString("") + return b.String() +} + +func quotaBar(c CacheState) string { + used := c.TotalGB / quotaGB * 100 + if used > 100 { + used = 100 + } + cls := "ok" + if used >= 90 { + cls = "over" + } else if used >= 75 { + cls = "warn" + } + return fmt.Sprintf("
"+ + "%.2f GB / %.0f GB (%.0f%%) across %d entries
", + cls, used, c.TotalGB, quotaGB, used, c.Count) +} + +func buildRows(stats map[string]*JobStat) []barRow { + var rows []barRow + for n, s := range stats { + if s.Runs == 0 || isGate(n) { + continue + } + rows = append(rows, barRow{Name: n, Queue: pctl(s.Queue, 0.5), Exec: pctl(s.Exec, 0.5), Runs: s.Runs, Skipped: s.Skipped}) + } + sort.Slice(rows, func(i, j int) bool { + a, b := rows[i].Queue+rows[i].Exec, rows[j].Queue+rows[j].Exec + if a == b { + return rows[i].Name < rows[j].Name + } + return a > b + }) + if len(rows) > 16 { + rows = rows[:16] + } + return rows +} + +func renderHTML(repo, workflow string, runs []Run, stats map[string]*JobStat, + poles map[string]int, poleRuns int, cache CacheState, causes []Cause, + wall float64, generated string, truncated bool) string { + + var causeHTML strings.Builder + for _, c := range causes { + cls := "cause" + if c.Note { + cls += " note" + } + fmt.Fprintf(&causeHTML, "
%s
%s
%s
", + cls, esc(c.Title), esc(c.Detail), esc(c.Fix)) + } + if len(causes) == 0 { + causeHTML.WriteString("

Nothing is dominating the wall clock right now.

") + } + + type famRow struct { + name string + size float64 + } + var fams []famRow + for n, s := range cache.Families { + fams = append(fams, famRow{n, s}) + } + sort.Slice(fams, func(i, j int) bool { + if fams[i].size == fams[j].size { + return fams[i].name < fams[j].name + } + return fams[i].size > fams[j].size + }) + var famHTML strings.Builder + for i, f := range fams { + if i >= 8 { + break + } + fmt.Fprintf(&famHTML, "%s%.2f GB%.0f%%%d", + esc(f.name), f.size, f.size/quotaGB*100, cache.Copies[f.name]) + } + + type poleRow struct { + name string + n int + } + var pl []poleRow + for n, c := range poles { + pl = append(pl, poleRow{n, c}) + } + sort.Slice(pl, func(i, j int) bool { + if pl[i].n == pl[j].n { + return pl[i].name < pl[j].name + } + return pl[i].n > pl[j].n + }) + var poleHTML strings.Builder + if poleRuns > 0 { + poleHTML.WriteString("

What finishes last

") + for i, p := range pl { + if i >= 8 { + break + } + fmt.Fprintf(&poleHTML, "", + esc(p.name), p.n, float64(p.n)/float64(poleRuns)*100) + } + poleHTML.WriteString("
JobRuns where it was lastShare
%s%d%.0f%%
") + } + + mins := make([]float64, 0, len(runs)) + for _, r := range runs { + mins = append(mins, r.Minutes) + } + weeks := weekly(runs, truncated) + span := "no runs" + if len(runs) > 0 { + span = fmt.Sprintf("%s to %s", runs[len(runs)-1].Start.Format("2006-01-02"), runs[0].Start.Format("2006-01-02")) + } + + stranded := "" + if cache.StrandedGB > 0 { + stranded = fmt.Sprintf("

%.2f GB is stored on gh-readonly-queue refs. "+ + "Those branches are deleted when the merge queue drains, so the entries can never be restored but still count against quota.

", cache.StrandedGB) + } + + return fmt.Sprintf(` + +Build health: %s
+

Why is my build slow?

+

%s · workflow %s · %d successful runs (%s) · +per-job detail from the most recent %d runs · generated %s

+ +

Ranked causes

%s
+ +

Wall clock

+
+
%.1f min
median run
+
%.1f min
p90 run
+
%d
weeks of history
+
+%s +
medianp90 +· hover a point for the week and run count
+ +

Where each job's time goes

+%s +
waiting for a runner +executing · median per job; gate jobs excluded
+ +%s + +

Cache

+%s +

Largest families. GitHub allows %.0f GB per repository and +evicts least-recently-used entries once full.

+%s
FamilySizeQuotaCopies
+%s
+
`, + esc(repo), dashboardCSS, + esc(repo), esc(workflow), len(runs), esc(span), poleRuns, esc(generated), + causeHTML.String(), + wall, pctl(mins, 0.9), len(weeks), + lineChart(weeks), + stackedBars(buildRows(stats)), + poleHTML.String(), + quotaBar(cache), quotaGB, famHTML.String(), stranded) +} diff --git a/tools/ci/ci-health b/tools/ci/ci-health new file mode 100755 index 000000000..861e4078c --- /dev/null +++ b/tools/ci/ci-health @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Stable entrypoint for the ci-health Go tool, which answers "why is my build +# slow" for a GitHub Actions repository. +# +# tools/ci/ci-health --dashboard # visual report, opens in a browser +# tools/ci/ci-health --why # same findings, as text +# tools/ci/ci-health # cache and quota only +# tools/ci/ci-health --durations # duration percentiles by week +# tools/ci/ci-health --merge-times # PR open to merge latency +# tools/ci/ci-health --help +# +# Requires Go and an authenticated `gh` CLI. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec go run -C "${here}/../ci-health" . "$@"