Skip to content

Commit 2c70b95

Browse files
bartlomiejuclaudethisisjofrank
authored
ci: optimize build pipeline with caching and parallelization (#2974)
## Summary Optimizes the CI build pipeline from ~7 min to ~2.5 min (warm cache) or ~5 min (cold cache). ### What changed **1. Cache reference types + docs** (`reference_gen/types/` + `reference_gen/gen/`) - On cache hit, skips type extraction (86s) and doc generation (162s) entirely - Keyed on `hashFiles('reference_gen/**')` — only regenerates when scripts, configs, or dependency versions change - First run after merge seeds the cache; all subsequent PRs hit it **2. Cache std-docs** (`runtime/reference/std/`) - Skips 43 sequential JSR API fetches - Keyed on `hashFiles('scripts/generate_std_docs.ts')` **3. Split Lume into two builds** - New `_config-reference.ts`: minimal config that renders only API reference pages (JSX only — no markdown-it, esbuild, tailwind, sitemap, toc, or OG images) - CI runs reference build first, then main build with `SKIP_REFERENCE=1` - Both write to `_site/` with `emptyDest: false` - Main build no longer renders hundreds of API pages through the full plugin chain **4. Eliminate redundant reference generation** - Build step runs `deno task lume` directly instead of `deno task build` which re-ran `generate:reference` **5. Parallelize I/O** - afterBuild tasks (lint rule copy, LLM files, md source copy) run concurrently via `Promise.all` - Std-docs JSR fetches batched 10 at a time instead of 43 sequential **6. Fix `SKIP_REFERENCE` check** - Use `=== "1"` instead of `env.has()`, which incorrectly triggered on empty strings **7. Add build profiling** (`LUME_PROFILE=1`) - Instruments build phases and individual processor timings - `LUME_PROFILE=1 deno task build:light` to use locally ### Expected CI times | Step | Before | After (warm cache) | After (cold cache) | |------|--------|-------------------|-------------------| | Setup + fmt + test | 20s | 20s | 20s | | Reference types + docs | 248s | **~2s** | 248s | | Std-docs | 15s | **~1s** | ~5s | | Build: reference pages | _(in main build)_ | **~30-40s** | ~30-40s | | Build: main site | 158s | **~80s** | ~80s | | Server + link checker | 35s | 35s | 35s | | **Total** | **~7 min** | **~2.5 min** | **~5 min** | ### New tasks - `deno task lume:reference` — run the reference-only Lume build - `deno task build:split` — full build using the split strategy Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: Jo Franchetti <jofranchetti@gmail.com>
1 parent 5533c85 commit 2c70b95

6 files changed

Lines changed: 376 additions & 110 deletions

File tree

‎.github/workflows/lint.yml‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,45 @@ jobs:
2525
- run: deno fmt --check
2626
- run: deno task test
2727

28+
# Cache reference types + generated docs (types: 86s, docs: 162s)
29+
- name: Restore reference cache
30+
uses: actions/cache@v4
31+
id: ref-cache
32+
with:
33+
path: |
34+
reference_gen/types
35+
reference_gen/gen
36+
key: reference-${{ hashFiles('reference_gen/**') }}
37+
restore-keys: reference-
38+
2839
- name: "Reference: install"
40+
if: steps.ref-cache.outputs.cache-hit != 'true'
2941
working-directory: "reference_gen"
3042
run: deno install
3143

3244
- name: "Reference: generate types"
45+
if: steps.ref-cache.outputs.cache-hit != 'true'
3346
working-directory: "reference_gen"
3447
run: deno task types
3548

3649
- name: "Reference: generate docs"
50+
if: steps.ref-cache.outputs.cache-hit != 'true'
3751
working-directory: "reference_gen"
3852
run: deno task doc
3953

54+
# Cache std-docs to avoid 43+ network requests to JSR API on every build
55+
- name: Restore std-docs cache
56+
uses: actions/cache@v4
57+
id: std-cache
58+
with:
59+
path: runtime/reference/std
60+
key: std-docs-${{ hashFiles('scripts/generate_std_docs.ts') }}
61+
restore-keys: std-docs-
62+
63+
- name: Generate std-docs
64+
if: steps.std-cache.outputs.cache-hit != 'true'
65+
run: deno task generate:std-docs
66+
4067
- name: Restore OG image cache
4168
uses: actions/cache@v4
4269
id: og-cache
@@ -45,13 +72,21 @@ jobs:
4572
key: og-images-${{ hashFiles('open_graph/**') }}
4673
restore-keys: og-images-
4774

48-
- name: Build
75+
# Build reference pages first with a minimal Lume config (JSX only,
76+
# no markdown/esbuild/tailwind/sitemap), then build the main site
77+
# skipping reference pages. Outputs merge since both use emptyDest:false.
78+
- name: "Build: reference pages"
79+
run: deno task lume:reference
80+
81+
- name: "Build: main site"
4982
env:
5083
ORAMA_PROJECT_ID: ${{ vars.ORAMA_PROJECT_ID }}
5184
ORAMA_DATASOURCE_ID: ${{ vars.ORAMA_DATASOURCE_ID }}
5285
ORAMA_PRIVATE_API_KEY: ${{ secrets.ORAMA_PRIVATE_API_KEY }}
5386
SKIP_OG: ${{ steps.og-cache.outputs.cache-hit == 'true' && '1' || '' }}
54-
run: deno task build
87+
SKIP_REFERENCE: "1"
88+
BUILD_TYPE: FULL
89+
run: deno task lume
5590

5691
- name: Restore cached OG images
5792
if: steps.og-cache.outputs.cache-hit == 'true'

‎_config-reference.ts‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/**
2+
* Minimal Lume config for building only the API reference pages.
3+
* Run in parallel with the main build to speed up CI.
4+
*
5+
* Usage: deno run --env-file -P=lume lume/cli.ts --config=_config-reference.ts
6+
*/
7+
import lume from "lume/mod.ts";
8+
import jsx from "lume/plugins/jsx.ts";
9+
10+
import denoCategories from "./reference_gen/deno-categories.json" with {
11+
type: "json",
12+
};
13+
import webCategories from "./reference_gen/web-categories.json" with {
14+
type: "json",
15+
};
16+
import nodeRewriteMap from "./reference_gen/node-rewrite-map.json" with {
17+
type: "json",
18+
};
19+
20+
const nodeCategories = Object.keys(nodeRewriteMap);
21+
22+
const site = lume({
23+
location: new URL("https://docs.deno.com"),
24+
caseSensitiveUrls: true,
25+
emptyDest: false,
26+
});
27+
28+
// Only need JSX for rendering reference components
29+
site.use(jsx());
30+
31+
// Ignore everything except the reference page generator and its dependencies
32+
site.ignore(
33+
"old",
34+
"README.md",
35+
"runtime",
36+
"deploy",
37+
"examples",
38+
"sandbox",
39+
"subhosting",
40+
"lint",
41+
"api",
42+
"styleguide",
43+
"middleware",
44+
"js",
45+
"style.css",
46+
"static",
47+
"open_graph",
48+
"orama",
49+
"scripts",
50+
(path) => path.match(/\/reference_gen.*.ts/) !== null,
51+
(path) => path.includes("/reference_gen/node_modules"),
52+
(path) => path.includes("/reference_gen/node_descriptions"),
53+
);
54+
55+
// Copy reference CSS/JS assets
56+
site.copy("reference_gen/gen/deno/page.css", "/api/deno/page.css");
57+
site.copy("reference_gen/gen/deno/styles.css", "/api/deno/styles.css");
58+
site.copy("reference_gen/gen/deno/script.js", "/api/deno/script.js");
59+
60+
site.copy("reference_gen/gen/web/page.css", "/api/web/page.css");
61+
site.copy("reference_gen/gen/web/styles.css", "/api/web/styles.css");
62+
site.copy("reference_gen/gen/web/script.js", "/api/web/script.js");
63+
64+
site.copy("reference_gen/gen/node/page.css", "/api/node/page.css");
65+
site.copy("reference_gen/gen/node/styles.css", "/api/node/styles.css");
66+
site.copy("reference_gen/gen/node/script.js", "/api/node/script.js");
67+
68+
site.data("layout", "doc.tsx");
69+
70+
site.data("apiCategories", {
71+
deno: {
72+
title: "Deno APIs",
73+
categories: Object.keys(denoCategories),
74+
descriptions: denoCategories,
75+
getCategoryHref: (categoryName: string) => {
76+
if (categoryName === "I/O") {
77+
return `/api/deno/io`;
78+
}
79+
return `/api/deno/${categoryName.toLowerCase().replace(/\s+/g, "-")}`;
80+
},
81+
},
82+
web: {
83+
title: "Web APIs",
84+
categories: Object.keys(webCategories),
85+
descriptions: webCategories,
86+
getCategoryHref: (categoryName: string) => {
87+
if (categoryName === "I/O") {
88+
return `/api/web/io`;
89+
}
90+
return `/api/web/${categoryName.toLowerCase().replace(/\s+/g, "-")}`;
91+
},
92+
},
93+
node: {
94+
title: "Node APIs",
95+
categories: nodeCategories,
96+
descriptions: {} as Record<string, string>,
97+
getCategoryHref: (categoryName: string) => `/api/node/${categoryName}/`,
98+
},
99+
});
100+
101+
export default site;

0 commit comments

Comments
 (0)