diff --git a/.github/workflows/refresh-docs.yml b/.github/workflows/refresh-docs.yml index ff79ac6..34c012e 100644 --- a/.github/workflows/refresh-docs.yml +++ b/.github/workflows/refresh-docs.yml @@ -1,10 +1,13 @@ name: Refresh docs # Triggered when Iterable/iterable-docs publishes a change to an SDK doc -# path. The docs repo dispatches `iterable-docs-changed`; this workflow -# pulls the fresh markdown, runs the deterministic transform, and opens a -# PR. There is no LLM step — the corpus is the docs reshaped. A reviewer -# refreshes the snapshot and merges. +# path. The docs repo dispatches `iterable-docs-changed` (optionally with a +# `client_payload.ref` = the docs commit); this workflow resolves that ref +# (or the docs default-branch head on a manual run), fetches the fresh +# markdown at that commit, runs the deterministic transform, bumps the +# pinned `source.ref` in pipeline/config to match, and opens a PR. There is +# no LLM step — the corpus is the docs reshaped. A reviewer refreshes the +# snapshot and merges. # # `workflow_dispatch` is kept as a manual fallback for re-running a # refresh outside of a docs-side trigger (e.g. when validating a config @@ -50,10 +53,32 @@ jobs: working-directory: pipeline run: pnpm install --frozen-lockfile + - name: Resolve source ref + id: ref + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # The docs commit that triggered the dispatch, if any. Empty on a + # manual workflow_dispatch → we resolve the docs default branch head. + PAYLOAD_REF: ${{ github.event.client_payload.ref }} + run: | + set -euo pipefail + docs_repo="Iterable/iterable-docs" + req="${PAYLOAD_REF:-}" + if [[ -z "$req" ]]; then + req=$(gh api "repos/${docs_repo}" --jq '.default_branch') + fi + # Resolve whatever we have (branch name or sha) to a full commit sha, + # so the pin we write back is immutable and reproducible. + sha=$(gh api "repos/${docs_repo}/commits/${req}" --jq '.sha') + echo "sha=${sha}" >> "$GITHUB_OUTPUT" + echo "label=master @ $(date -u +%Y-%m-%d)" >> "$GITHUB_OUTPUT" + echo "Resolved ${req} → ${sha}" + - name: Fetch sources (uses GITHUB_TOKEN for gh api) working-directory: pipeline env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SOURCE_REF: ${{ steps.ref.outputs.sha }} run: pnpm fetch:sources -- ${{ github.event.inputs.platform || github.event.client_payload.platform || 'android' }} - name: Polish Layer A @@ -74,6 +99,15 @@ jobs: | sort -u | paste -sd ', ' -) echo "slugs=${slugs}" >> "$GITHUB_OUTPUT" + - name: Bump pinned source ref + if: steps.changes.outputs.changed == 'true' + working-directory: pipeline + run: >- + pnpm set:source-ref -- + ${{ github.event.inputs.platform || github.event.client_payload.platform || 'android' }} + ${{ steps.ref.outputs.sha }} + "${{ steps.ref.outputs.label }}" + - name: Open refresh PR if: steps.changes.outputs.changed == 'true' uses: peter-evans/create-pull-request@v6 @@ -93,6 +127,8 @@ jobs: a deterministic transform of the docs. **Slugs touched:** ${{ steps.changes.outputs.slugs }} + **Docs commit:** `${{ steps.ref.outputs.sha }}` + (`source.ref` in `pipeline/config` is bumped to this in the diff). ## Reviewer steps @@ -100,11 +136,13 @@ jobs: 2. Spot-check the diff against `sources/` for content fidelity — the transform only strips boilerplate / normalizes structure, so headings and code should match the upstream docs. - 3. Run `pnpm snapshot:refresh` and commit the resulting + 3. Confirm the `source.ref` bump in `pipeline/config` matches the + docs commit above (provenance stays honest). + 4. Run `pnpm snapshot:refresh` and commit the resulting `iterable-android/snapshot/` changes. CI's `snapshot:verify` gate will fail the build otherwise. - 4. Confirm `pnpm check:all` is green locally. - 5. Merge to `main`. Context7 picks up the change on its next + 5. Confirm `pnpm check:all` is green locally. + 6. Merge to `main`. Context7 picks up the change on its next crawl (`context7.json` controls scope). labels: | docs-refresh diff --git a/pipeline/package.json b/pipeline/package.json index 4648bed..182d98c 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -8,6 +8,7 @@ }, "scripts": { "fetch:sources": "tsx src/fetch.ts", + "set:source-ref": "tsx src/set-source-ref.ts", "polish:a": "tsx src/polish-layer-a.ts", "validate:polished": "tsx src/validate-polished.ts", "validate:plugins": "tsx src/validate-plugins.ts", diff --git a/pipeline/src/fetch.ts b/pipeline/src/fetch.ts index 57370d2..c496b98 100644 --- a/pipeline/src/fetch.ts +++ b/pipeline/src/fetch.ts @@ -12,6 +12,12 @@ * Usage: * pnpm fetch:sources # picks the single config in pipeline/config * pnpm fetch:sources -- android # explicit platform when several configs exist + * + * Ref override: set SOURCE_REF to fetch from a specific commit/branch instead + * of the config's pinned `source.ref`. The auto-refresh workflow passes the + * docs commit that triggered it (or `master`) so a dispatch actually pulls the + * new docs; `set-source-ref.ts` then writes the resolved SHA back into the + * config. Unset (local/manual runs) → the config pin is used, unchanged. */ import { execFileSync } from "node:child_process"; @@ -106,9 +112,15 @@ function main() { const outDir = resolve(REPO_ROOT, config.paths.sources_dir); mkdirSync(outDir, { recursive: true }); - const refDisplay = config.source.ref_label - ? `${config.source.ref.slice(0, 7)} (${config.source.ref_label})` - : config.source.ref.slice(0, 7); + // SOURCE_REF overrides the pinned config ref (see header). A branch name + // (e.g. "master") resolves to its head at fetch time via the contents API. + const refOverride = process.env.SOURCE_REF?.trim(); + const sourceRef = refOverride || config.source.ref; + const refDisplay = refOverride + ? `${sourceRef} (SOURCE_REF override)` + : config.source.ref_label + ? `${config.source.ref.slice(0, 7)} (${config.source.ref_label})` + : config.source.ref.slice(0, 7); console.log( `Fetching ${config.articles.length} ${config.platform} articles from ${config.source.repo}@${refDisplay}`, ); @@ -119,7 +131,7 @@ function main() { for (const article of config.articles) { const outPath = resolve(outDir, `${article.slug}.md`); const previous = existingSourceSha(outPath); - const { sha, body } = fetchArticle(config.source.repo, config.source.ref, article.source_path); + const { sha, body } = fetchArticle(config.source.repo, sourceRef, article.source_path); if (previous === sha) { console.log(` skip ${article.slug} (unchanged)`); @@ -129,7 +141,9 @@ function main() { const stamped = stampedSource(body, { sourceRepo: config.source.repo, sourcePath: article.source_path, - sourceRef: config.source.ref, + // Record the resolved ref: when SOURCE_REF is a branch, the blob sha is + // per-file, so keep source_ref as what was requested for provenance. + sourceRef, sourceSha: sha, }); writeFileSync(outPath, stamped, "utf8"); diff --git a/pipeline/src/set-source-ref.ts b/pipeline/src/set-source-ref.ts new file mode 100644 index 0000000..8f8cda3 --- /dev/null +++ b/pipeline/src/set-source-ref.ts @@ -0,0 +1,63 @@ +/** + * Writes a resolved source ref back into pipeline/config/.yml. + * + * The auto-refresh workflow fetches docs at a moving ref (the dispatched docs + * commit, or `master`), then calls this to advance the config's pinned + * `source.ref` + `ref_label` to that commit — so the pin stays an honest record + * of "last refreshed at" and future diffs reflect only new upstream changes. + * Comments in the YAML are preserved (parseDocument, not parse+stringify). + * + * Usage: + * tsx src/set-source-ref.ts [label] + * pnpm set:source-ref -- android "master @ 2026-07-28" + * + * No-ops (exit 0, prints "unchanged") when the config already pins , so + * the workflow can call it unconditionally. + */ + +import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseDocument } from "yaml"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(HERE, "../.."); +const CONFIG_DIR = resolve(REPO_ROOT, "pipeline/config"); + +const SHA_RE = /^[0-9a-f]{40}$/; + +function main(): void { + const [platform, sha, label] = process.argv.slice(2); + if (!platform || !sha) { + console.error("Usage: set-source-ref.ts [label]"); + process.exit(1); + } + if (!SHA_RE.test(sha)) { + console.error(`Refusing to pin a non-40-hex ref: "${sha}". Resolve the branch to a full commit sha first.`); + process.exit(1); + } + + const configPath = resolve(CONFIG_DIR, `${platform}.yml`); + if (!existsSync(configPath)) { + console.error(`Config not found: ${configPath}`); + process.exit(1); + } + + const doc = parseDocument(readFileSync(configPath, "utf8")); + const current = doc.getIn(["source", "ref"]); + if (current === sha) { + console.log(`source.ref already ${sha.slice(0, 7)} — unchanged.`); + return; + } + + doc.setIn(["source", "ref"], sha); + if (label) doc.setIn(["source", "ref_label"], label); + + writeFileSync(configPath, doc.toString(), "utf8"); + console.log( + `Pinned source.ref ${String(current).slice(0, 7)} → ${sha.slice(0, 7)}` + + (label ? ` (${label})` : ""), + ); +} + +main();