Auto-refresh: fetch dispatched docs ref and bump the pin

.github/workflows/refresh-docs.yml

@@ -1,10 +1,13 @@
 name: Refresh docs
 
 # Triggered when Iterable/iterable-docs publishes a change to an SDK doc
-# path. The docs repo dispatches `iterable-docs-changed`; this workflow
-# pulls the fresh markdown, runs the deterministic transform, and opens a
-# PR. There is no LLM step — the corpus is the docs reshaped. A reviewer
-# refreshes the snapshot and merges.
+# path. The docs repo dispatches `iterable-docs-changed` (optionally with a
+# `client_payload.ref` = the docs commit); this workflow resolves that ref
+# (or the docs default-branch head on a manual run), fetches the fresh
+# markdown at that commit, runs the deterministic transform, bumps the
+# pinned `source.ref` in pipeline/config to match, and opens a PR. There is
+# no LLM step — the corpus is the docs reshaped. A reviewer refreshes the
+# snapshot and merges.
 #
 # `workflow_dispatch` is kept as a manual fallback for re-running a
 # refresh outside of a docs-side trigger (e.g. when validating a config
@@ -50,10 +53,32 @@ jobs:
         working-directory: pipeline
         run: pnpm install --frozen-lockfile
 
+      - name: Resolve source ref
+        id: ref
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # The docs commit that triggered the dispatch, if any. Empty on a
+          # manual workflow_dispatch → we resolve the docs default branch head.
+          PAYLOAD_REF: ${{ github.event.client_payload.ref }}
+        run: |
+          set -euo pipefail
+          docs_repo="Iterable/iterable-docs"
+          req="${PAYLOAD_REF:-}"
+          if [[ -z "$req" ]]; then
+            req=$(gh api "repos/${docs_repo}" --jq '.default_branch')
+          fi
+          # Resolve whatever we have (branch name or sha) to a full commit sha,
+          # so the pin we write back is immutable and reproducible.
+          sha=$(gh api "repos/${docs_repo}/commits/${req}" --jq '.sha')
+          echo "sha=${sha}" >> "$GITHUB_OUTPUT"
+          echo "label=master @ $(date -u +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
+          echo "Resolved ${req} → ${sha}"
+
       - name: Fetch sources (uses GITHUB_TOKEN for gh api)
         working-directory: pipeline
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SOURCE_REF: ${{ steps.ref.outputs.sha }}
         run: pnpm fetch:sources -- ${{ github.event.inputs.platform || github.event.client_payload.platform || 'android' }}
 
       - name: Polish Layer A
@@ -74,6 +99,15 @@ jobs:
             | sort -u | paste -sd ', ' -)
           echo "slugs=${slugs}" >> "$GITHUB_OUTPUT"
 
+      - name: Bump pinned source ref
+        if: steps.changes.outputs.changed == 'true'
+        working-directory: pipeline
+        run: >-
+          pnpm set:source-ref --
+          ${{ github.event.inputs.platform || github.event.client_payload.platform || 'android' }}
+          ${{ steps.ref.outputs.sha }}
+          "${{ steps.ref.outputs.label }}"
+
       - name: Open refresh PR
         if: steps.changes.outputs.changed == 'true'
         uses: peter-evans/create-pull-request@v6
@@ -93,18 +127,22 @@ jobs:
             a deterministic transform of the docs.
 
             **Slugs touched:** ${{ steps.changes.outputs.slugs }}
+            **Docs commit:** `${{ steps.ref.outputs.sha }}`
+            (`source.ref` in `pipeline/config` is bumped to this in the diff).
 
             ## Reviewer steps
 
             1. Check out this branch.
             2. Spot-check the diff against `sources/` for content fidelity —
                the transform only strips boilerplate / normalizes structure,
                so headings and code should match the upstream docs.
-            3. Run `pnpm snapshot:refresh` and commit the resulting
+            3. Confirm the `source.ref` bump in `pipeline/config` matches the
+               docs commit above (provenance stays honest).
+            4. Run `pnpm snapshot:refresh` and commit the resulting
                `iterable-android/snapshot/` changes. CI's `snapshot:verify`
                gate will fail the build otherwise.
-            4. Confirm `pnpm check:all` is green locally.
-            5. Merge to `main`. Context7 picks up the change on its next
+            5. Confirm `pnpm check:all` is green locally.
+            6. Merge to `main`. Context7 picks up the change on its next
                crawl (`context7.json` controls scope).
           labels: |
             docs-refresh

pipeline/package.json

@@ -8,6 +8,7 @@
   },
   "scripts": {
     "fetch:sources": "tsx src/fetch.ts",
+    "set:source-ref": "tsx src/set-source-ref.ts",
     "polish:a": "tsx src/polish-layer-a.ts",
     "validate:polished": "tsx src/validate-polished.ts",
     "validate:plugins": "tsx src/validate-plugins.ts",

pipeline/src/fetch.ts

@@ -12,6 +12,12 @@
  * Usage:
  *   pnpm fetch:sources              # picks the single config in pipeline/config
  *   pnpm fetch:sources -- android   # explicit platform when several configs exist
+ *
+ * Ref override: set SOURCE_REF to fetch from a specific commit/branch instead
+ * of the config's pinned `source.ref`. The auto-refresh workflow passes the
+ * docs commit that triggered it (or `master`) so a dispatch actually pulls the
+ * new docs; `set-source-ref.ts` then writes the resolved SHA back into the
+ * config. Unset (local/manual runs) → the config pin is used, unchanged.
  */
 
 import { execFileSync } from "node:child_process";
@@ -106,9 +112,15 @@ function main() {
   const outDir = resolve(REPO_ROOT, config.paths.sources_dir);
   mkdirSync(outDir, { recursive: true });
 
-  const refDisplay = config.source.ref_label
-    ? `${config.source.ref.slice(0, 7)} (${config.source.ref_label})`
-    : config.source.ref.slice(0, 7);
+  // SOURCE_REF overrides the pinned config ref (see header). A branch name
+  // (e.g. "master") resolves to its head at fetch time via the contents API.
+  const refOverride = process.env.SOURCE_REF?.trim();
+  const sourceRef = refOverride || config.source.ref;
+  const refDisplay = refOverride
+    ? `${sourceRef} (SOURCE_REF override)`
+    : config.source.ref_label
+      ? `${config.source.ref.slice(0, 7)} (${config.source.ref_label})`
+      : config.source.ref.slice(0, 7);
   console.log(
     `Fetching ${config.articles.length} ${config.platform} articles from ${config.source.repo}@${refDisplay}`,
   );
@@ -119,7 +131,7 @@ function main() {
   for (const article of config.articles) {
     const outPath = resolve(outDir, `${article.slug}.md`);
     const previous = existingSourceSha(outPath);
-    const { sha, body } = fetchArticle(config.source.repo, config.source.ref, article.source_path);
+    const { sha, body } = fetchArticle(config.source.repo, sourceRef, article.source_path);
 
     if (previous === sha) {
       console.log(`  skip   ${article.slug}  (unchanged)`);
@@ -129,7 +141,9 @@ function main() {
     const stamped = stampedSource(body, {
       sourceRepo: config.source.repo,
       sourcePath: article.source_path,
-      sourceRef: config.source.ref,
+      // Record the resolved ref: when SOURCE_REF is a branch, the blob sha is
+      // per-file, so keep source_ref as what was requested for provenance.
+      sourceRef,
       sourceSha: sha,
     });
     writeFileSync(outPath, stamped, "utf8");

pipeline/src/set-source-ref.ts

@@ -0,0 +1,63 @@
+/**
+ * Writes a resolved source ref back into pipeline/config/<platform>.yml.
+ *
+ * The auto-refresh workflow fetches docs at a moving ref (the dispatched docs
+ * commit, or `master`), then calls this to advance the config's pinned
+ * `source.ref` + `ref_label` to that commit — so the pin stays an honest record
+ * of "last refreshed at" and future diffs reflect only new upstream changes.
+ * Comments in the YAML are preserved (parseDocument, not parse+stringify).
+ *
+ * Usage:
+ *   tsx src/set-source-ref.ts <platform> <sha> [label]
+ *   pnpm set:source-ref -- android <sha> "master @ 2026-07-28"
+ *
+ * No-ops (exit 0, prints "unchanged") when the config already pins <sha>, so
+ * the workflow can call it unconditionally.
+ */
+
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseDocument } from "yaml";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = resolve(HERE, "../..");
+const CONFIG_DIR = resolve(REPO_ROOT, "pipeline/config");
+
+const SHA_RE = /^[0-9a-f]{40}$/;
+
+function main(): void {
+  const [platform, sha, label] = process.argv.slice(2);
+  if (!platform || !sha) {
+    console.error("Usage: set-source-ref.ts <platform> <sha> [label]");
+    process.exit(1);
+  }
+  if (!SHA_RE.test(sha)) {
+    console.error(`Refusing to pin a non-40-hex ref: "${sha}". Resolve the branch to a full commit sha first.`);
+    process.exit(1);
+  }
+
+  const configPath = resolve(CONFIG_DIR, `${platform}.yml`);
+  if (!existsSync(configPath)) {
+    console.error(`Config not found: ${configPath}`);
+    process.exit(1);
+  }
+
+  const doc = parseDocument(readFileSync(configPath, "utf8"));
+  const current = doc.getIn(["source", "ref"]);
+  if (current === sha) {
+    console.log(`source.ref already ${sha.slice(0, 7)} — unchanged.`);
+    return;
+  }
+
+  doc.setIn(["source", "ref"], sha);
+  if (label) doc.setIn(["source", "ref_label"], label);
+
+  writeFileSync(configPath, doc.toString(), "utf8");
+  console.log(
+    `Pinned source.ref ${String(current).slice(0, 7)} → ${sha.slice(0, 7)}` +
+      (label ? ` (${label})` : ""),
+  );
+}
+
+main();