Add CI Job indexing pages after build to an AI Search instance

.github/workflows/publish-production.yml

@@ -3,10 +3,16 @@ on:
   push:
     branches:
       - production
+  workflow_dispatch:
+    inputs:
+      force_full_reindex:
+        description: "Rebuild the AI Search index from scratch (send every page, ignore the previous manifest)."
+        type: boolean
+        default: false
 
 jobs:
   publish:
-    if: github.repository == 'cloudflare/cloudflare-docs'
+    if: github.repository == 'cloudflare/cloudflare-docs' && github.ref == 'refs/heads/production'
     name: Production
     runs-on: ubuntu-22.04
     permissions:
@@ -78,3 +84,117 @@ jobs:
           curl -X POST "$WEBHOOK_URL" \
             -H "Content-Type: application/json" \
             -d "$JSON_PAYLOAD"
+
+  reindex:
+    name: AI Search reindex
+    if: github.repository == 'cloudflare/cloudflare-docs' && github.ref == 'refs/heads/production'
+    needs: publish
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      # gh CLI needs actions:read to list/download artifacts from prior runs.
+      actions: read
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          # Match the source extractors to the exact commit that produced the
+          # downloaded dist artifact, even if production advances meanwhile.
+          ref: ${{ github.sha }}
+      - name: Set up pnpm
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5
+        with:
+          version: 11
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
+        id: setup-node
+        with:
+          node-version: 24.x
+          cache: pnpm
+      - name: Restore node_modules (cache hit)
+        id: node-modules-cache
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
+        with:
+          path: node_modules
+          key: node-modules-${{ runner.os }}-node-${{ steps.setup-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ hashFiles('package.json') }}
+      - name: Install node_modules (cache miss)
+        run: pnpm install --frozen-lockfile
+        if: steps.node-modules-cache.outputs.cache-hit != 'true'
+
+      # Reuse the build artifact this run's publish job uploaded (no rebuild).
+      - name: Download built site
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh run download "${{ github.run_id }}" --name site-html --dir dist
+
+      # Best-effort: pull the manifest from the last successful run so the diff
+      # only sends changed pages. Missing/expired artifact => baseline.
+      - name: Restore previous manifest
+        id: previous-manifest
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mkdir -p .ai-search
+          prev_run=$(gh run list --workflow publish-production.yml --branch production \
+            --status success --limit 1 --json databaseId --jq '.[0].databaseId' 2>/dev/null || true)
+          if [ -n "$prev_run" ] && gh run download "$prev_run" --name ai-search-manifest --dir .ai-search 2>/dev/null; then
+            echo "Restored previous manifest from run $prev_run."
+            echo "restored=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No previous manifest artifact. Refusing to advance an empty baseline."
+            echo "restored=false" >> "$GITHUB_OUTPUT"
+            if [ "${{ inputs.force_full_reindex }}" != "true" ]; then
+              echo "Restore a prior manifest or manually run against a known-empty/reset instance with force_full_reindex=true." >&2
+              exit 1
+            fi
+          fi
+
+      - name: Reindex AI Search
+        env:
+          # The indexer sits behind Cloudflare Access, so the reindex call needs
+          # an Access service token (send.ts sends these as CF-Access-Client-*).
+          CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }}
+          CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}
+          # Absolute URL of the indexer worker's reindex endpoint.
+          REINDEX_URL: ${{ vars.AI_SEARCH_REINDEX_URL }}
+        run: |
+          : "${REINDEX_URL:?AI_SEARCH_REINDEX_URL repository variable is required}"
+          : "${CF_ACCESS_CLIENT_ID:?CF_ACCESS_CLIENT_ID secret is required}"
+          : "${CF_ACCESS_CLIENT_SECRET:?CF_ACCESS_CLIENT_SECRET secret is required}"
+
+          # Read + rewrite the same file so the updated manifest is ready to
+          # upload as this run's artifact (previous==manifest is safe: run.ts
+          # reads the previous manifest before overwriting it).
+          # force_full_reindex (workflow_dispatch toggle) ignores the previous
+          # manifest and sends every page. Empty on push events => incremental.
+          EXTRA=""
+          if [ "${{ inputs.force_full_reindex }}" = "true" ]; then
+            EXTRA="--force-full-reindex"
+          fi
+          pnpm exec tsx bin/ai-search-doc-events.ts \
+            --previous .ai-search/page-hashes.json \
+            --manifest .ai-search/page-hashes.json \
+            --batch-size 25 \
+            --send-url "$REINDEX_URL" \
+            $EXTRA
+
+      - name: Save updated manifest
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
+        with:
+          name: ai-search-manifest
+          path: .ai-search/page-hashes.json
+          retention-days: 7
+          overwrite: true
+
+      - name: Notify Google Chat on failure
+        if: failure()
+        env:
+          WEBHOOK_URL: ${{ secrets.CED_TEAM_ALERTS_CHANNEL_WEBHOOK }}
+          ACTOR: ${{ github.actor }}
+          REPO: ${{ github.repository }}
+          RUN_ID: ${{ github.run_id }}
+        run: |
+          MESSAGE="⚠️ *AI Search reindex* failed (site still deployed).\nActor: $ACTOR\n<https://github.com/$REPO/actions/runs/$RUN_ID|View run>"
+          JSON_PAYLOAD=$(jq -n --arg text "$MESSAGE" '{text: $text}')
+          curl -X POST "$WEBHOOK_URL" \
+            -H "Content-Type: application/json" \
+            -d "$JSON_PAYLOAD"

.gitignore

@@ -9,6 +9,9 @@ distllms/
 # Astro build cache
 .astro-cache/
 
+# Local AI Search indexing state
+.ai-search/
+
 # skills/ is fetched from middlecache via bin/fetch-skills.ts
 skills/
 !.agents/skills/

bin/ai-search-doc-events.ts

@@ -0,0 +1,3 @@
+import { run } from "./ai-search-doc-events/run";
+
+await run();

bin/ai-search-doc-events/args.ts

@@ -0,0 +1,102 @@
+import { join } from "node:path";
+import { parseArgs as nodeParseArgs } from "node:util";
+import type { Args } from "./types";
+
+export function parseArgs(): Args {
+	const { values } = nodeParseArgs({
+		args: process.argv.slice(2).filter((arg) => arg !== "--"),
+		options: {
+			dist: { type: "string", default: "dist" },
+			"source-docs-dir": { type: "string", default: "src/content/docs" },
+			"state-dir": { type: "string", default: ".ai-search" },
+			previous: { type: "string" },
+			manifest: { type: "string" },
+			events: { type: "string" },
+			"include-path-prefix": { type: "string", multiple: true, default: [] },
+			"send-url": { type: "string" },
+			"send-token-env": { type: "string" },
+			"batch-size": { type: "string", default: "100" },
+			commit: { type: "boolean", default: false },
+			"force-full-reindex": { type: "boolean", default: false },
+			concurrency: { type: "string", default: "1" },
+			"max-retries": { type: "string", default: "5" },
+			"resume-file": { type: "string" },
+			help: { type: "boolean", default: false },
+		},
+	});
+
+	if (values.help) {
+		printHelp();
+		process.exit(0);
+	}
+
+	const batchSize = Number.parseInt(values["batch-size"], 10);
+	if (!Number.isInteger(batchSize) || batchSize < 1 || batchSize > 100) {
+		throw new Error("--batch-size must be an integer from 1 to 100");
+	}
+
+	const concurrency = Number.parseInt(values.concurrency, 10);
+	if (!Number.isInteger(concurrency) || concurrency < 1) {
+		throw new Error("--concurrency must be a positive integer");
+	}
+
+	const maxRetries = Number.parseInt(values["max-retries"], 10);
+	if (!Number.isInteger(maxRetries) || maxRetries < 0) {
+		throw new Error("--max-retries must be a non-negative integer");
+	}
+
+	const stateDir = values["state-dir"];
+
+	return {
+		dist: values.dist,
+		sourceDocsDir: values["source-docs-dir"],
+		stateDir,
+		previous: values.previous ?? join(stateDir, "page-hashes.json"),
+		manifest: values.manifest ?? join(stateDir, "latest-page-hashes.json"),
+		events: values.events ?? join(stateDir, "docs-search-events.jsonl"),
+		includePathPrefixes: values["include-path-prefix"],
+		sendUrl: values["send-url"],
+		sendTokenEnv: values["send-token-env"],
+		batchSize,
+		commit: values.commit,
+		forceFullReindex: values["force-full-reindex"],
+		concurrency,
+		maxRetries,
+		resumeFile: values["resume-file"],
+	};
+}
+
+function printHelp() {
+	console.log(`Usage: pnpm exec tsx bin/ai-search-doc-events.ts [options]
+
+Build a hash manifest for rendered docs pages, diff it against local state, and optionally POST that diff.
+
+Options:
+  --dist <dir>             Astro build directory. Default: dist
+  --source-docs-dir <dir>  Source docs Markdown/MDX directory. Default: src/content/docs
+  --state-dir <dir>        Local state directory. Default: .ai-search
+  --previous <file>        Previous manifest JSON. Default: <state-dir>/page-hashes.json
+  --manifest <file>        New manifest JSON. Default: <state-dir>/latest-page-hashes.json
+  --events <file>          JSONL change events. Default: <state-dir>/docs-search-events.jsonl
+  --include-path-prefix <path>
+                           Only include built pages whose docs path starts with this prefix.
+                           Repeat to include multiple prefixes.
+  --send-url <url>         POST the diff payload to this endpoint. The endpoint
+                           enqueues each event for background indexing and
+                           returns 202 Accepted; a 2xx means the batch was queued.
+  --send-token-env <name>  Read a bearer token from this environment variable for --send-url.
+                           Default: AI_SEARCH_REINDEX_TOKEN when that variable is set.
+                           For Cloudflare Access, set CF_ACCESS_CLIENT_ID + CF_ACCESS_CLIENT_SECRET
+                           (service token) or CF_ACCESS_TOKEN (JWT from cloudflared access token).
+  --batch-size <count>     Number of events per POST when using --send-url. Default: 100
+  --concurrency <n>        Number of batches to POST in parallel. Default: 1
+  --max-retries <n>        Batch POST retry attempts on transient HTTP/network failures. Default: 5
+  --resume-file <file>     Append-only JSONL of enqueued page paths; skipped on restart
+                           so an interrupted run does not re-enqueue them.
+  --force-full-reindex     Ignore any previous manifest and send every page as docs.page.changed
+                           (full re-index). Use to rebuild the index from scratch. Without it, a
+                           run with no previous manifest baselines (sends nothing).
+  --commit                 Save the latest manifest as the new previous manifest after a successful send/diff.
+  --help                   Show this help.
+`);
+}

bin/ai-search-doc-events/content-processors/changelog-post.ts

@@ -0,0 +1,68 @@
+import { readFile } from "node:fs/promises";
+import { relative, sep } from "node:path";
+import fg from "fast-glob";
+import matter from "gray-matter";
+import { parse as parseYaml } from "yaml";
+import type { ContentTransformer, RawSection } from "../types";
+import { asArray, asRecord, asString, compactLine } from "./data";
+import { documentText } from "./mdast";
+import { makeSections } from "./utils";
+
+async function directoryEntryName(id: string) {
+	const source = await readFile(
+		`src/content/directory/${id}.yaml`,
+		"utf8",
+	).catch((error: NodeJS.ErrnoException) => {
+		if (error.code === "ENOENT") return undefined;
+		throw error;
+	});
+	if (!source) return undefined;
+	const data = asRecord(parseYaml(source));
+	return asString(data?.name) ?? asString(asRecord(data?.entry)?.title);
+}
+
+async function changelogPostSectionsForPath(
+	path: string,
+	pageTitle: string,
+): Promise<RawSection[] | undefined> {
+	const slug = path.match(/^\/changelog\/post\/([^/]+)\/$/)?.[1];
+	if (!slug) throw new Error(`Invalid changelog post path: ${path}`);
+
+	const candidates = await fg(`**/${slug}.{md,mdx}`, {
+		cwd: "src/content/changelog",
+		absolute: true,
+	});
+	const sourceFile = candidates.sort()[0];
+	if (!sourceFile) return undefined;
+
+	const source = await readFile(sourceFile, "utf8");
+	const { data: frontmatter } = matter(source);
+	const productId = relative("src/content/changelog", sourceFile).split(sep)[0];
+	const productName =
+		(await directoryEntryName(productId)) ??
+		asArray(frontmatter.products).map(String).join(", ") ??
+		productId;
+	const title = asString(frontmatter.title) ?? pageTitle;
+	const description = asString(frontmatter.description);
+	const date =
+		frontmatter.date instanceof Date
+			? frontmatter.date.toISOString().slice(0, 10)
+			: (asString(frontmatter.date) ?? undefined);
+	const bodyText = documentText(source, sourceFile.endsWith(".mdx"));
+	const text = [
+		compactLine("Title", title),
+		compactLine("Date", date),
+		compactLine("Product", productName),
+		description,
+		bodyText,
+	]
+		.filter(Boolean)
+		.join("\n\n");
+
+	return makeSections([{ anchor: "", heading: title, text }]);
+}
+
+export const changelogPostProcessor: ContentTransformer = {
+	name: "changelog-post",
+	transform: ({ path, title }) => changelogPostSectionsForPath(path, title),
+};

bin/ai-search-doc-events/content-processors/data.ts

@@ -0,0 +1,47 @@
+import { readFile } from "node:fs/promises";
+import { normalizeText, truncateText } from "../shared";
+
+export async function readJsonFile(path: string): Promise<unknown | undefined> {
+	try {
+		return JSON.parse(await readFile(path, "utf8")) as unknown;
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
+		throw error;
+	}
+}
+
+export function asRecord(value: unknown): Record<string, unknown> | undefined {
+	return value && typeof value === "object" && !Array.isArray(value)
+		? (value as Record<string, unknown>)
+		: undefined;
+}
+
+export function asString(value: unknown) {
+	return typeof value === "string" ? value : undefined;
+}
+
+export function asArray(value: unknown) {
+	return Array.isArray(value) ? value : [];
+}
+
+export function compactLine(label: string, value: unknown) {
+	if (value === undefined || value === null || value === "") return undefined;
+	if (Array.isArray(value) && value.length === 0) return undefined;
+	return `${label}: ${Array.isArray(value) ? value.join(", ") : String(value)}`;
+}
+
+export function excerpt(value: unknown, maxLength = 800): string | undefined {
+	if (value === undefined || value === null) return undefined;
+	let text: string;
+	if (typeof value === "string") text = value;
+	else if (Array.isArray(value)) {
+		text = value
+			.map((item) => (typeof item === "string" ? item : undefined))
+			.filter(Boolean)
+			.join("");
+	} else {
+		text = JSON.stringify(value);
+	}
+	const normalized = normalizeText(text.replace(/<[^>]+>/g, ""));
+	return normalized ? truncateText(normalized, maxLength) : undefined;
+}