GRO-683 - 2 - Use astro instead of jekyll to render all individual doc pages

_config.yml

@@ -81,7 +81,6 @@ available_versions:
   - v0.62
   - v0.63
 collections:
-  docs: { output: true }
   community-posts: { output: true }
 defaults:
   - {

astro.config.mjs

@@ -1,5 +1,33 @@
 // @ts-check
-import { defineConfig } from 'astro/config';
+import { defineConfig } from "astro/config";
+import { viteStaticCopy } from "vite-plugin-static-copy";
+import { noopMarkdownProcessor } from "./src/lib/markdown/noopMarkdownProcessor";
 
 // https://astro.build/config
-export default defineConfig({});
+export default defineConfig({
+  vite: {
+    plugins: [
+      viteStaticCopy({
+        targets: [
+          {
+            // TODO: What other extensions?
+            src: "_docs/**/*.{jpg,png}",
+            dest: "docs",
+            rename: { stripBase: 1 }, // strips `_docs/`
+          },
+        ],
+      }),
+    ],
+  },
+  markdown: {
+    // Use `getMarkdownRenderer` instead for faster dev builds. There are
+    // thousands of docs md files, and astro processes the markdown for all of
+    // them even if you don't even navigate to a markdown-generated page. Also,
+    // docs md files use liquid syntax which must be processed before the
+    // markdown is converted to html. Processing liquid as part of a satteri
+    // plugin would be unnecessarily and frustratingly slow for dev builds
+    // since it would need to resolve all the includes for thousands of
+    // markdown files.
+    processor: noopMarkdownProcessor,
+  },
+});

package.json

@@ -88,6 +88,7 @@
     "stylelint": "^14.9.0",
     "stylelint-config-standard-scss": "^4.0.0",
     "stylelint-order": "^5.0.0",
+    "vite-plugin-static-copy": "^4.1.1",
     "wait-on": "^9.0.10"
   },
   "resolutions": {

src/constants.ts

@@ -0,0 +1 @@
+export const DOCS_SRC_ROOT = "_docs";

src/content.config.ts

@@ -1,10 +1,12 @@
 import { defineCollection } from "astro:content";
 import { glob } from "astro/loaders";
+import { DOCS_SRC_ROOT } from "./constants";
 
-const examples = defineCollection({
+const docs = defineCollection({
   loader: glob({
-    pattern: ["src/example-collection/**/*.md"],
+    pattern: ["**/*.md", "!**/embedding/sdk/api/snippets/**"],
+    base: DOCS_SRC_ROOT,
   }),
 });
 
-export const collections = { examples };
+export const collections = { docs };

src/lib/liquid/liquidRenderer.ts

@@ -1,9 +1,44 @@
 import fs from "node:fs";
 import path from "node:path";
-import { Liquid } from "liquidjs";
+import { Liquid, ParseError, TokenizationError } from "liquidjs";
 import YAML from "yamljs";
 import { registerIncludeFileTag } from "./tags/includeFileTag";
 
+const MAX_LIQUID_SYNTAX_ERRORS = 25;
+
+// Old markdown docs sometimes contain text that merely looks like Liquid
+// (e.g. "{{#...}}" used to describe template tag syntax) but isn't valid
+// Liquid. Jekyll/Ruby rendered these as blank rather than failing the build,
+// so on a syntax error we cut out just the offending "{{ }}"/"{% %}" span and
+// retry, instead of taking down the whole page.
+const stripInvalidLiquidSpan = (
+  source: string,
+  err: unknown,
+): string | null => {
+  if (!(err instanceof TokenizationError) && !(err instanceof ParseError)) {
+    return null;
+  }
+
+  let { begin, end } = err.token;
+  if (!(source.startsWith("{{", begin) || source.startsWith("{%", begin))) {
+    const outputOpen = source.lastIndexOf("{{", begin);
+    const tagOpen = source.lastIndexOf("{%", begin);
+    let openStart = outputOpen;
+    let closer = "}}";
+    if (tagOpen > outputOpen) {
+      openStart = tagOpen;
+      closer = "%}";
+    }
+    if (openStart === -1) return null;
+    const closeIdx = source.indexOf(closer, end);
+    if (closeIdx === -1) return null;
+    begin = openStart;
+    end = closeIdx + closer.length;
+  }
+
+  return source.slice(0, begin) + source.slice(end);
+};
+
 const ROOT = process.cwd();
 const INCLUDES_ROOT = path.join(ROOT, "_includes");
 
@@ -63,14 +98,31 @@ export const getLiquidRenderer = ({
   };
 
   return {
-    render: (
+    render: async (
       html: string,
       { include }: { include?: Record<string, unknown> } = {},
     ) => {
-      return liquidEngine.parseAndRender(html, {
-        ...ctx,
-        include,
-      });
+      let source = html;
+      for (let attempt = 0; attempt < MAX_LIQUID_SYNTAX_ERRORS; attempt++) {
+        try {
+          return await liquidEngine.parseAndRender(source, {
+            ...ctx,
+            include,
+          });
+        } catch (err) {
+          const stripped = stripInvalidLiquidSpan(source, err);
+          if (stripped === null) throw err;
+          console.warn(
+            `[liquid] Ignoring invalid Liquid syntax in ${dirname}: ${
+              (err as Error).message
+            }`,
+          );
+          source = stripped;
+        }
+      }
+      throw new Error(
+        `[liquid] Too many invalid Liquid syntax errors in ${dirname}`,
+      );
     },
   };
 };

src/lib/markdown/markdownRenderer.ts

@@ -1,13 +1,15 @@
 import { satteri } from "@astrojs/markdown-satteri";
 import { ialHastPlugin } from "./plugins/ialHastPlugin";
 import { inlineCodeHastPlugin } from "./plugins/inlineCodeHastPlugin";
+import { relativeImagePlugin } from "./plugins/relativeImagePlugin";
 import { responsiveTableLabelsHastPlugin } from "./plugins/responsiveTableLabelsHastPlugin";
 
 const docsMarkdownProcessor = satteri({
   hastPlugins: [
     ialHastPlugin,
     inlineCodeHastPlugin,
     responsiveTableLabelsHastPlugin,
+    relativeImagePlugin,
   ],
 });
 

src/lib/markdown/noopMarkdownProcessor.ts

@@ -0,0 +1,17 @@
+import type { MarkdownProcessor } from "astro/markdown";
+
+export const noopMarkdownProcessor: MarkdownProcessor = {
+  name: "no-op",
+  options: {},
+  createRenderer: async () => ({
+    render: async () => ({
+      code: "",
+      metadata: {
+        headings: [],
+        localImagePaths: [],
+        remoteImagePaths: [],
+        frontmatter: {},
+      },
+    }),
+  }),
+};

src/lib/markdown/plugins/relativeImagePlugin.ts

@@ -0,0 +1,28 @@
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { DOCS_SRC_ROOT } from "@/constants";
+import { defineHastPlugin } from "satteri";
+
+// Resolves relative images from docs md files.
+// The images themselves are copied via viteStaticCopy in astro.config.mjs.
+
+export const relativeImagePlugin = defineHastPlugin({
+  name: "relative-image-resolver",
+  element: {
+    filter: ["img"],
+    visit(node, ctx) {
+      const rawSrc = node.properties?.src;
+      if (typeof rawSrc !== "string" || rawSrc === "") return;
+      // Absolute site paths, absolute URLs, and data: URIs need no rewriting.
+      if (rawSrc.startsWith("/") || URL.canParse(rawSrc)) return;
+      if (!ctx.fileURL) return;
+
+      const absPath = fileURLToPath(new URL(decodeURI(rawSrc), ctx.fileURL));
+      const relPath = path.relative(DOCS_SRC_ROOT, absPath);
+      if (relPath.startsWith("..") || path.isAbsolute(relPath)) return;
+
+      const newSrc = `/docs/${relPath.split(path.sep).map(encodeURIComponent).join("/")}`;
+      ctx.setProperty(node, "src", newSrc);
+    },
+  },
+});

src/pages/docs/[version]/[...slug].astro

@@ -0,0 +1,53 @@
+---
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import NewDocsLayout from "@/layouts/NewDocsLayout.astro";
+import { getLiquidRenderer } from "@/lib/liquid/liquidRenderer";
+import { getMarkdownRenderer } from "@/lib/markdown/markdownRenderer";
+import { getCollection, type DataEntryMap } from "astro:content";
+
+type Props = {
+  doc: DataEntryMap["docs"][number];
+};
+
+export const getStaticPaths = async () => {
+  const splitAtFirst = (str: string, delimiter: string) => {
+    const index = str.indexOf(delimiter);
+    return index !== -1 ? [str.slice(0, index), str.slice(index + 1)] : [str];
+  };
+  // TODO: Handle README -> index
+  // TODO: Handle permalink in frontmatter
+  // TODO: Handle redirect_to/redirect_from
+  const docs = await getCollection("docs");
+  return [
+    ...docs.map((doc) => {
+      const [version, slug] = splitAtFirst(doc.id, "/");
+      return {
+        props: { doc },
+        params: { version, slug },
+      };
+    }),
+  ];
+};
+
+const { doc } = Astro.props;
+const dirname = path.dirname(doc.filePath!);
+
+// Process liquid first (e.g. control flow, includes, variables, etc)
+const lq = getLiquidRenderer({ page: doc.data, dirname });
+const lqOutput = await lq.render(doc.body!);
+
+// Remove whitespace between elements so satteri doesn't turn them into code snippets.
+// Mimics previous behavior (which used jekyll + kramdown).
+const lqProcessed = lqOutput.replace(/>\s+</g, "><");
+
+// Convert processed markdown to html
+const md = await getMarkdownRenderer();
+const renderResult = await md.render(lqProcessed, {
+  fileURL: pathToFileURL(path.resolve(doc.filePath!)),
+});
+---
+
+<NewDocsLayout page={doc.data} dirname={dirname}>
+  <Fragment set:html={renderResult.code} />
+</NewDocsLayout>