docs: publish llms.txt and markdown renditions of the docs (#3024)

Author: maxisbey

docs/hooks/llms_txt.py

@@ -0,0 +1,184 @@
+"""Generate llms.txt, llms-full.txt, and per-page markdown (https://llmstxt.org/).
+
+The hook publishes three artifacts into the built site:
+
+- `llms.txt`: a markdown index of the documentation, one link per page,
+  grouped by nav section.
+- a `.md` rendition of every prose page next to its HTML (e.g.
+  `tutorial/tools/index.md`), which is what the llms.txt links point at.
+- `llms-full.txt`: every prose page concatenated for single-fetch consumption.
+
+Page markdown is the source markdown with `--8<--` snippet includes resolved
+(so the `docs_src/` code examples appear inline) and relative links rewritten
+to absolute URLs. The API reference pages under `api/` are mkdocstrings stubs
+with no markdown source, so they are linked as rendered HTML from an Optional
+section instead of being embedded.
+
+Incremental builds (`mkdocs build --dirty`) are rejected: they skip unmodified
+pages, which would silently truncate the generated artifacts.
+"""
+
+from __future__ import annotations
+
+import posixpath
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from mkdocs.config.defaults import MkDocsConfig
+from mkdocs.exceptions import PluginError
+from mkdocs.structure.files import File, Files
+from mkdocs.structure.nav import Navigation, Section
+from mkdocs.structure.pages import Page
+
+# Pages with no markdown source, linked as HTML under "## Optional".
+_OPTIONAL_PAGES = [
+    ("api/mcp/index.md", "mcp API reference", "Auto-generated API reference for the mcp package (rendered HTML)"),
+    (
+        "api/mcp_types/index.md",
+        "mcp-types API reference",
+        "Auto-generated API reference for the mcp-types package (rendered HTML)",
+    ),
+]
+
+_SNIPPET_LINE = re.compile(r'^(?P<indent>[ \t]*)--8<-- "(?P<path>[^"\n]+)"$', flags=re.MULTILINE)
+_MD_LINK = re.compile(r'(\]\()([^)\s]+\.md)(#[^)\s]*)?( +"[^"]*")?(\))')
+
+
+@dataclass
+class _State:
+    page_markdown: dict[str, str] = field(default_factory=dict)
+    rendition_uris: set[str] = field(default_factory=set)
+    nav: Navigation | None = None
+    files: Files | None = None
+
+
+_state = _State()
+
+
+def _site_url(config: MkDocsConfig) -> str:
+    assert config.site_url is not None
+    return config.site_url.rstrip("/") + "/"
+
+
+def _md_uri(file: File) -> str:
+    return re.sub(r"\.html$", ".md", file.dest_uri)
+
+
+def on_config(config: MkDocsConfig) -> None:
+    # `mkdocs serve` rebuilds reuse the imported module; start each build clean.
+    _state.page_markdown.clear()
+    _state.rendition_uris.clear()
+    _state.nav = _state.files = None
+
+
+def on_nav(nav: Navigation, config: MkDocsConfig, files: Files) -> None:
+    _state.nav = nav
+    _state.files = files
+    _state.rendition_uris.update(page.file.src_uri for page in nav.pages if not page.file.src_uri.startswith("api/"))
+
+
+def on_page_markdown(markdown: str, page: Page, config: MkDocsConfig, files: Files) -> str | None:
+    if page.file.src_uri not in _state.rendition_uris:
+        return None
+
+    # Same anchor as the pymdownx.snippets `base_path` in mkdocs.yml.
+    repo_root = Path(config.config_file_path).parent
+
+    def include(match: re.Match[str]) -> str:
+        indent, path = match["indent"], match["path"]
+        # Mirror the snippets extension's restrict_base_path: reject paths
+        # that resolve outside the repo root.
+        resolved_path = (repo_root / path).resolve()
+        if not resolved_path.is_relative_to(repo_root.resolve()):
+            raise PluginError(f"llms_txt: snippet path {path!r} in {page.file.src_uri} escapes the repo root")
+        try:
+            content = resolved_path.read_text(encoding="utf-8").rstrip("\n")
+        except OSError as exc:
+            raise PluginError(f"llms_txt: cannot read snippet {path!r} in {page.file.src_uri}") from exc
+        # Keep a pointer to the embedded file so readers can find it on disk.
+        if path.endswith(".py"):
+            content = f"# {path}\n{content}"
+        if indent:
+            content = "\n".join(indent + line if line else line for line in content.split("\n"))
+        return content
+
+    resolved, substitutions = _SNIPPET_LINE.subn(include, markdown)
+    if substitutions != sum("--8<--" in line for line in markdown.splitlines()):
+        raise PluginError(f"llms_txt: unresolved snippet include in {page.file.src_uri}")
+
+    site_url = _site_url(config)
+    src_dir = posixpath.dirname(page.file.src_uri)
+
+    def rewrite(match: re.Match[str]) -> str:
+        opening, target, anchor, title, closing = match.groups()
+        if "://" in target:
+            return match.group(0)
+        linked = files.get_file_from_path(posixpath.normpath(posixpath.join(src_dir, target)))
+        if linked is None:
+            raise PluginError(f"llms_txt: cannot resolve link target {target!r} in {page.file.src_uri}")
+        # Pages without a markdown rendition (the api/ stubs) link to their HTML instead.
+        url = _md_uri(linked) if linked.src_uri in _state.rendition_uris else linked.url
+        return f"{opening}{site_url}{url}{anchor or ''}{title or ''}{closing}"
+
+    _state.page_markdown[page.file.src_uri] = _MD_LINK.sub(rewrite, resolved)
+    return None
+
+
+def _section_pages(section: Section) -> list[Page]:
+    pages: list[Page] = []
+    for child in section.children:
+        if isinstance(child, Page) and child.file.src_uri in _state.rendition_uris:
+            pages.append(child)
+        elif isinstance(child, Section):
+            pages.extend(_section_pages(child))
+    return pages
+
+
+def on_post_build(config: MkDocsConfig) -> None:
+    assert _state.nav is not None and _state.files is not None
+    missing = _state.rendition_uris - _state.page_markdown.keys()
+    if missing:
+        raise PluginError(f"llms_txt: pages skipped this build (is this a --dirty build?): {sorted(missing)}")
+
+    site_dir = Path(config.site_dir)
+    site_url = _site_url(config)
+
+    top_level = [
+        item for item in _state.nav.items if isinstance(item, Page) and item.file.src_uri in _state.rendition_uris
+    ]
+    sections: list[tuple[str, list[Page]]] = [("Docs", top_level)] if top_level else []
+    for item in _state.nav.items:
+        if isinstance(item, Section):
+            pages = _section_pages(item)
+            if pages:
+                sections.append((item.title, pages))
+
+    index = [f"# {config.site_name}", "", f"> {config.site_description}", ""]
+    full: list[str] = []
+    for title, pages in sections:
+        index += [f"## {title}", ""]
+        for page in pages:
+            markdown = _state.page_markdown[page.file.src_uri]
+            (site_dir / _md_uri(page.file)).write_text(markdown, encoding="utf-8")
+
+            description = page.meta.get("description")
+            tail = f": {description}" if description else ""
+            index.append(f"- [{page.title}]({site_url}{_md_uri(page.file)}){tail}")
+
+            body, h1_found = re.subn(r"\A\s*# .+\n", "", markdown)
+            if not h1_found:
+                raise PluginError(f"llms_txt: page {page.file.src_uri} does not start with an H1")
+            full += [f"# {page.title}", "", f"Source: {page.canonical_url}", "", body.strip(), ""]
+        index.append("")
+
+    index += ["## Optional", ""]
+    for src_uri, title, description in _OPTIONAL_PAGES:
+        linked = _state.files.get_file_from_path(src_uri)
+        if linked is None:
+            raise PluginError(f"llms_txt: optional page {src_uri} not found")
+        index.append(f"- [{title}]({site_url}{linked.url}): {description}")
+    index.append("")
+
+    (site_dir / "llms.txt").write_text("\n".join(index), encoding="utf-8")
+    (site_dir / "llms-full.txt").write_text("\n".join(full), encoding="utf-8")

docs/index.md

@@ -91,3 +91,6 @@ You wrote two Python functions with type hints and a docstring. The SDK does the
 * The **[Tutorial](tutorial/index.md)** walks through everything a server can do, one small step at a time.
 * Migrating from v1? Start with the **[Migration Guide](migration.md)**.
 * Hunting for an exact signature? The **[API Reference](api/mcp/index.md)** is generated from the source.
+* Reading with an LLM? This documentation is also published in the [llms.txt](https://llmstxt.org/) format:
+  [llms.txt](https://py.sdk.modelcontextprotocol.io/v2/llms.txt) is an index of the pages, and
+  [llms-full.txt](https://py.sdk.modelcontextprotocol.io/v2/llms-full.txt) contains every page in a single file.

mkdocs.yml

@@ -115,10 +115,12 @@ markdown_extensions:
   - pymdownx.superfences
   # Code examples live as complete, importable, tested files under `docs_src/`
   # and are included into pages with `--8<-- "docs_src/<chapter>/tutorialNNN.py"`
-  # (resolved against the repo root, the extension's default base_path).
+  # (resolved against the repo root regardless of the build's working
+  # directory; the extension's default base_path is the CWD).
   # `check_paths: true` + `strict: true` turn a renamed/deleted example into a
   # build failure instead of a silently empty code block.
   - pymdownx.snippets:
+      base_path: !relative $config_dir
       check_paths: true
   - pymdownx.tilde
   - pymdownx.inlinehilite
@@ -146,6 +148,9 @@ watch:
   - src
   - docs_src
 
+hooks:
+  - docs/hooks/llms_txt.py
+
 plugins:
   - search
   - social: