statamic · joshuablum · Aug 10, 2026 · Aug 10, 2026 · Aug 10, 2026 · Aug 10, 2026
diff --git a/app/Http/Controllers/DocsMarkdownController.php b/app/Http/Controllers/DocsMarkdownController.php
@@ -8,10 +8,12 @@
 
 class DocsMarkdownController extends Controller
 {
-    public function __invoke(string $uri)
+    public function __invoke(string $uri = '')
     {
+        $uri = $this->normalizeUri($uri);
+
         $markdown = Cache::rememberForever("markdown.$uri", function () use ($uri) {
-            $entry = Data::findByUri('/'.$uri);
+            $entry = Data::findByUri($uri);
 
             throw_unless($entry, new NotFoundHttpException);
 
@@ -29,31 +31,59 @@ public function __invoke(string $uri)
         ]);
     }
 
+    /**
+     * `/index.md` is the conventional Markdown twin of the home page — agents guess it, and
+     * appending `.md` to the root URL isn't possible. Everything else maps straight across.
+     */
+    private function normalizeUri(string $uri): string
+    {
+        $uri = trim($uri, '/');
+
+        return ($uri === '' || $uri === 'index') ? '/' : '/'.$uri;
+    }
+
+    /**
+     * Point internal links at their Markdown twins, so an agent following links from one
+     * `.md` page stays in Markdown instead of falling back into HTML.
+     */
     private function appendMdExtensionToInternalLinks(string $markdown): string
     {
         return preg_replace_callback(
             '/(?<!!)\[([^\]]+)\]\(([^)]+)\)/',
             function ($matches) {
-                $text = $matches[1];
-                $url = $matches[2];
-
-                if ($this->shouldAppendMdExtension($url)) {
-                    $url .= '.md';
-                }
+                [, $text, $url] = $matches;
 
-                return "[$text]($url)";
+                return "[$text]({$this->markdownUrl($url)})";
             },
             $markdown
         );
     }
 
-    private function shouldAppendMdExtension(string $url): bool
+    private function markdownUrl(string $url): string
     {
-        if (preg_match('/^https?:\/\//', $url)) {
+        // Split the fragment/query off first: the extension belongs on the path, so
+        // "/tags/collection#parameters" has to become "/tags/collection.md#parameters".
+        $path = preg_split('/(?=[#?])/', $url, 2);
+        $suffix = $path[1] ?? '';
+        $path = $path[0];
+
+        return $this->shouldAppendMdExtension($path) ? $path.'.md'.$suffix : $url;
+    }
+
+    private function shouldAppendMdExtension(string $path): bool
+    {
+        // Empty path means the link was a bare fragment like "#overview".
+        if ($path === '') {
+            return false;
+        }
+
+        // Absolute URLs, protocol-relative URLs, and non-HTTP schemes (mailto:, tel:).
+        if (preg_match('/^([a-z][a-z0-9+.-]*:|\/\/)/i', $path)) {
             return false;
         }
 
-        if (preg_match('/\.[a-z0-9]{2,4}$/i', $url)) {
+        // Already points at a file.
+        if (preg_match('/\.[a-z0-9]{2,4}$/i', $path)) {
             return false;
         }
 

diff --git a/app/Http/Controllers/LlmsTxtController.php b/app/Http/Controllers/LlmsTxtController.php
@@ -2,68 +2,168 @@
 
 namespace App\Http\Controllers;
 
+use App\Support\MarkdownUrl;
 use Illuminate\Support\Facades\Cache;
+use Statamic\Contracts\Entries\Entry as EntryContract;
 use Statamic\Facades\Collection;
 use Statamic\Facades\Entry;
 
 class LlmsTxtController extends Controller
 {
+    /**
+     * Reference collections, appended after the main docs tree. These hold the bulk of the
+     * site — ~400 entries covering every tag, modifier, fieldtype and variable — and an agent
+     * that can't see them here has no way to discover that `{{ collection }}` exists.
+     */
+    private const REFERENCE_COLLECTIONS = [
+        'tags',
+        'modifiers',
+        'fieldtypes',
+        'variables',
+        'widgets',
+        'tips',
+        'troubleshooting',
+        'resource_apis',
+    ];
+
     public function __invoke()
     {
-        $lines = Cache::rememberForever("llms.txt", function () {
-            $tree = Collection::find('pages')->structure()->trees()->first()->tree();
-            $lines = ['# Statamic Documentation', ''];
+        $lines = Cache::rememberForever('llms.txt', fn () => $this->build());
 
-            foreach ($tree as $section) {
-                $children = $section['children'] ?? [];
+        return response(implode("\n", $lines), 200, [
+            'Content-Type' => 'text/plain; charset=UTF-8',
+        ]);
+    }
 
-                if (! $children) {
-                    continue;
-                }
+    private function build(): array
+    {
+        $docsVersion = config('docs.version');
+
+        $lines = [
+            '# Statamic Documentation',
+            '',
+            "> Statamic is a Laravel-powered CMS that stores content in flat files by default. This is the documentation for Statamic {$docsVersion}. Every page listed here is also available as Markdown — the `.md` URLs below return plain Markdown rather than HTML.",
+            '',
+        ];
+
+        foreach ($this->guide() as $line) {
+            $lines[] = $line;
+        }
+
+        foreach (self::REFERENCE_COLLECTIONS as $handle) {
+            foreach ($this->referenceSection($handle) as $line) {
+                $lines[] = $line;
+            }
+        }
+
+        return $lines;
+    }
+
+    /**
+     * The main docs, following the full depth of the page tree rather than just each
+     * section's immediate children.
+     */
+    private function guide(): array
+    {
+        $tree = Collection::find('pages')->structure()->trees()->first()->tree();
+        $lines = [];
+
+        foreach ($tree as $section) {
+            if (! $children = $section['children'] ?? []) {
+                continue;
+            }
 
-                $sectionEntry = Entry::find($section['entry']);
-                $lines[] = '## '.$sectionEntry->value('title');
+            if (! $sectionEntry = Entry::find($section['entry'])) {
+                continue;
+            }
+
+            $lines[] = '## '.$sectionEntry->value('title');
 
-                $firstChild = Entry::find($children[0]['entry']);
-                if ($firstChild && str_contains($firstChild->slug(), 'overview')) {
-                    if ($intro = $firstChild->value('intro')) {
-                        $lines[] = '> '.str_replace("\n", ' ', $intro);
-                    }
+            // Sections lead with an "overview" child whose intro describes the whole section.
+            $firstChild = Entry::find($children[0]['entry']);
+            if ($firstChild && str_contains($firstChild->slug(), 'overview')) {
+                if ($intro = $firstChild->value('intro')) {
+                    $lines[] = '> '.$this->oneLine($intro);
                 }
+            }
 
-                $lines[] = '';
+            $lines[] = '';
 
-                foreach ($children as $child) {
-                    $entry = Entry::find($child['entry']);
-                    if (! $entry) {
-                        continue;
-                    }
+            foreach ($this->flatten($children) as $entry) {
+                $lines[] = $this->entryLine($entry);
+            }
 
-                    $url = $entry->url();
-                    if (! $url) {
-                        continue;
-                    }
+            $lines[] = '';
+        }
 
-                    $title = $entry->value('title');
-                    $isExternal = str_starts_with($url, 'http');
-                    $href = $isExternal ? $url : url($url).'.md';
-                    $line = '- ['.$title.']('.$href.')';
+        return $lines;
+    }
 
-                    if ($intro = $entry->value('intro')) {
-                        $line .= ': '.str_replace("\n", ' ', $intro);
-                    }
+    /**
+     * Walk a tree branch to any depth, returning entries in reading order.
+     */
+    private function flatten(array $branch): array
+    {
+        $entries = [];
 
-                    $lines[] = $line;
-                }
+        foreach ($branch as $node) {
+            $entry = Entry::find($node['entry'] ?? null);
 
-                $lines[] = '';
+            if ($entry && $entry->published()) {
+                $entries[] = $entry;
             }
 
-            return $lines;
-        });
+            foreach ($this->flatten($node['children'] ?? []) as $descendant) {
+                $entries[] = $descendant;
+            }
+        }
 
-        return response(implode("\n", $lines), 200, [
-            'Content-Type' => 'text/plain; charset=UTF-8',
-        ]);
+        return $entries;
+    }
+
+    private function referenceSection(string $handle): array
+    {
+        if (! $collection = Collection::find($handle)) {
+            return [];
+        }
+
+        // "Reference" disambiguates these from the guide sections above, several of which
+        // share a name (the "Tags" guide explains tags; "Tags Reference" lists all 97).
+        $lines = ['## '.$collection->title().' Reference', ''];
+
+        $entries = $collection->queryEntries()
+            ->where('published', true)
+            ->orderBy('title', 'asc')
+            ->get();
+
+        foreach ($entries as $entry) {
+            $lines[] = $this->entryLine($entry);
+        }
+
+        $lines[] = '';
+
+        return $lines;
+    }
+
+    private function entryLine(EntryContract $entry): string
+    {
+        $url = $entry->url();
+
+        // Some tree nodes link off-site (ui.statamic.dev, YouTube). Those have no Markdown
+        // twin, so they're linked as-is.
+        $href = MarkdownUrl::for($url) ?? $url;
+
+        $line = '- ['.$entry->value('title').']('.$href.')';
+
+        if ($description = $entry->value('meta_description')) {
+            $line .= ': '.$this->oneLine($description);
+        }
+
+        return $line;
+    }
+
+    private function oneLine(string $text): string
+    {
+        return trim(preg_replace('/\s+/', ' ', $text) ?? $text);
     }
 }
diff --git a/app/Http/Controllers/RobotsTxtController.php b/app/Http/Controllers/RobotsTxtController.php
@@ -0,0 +1,69 @@
+<?php
+
+namespace App\Http\Controllers;
+
+class RobotsTxtController extends Controller
+{
+    /**
+     * AI crawlers we name explicitly. The policy is the same as the wildcard group,
+     * but stating it per-agent makes our stance unambiguous to both operators and
+     * agent-readiness scanners. Content-Signal is not inherited from `*`, so any
+     * agent listed here needs the directive repeated in its own group.
+     */
+    private array $aiCrawlers = [
+        'GPTBot',
+        'OAI-SearchBot',
+        'ChatGPT-User',
+        'ClaudeBot',
+        'Claude-User',
+        'Claude-SearchBot',
+        'PerplexityBot',
+        'Google-Extended',
+        'Applebot-Extended',
+        'meta-externalagent',
+        'Bytespider',
+        'CCBot',
+    ];
+
+    public function __invoke()
+    {
+        $version = config('docs.version');
+
+        $lines = [
+            "# Statamic {$version} Documentation",
+            '#',
+            '# Machine-readable index: /llms.txt',
+            '# Every page is also available as Markdown by appending .md to its URL,',
+            '# e.g. /tags/collection.md — far cheaper to read than the HTML.',
+            '',
+            'User-agent: *',
+            'Allow: /',
+            $this->contentSignals(),
+            '',
+        ];
+
+        foreach ($this->aiCrawlers as $crawler) {
+            $lines[] = "User-agent: {$crawler}";
+        }
+
+        $lines[] = 'Allow: /';
+        $lines[] = $this->contentSignals();
+        $lines[] = '';
+        $lines[] = 'Sitemap: '.url('/sitemap.xml');
+        $lines[] = '';
+
+        return response(implode("\n", $lines), 200, [
+            'Content-Type' => 'text/plain; charset=UTF-8',
+        ]);
+    }
+
+    /**
+     * Content Signals (https://contentsignals.org) declare how this content may be used.
+     * The docs are open source, and models knowing Statamic is good for Statamic, so we
+     * permit all three uses.
+     */
+    private function contentSignals(): string
+    {
+        return 'Content-Signal: search=yes, ai-input=yes, ai-train=yes';
+    }
+}
diff --git a/app/Modifiers/MarkdownUrl.php b/app/Modifiers/MarkdownUrl.php
@@ -0,0 +1,17 @@
+<?php
+
+namespace App\Modifiers;
+
+use App\Support\MarkdownUrl as MarkdownUrlBuilder;
+use Statamic\Modifiers\Modifier;
+
+class MarkdownUrl extends Modifier
+{
+    /**
+     * Turn a docs URL into the URL of its Markdown twin.
+     */
+    public function index($value, $params)
+    {
+        return MarkdownUrlBuilder::for($value ? (string) $value : null);
+    }
+}