[Repo Assist] Add llms.txt/llms-full.txt generation

AGENTS.md

@@ -46,3 +46,15 @@ dotnet restore FSharp.Formatting.sln
 dotnet build FSharp.Formatting.sln --configuration Release
 dotnet test FSharp.Formatting.sln --configuration Release --no-build
 ```
+
+## Testing Locally Against Another Project
+
+After building the repo with `dotnet build`, run the tool directly from the build output in your project's directory:
+
+```bash
+# macOS / Linux
+/path/to/FSharp.Formatting/src/fsdocs-tool/bin/Debug/net10.0/fsdocs build
+
+# Windows
+\path\to\FSharp.Formatting\src\fsdocs-tool\bin\Debug\net10.0\fsdocs.exe build
+```

RELEASE_NOTES.md

@@ -3,13 +3,13 @@
 ## [Unreleased]
 
 ### Added
-
 * Add `///` documentation comments to all public types, modules and members, and succinct internal comments, as part of ongoing effort to document the codebase. [#1035](https://github.com/fsprojects/FSharp.Formatting/issues/1035)
 * Add "Copy" button to all code blocks in generated documentation, making it easy to copy code samples to the clipboard. [#72](https://github.com/fsprojects/FSharp.Formatting/issues/72)
 * Add `<FsDocsAllowExecutableProject>true</FsDocsAllowExecutableProject>` project file setting to include executable projects (OutputType=Exe/WinExe) in API documentation generation. [#918](https://github.com/fsprojects/FSharp.Formatting/issues/918)
 * Add `{{fsdocs-logo-alt}}` substitution (configurable via `<FsDocsLogoAlt>` MSBuild property, defaults to `Logo`) for accessible alt text on the header logo image. [#626](https://github.com/fsprojects/FSharp.Formatting/issues/626)
 * Add `fsdocs init` command to scaffold a minimal `docs/index.md` (and optionally `_template.html`) for new projects. [#872](https://github.com/fsprojects/FSharp.Formatting/issues/872)
 * `IFsiEvaluator` now inherits `IDisposable`; `FsiEvaluator` disposes its underlying FSI session when disposed, preventing session leaks in long-running processes. [#341](https://github.com/fsprojects/FSharp.Formatting/issues/341)
+* Generate `llms.txt` and `llms-full.txt` for LLM consumption by default (opt out via `<FsDocsGenerateLlmsTxt>false</FsDocsGenerateLlmsTxt>`); when enabled, markdown output is always generated alongside HTML (even without a user-provided `_template.md`) and `llms.txt` links point to the `.md` files. [#951](https://github.com/fsprojects/FSharp.Formatting/issues/951) [#980](https://github.com/fsprojects/FSharp.Formatting/pull/980)
 
 ### Fixed
 * Fix project restore detection for projects with nonstandard artifact locations (e.g. `<UseArtifactsOutput>` or the dotnet/fsharp repo layout): when the MSBuild call to locate `project.assets.json` fails, emit a warning and proceed instead of hard-failing. [#592](https://github.com/fsprojects/FSharp.Formatting/issues/592)

docs/styling.md

@@ -76,6 +76,25 @@ For example:
 </PropertyGroup>
 ```
 
+## LLM-Friendly Output
+
+By default, `fsdocs build` generates `llms.txt` and `llms-full.txt` in the output root,
+following the [llmstxt.org](https://llmstxt.org/) convention. These files provide a structured
+index (and full content) of all documentation pages and API reference entries, making it easy
+to add documentation context to LLMs and AI coding assistants.
+
+When this feature is enabled, markdown (`.md`) files are automatically generated alongside HTML
+for all documentation pages — even if you have not added a `_template.md` file. The `llms.txt`
+links point to these markdown files, which are more suitable for LLM consumption.
+
+To opt out, set the following property in your project file or `Directory.Build.props`:
+
+```xml
+<PropertyGroup>
+    <FsDocsGenerateLlmsTxt>false</FsDocsGenerateLlmsTxt>
+</PropertyGroup>
+```
+
 As an example, here is [a page with alternative styling](templates/leftside/styling.html).
 
 ## Customizing via CSS

src/fsdocs-tool/BuildCommand.fs

@@ -647,7 +647,7 @@ type internal DocContent
                           (Path.Combine(outputFolderRelativeToRoot, subInputFolderName))
                           filesWithFrontMatter ]
 
-    member _.Convert(rootInputFolderAsGiven, htmlTemplate, extraInputs) =
+    member _.Convert(rootInputFolderAsGiven, htmlTemplate, extraInputs, ?defaultMdTemplate: string) =
 
         let inputDirectories = extraInputs @ [ (rootInputFolderAsGiven, ".") ]
 
@@ -681,7 +681,14 @@ type internal DocContent
         [ for (rootInputFolderAsGiven, outputFolderRelativeToRoot) in inputDirectories do
               yield!
                   processFolder
-                      (htmlTemplate, None, None, None, None, false, Some rootInputFolderAsGiven, fullPathFileMap)
+                      (htmlTemplate,
+                       None,
+                       None,
+                       None,
+                       defaultMdTemplate,
+                       false,
+                       Some rootInputFolderAsGiven,
+                       fullPathFileMap)
                       rootInputFolderAsGiven
                       outputFolderRelativeToRoot
                       filesWithFrontMatter ]
@@ -1301,6 +1308,127 @@ module Serve =
 
         startWebServerAsync serverConfig app |> snd |> Async.Start
 
+/// Helpers for generating llms.txt and llms-full.txt content.
+module internal LlmsTxt =
+
+    /// Decode HTML entities (e.g. " → ", > → >) in a string.
+    let private decodeHtml (s: string) = System.Net.WebUtility.HtmlDecode(s)
+
+    /// Strip FSharp.Formatting --eval warning lines from content.
+    let private stripEvalWarnings (s: string) =
+        s.Split('\n')
+        |> Array.filter (fun line ->
+            not (
+                line
+                    .TrimStart()
+                    .StartsWith(
+                        "Warning: Output, it-value and value references require --eval",
+                        System.StringComparison.Ordinal
+                    )
+            ))
+        |> String.concat "\n"
+
+    /// Collapse three or more consecutive newlines into at most two.
+    let private collapseBlankLines (s: string) =
+        System.Text.RegularExpressions.Regex.Replace(s, @"\n{3,}", "\n\n")
+
+    /// Normalise a title: trim and collapse internal whitespace/newlines to a single space.
+    let private normaliseTitle (s: string) =
+        System.Text.RegularExpressions.Regex.Replace(s.Trim(), @"\s+", " ")
+
+    /// Decode HTML entities and remove --eval noise from content.
+    let private cleanContent (s: string) =
+        s
+        |> decodeHtml
+        |> stripEvalWarnings
+        |> collapseBlankLines
+        |> fun t -> t.Trim()
+
+    /// Build a section of llms.txt from a set of search index entries.
+    /// When <c>withContent</c> is true, entry content is appended under a heading per entry.
+    /// When false, entries are listed as bullet-point links (index format).
+    /// <c>uriTransform</c> is applied to each entry's URI before rendering.
+    let buildSection
+        sectionTitle
+        (entries: ApiDocsSearchIndexEntry array)
+        withContent
+        (uriTransform: string -> string)
+        =
+        if entries.Length = 0 then
+            ""
+        else
+            let sb = System.Text.StringBuilder()
+            sb.Append(sprintf "## %s\n\n" sectionTitle) |> ignore
+
+            for e in entries do
+                let title = normaliseTitle e.title
+                let uri = uriTransform e.uri
+
+                if withContent then
+                    sb.Append(sprintf "### [%s](%s)\n\n" title uri) |> ignore
+
+                    if not (System.String.IsNullOrWhiteSpace(e.content)) then
+                        sb.Append(cleanContent e.content) |> ignore
+                        sb.Append("\n\n") |> ignore
+                else
+                    sb.Append(sprintf "- [%s](%s)\n" title uri) |> ignore
+
+            sb.ToString()
+
+    /// Returns a URI transformer that rewrites links to use .md when markdown output is available.
+    /// <c>docContentUsesMarkdown</c> – doc pages were generated with a _template.md.
+    /// <c>apiDocUsesMarkdown</c> – API reference was generated with GenerateMarkdownPhased
+    ///   (URIs have no file extension; .md must be appended).
+    let buildUriTransform (docContentUsesMarkdown: bool) (apiDocUsesMarkdown: bool) (entryType: string) =
+        fun (uri: string) ->
+            match entryType with
+            | "content" when docContentUsesMarkdown ->
+                if uri.EndsWith(".html", System.StringComparison.OrdinalIgnoreCase) then
+                    uri.[.. uri.Length - 6] + ".md"
+                else
+                    uri
+            | "apiDocs" when apiDocUsesMarkdown ->
+                // In markdown mode InUrl="" so URIs have no extension; append .md.
+                // Strip any #anchor before appending, then re-attach it.
+                let hashIdx = uri.IndexOf('#')
+
+                if hashIdx >= 0 then
+                    uri.[.. hashIdx - 1] + ".md" + uri.[hashIdx..]
+                else
+                    uri + ".md"
+            | _ -> uri
+
+    /// Generate the text content of llms.txt (index) and llms-full.txt (with content).
+    /// Returns a tuple of (llms.txt content, llms-full.txt content).
+    /// When <c>docContentUsesMarkdown</c> is true, doc page links use .md extensions.
+    /// When <c>apiDocUsesMarkdown</c> is true, API reference links use .md extensions.
+    let buildContent
+        (collectionName: string)
+        (entries: ApiDocsSearchIndexEntry array)
+        (docContentUsesMarkdown: bool)
+        (apiDocUsesMarkdown: bool)
+        =
+        let contentEntries = entries |> Array.filter (fun e -> e.``type`` = "content")
+        let apiEntries = entries |> Array.filter (fun e -> e.``type`` = "apiDocs")
+        // For the index, exclude per-member entries (identified by a '#' anchor in the URI).
+        let apiIndexEntries = apiEntries |> Array.filter (fun e -> not (e.uri.Contains("#")))
+        let header = sprintf "# %s\n\n" collectionName
+
+        let contentTransform = buildUriTransform docContentUsesMarkdown apiDocUsesMarkdown "content"
+        let apiDocTransform = buildUriTransform docContentUsesMarkdown apiDocUsesMarkdown "apiDocs"
+
+        let llmsTxt =
+            header
+            + buildSection "Docs" contentEntries false contentTransform
+            + buildSection "API Reference" apiIndexEntries false apiDocTransform
+
+        let llmsFullTxt =
+            header
+            + buildSection "Docs" contentEntries true contentTransform
+            + buildSection "API Reference" apiEntries true apiDocTransform
+
+        llmsTxt, llmsFullTxt
+
 type CoreBuildOptions(watch) =
 
     [<Option("input", Required = false, Default = "docs", HelpText = "Input directory of documentation content.")>]
@@ -1458,7 +1586,7 @@ type CoreBuildOptions(watch) =
         // See https://github.com/ionide/proj-info/issues/123
         let prevDotnetHostPath = Environment.GetEnvironmentVariable("DOTNET_HOST_PATH")
 
-        let (root, collectionName, crackedProjects, paths, docsSubstitutions), _key =
+        let (root, collectionName, crackedProjects, paths, docsSubstitutions, generateLlmsTxt), _key =
             let projects = Seq.toList this.projects
             let cacheFile = ".fsdocs/cache"
 
@@ -1634,6 +1762,35 @@ type CoreBuildOptions(watch) =
             else
                 None
 
+        // Default markdown template – used when generateLlmsTxt is enabled and no user _template.md exists.
+        // An empty (or minimal) _template.md causes the processor to emit just the document content, which
+        // is ideal for LLM consumption.
+        let defaultMdTemplateAttempt1 =
+            Path.GetFullPath(Path.Combine(dir, "..", "..", "..", "templates", "_template.md"))
+
+        let defaultMdTemplateAttempt2 =
+            Path.GetFullPath(Path.Combine(dir, "..", "..", "..", "..", "..", "docs", "_template.md"))
+
+        let defaultMdTemplate =
+            if this.nodefaultcontent then
+                None
+            else if
+                (try
+                    File.Exists(defaultMdTemplateAttempt1)
+                 with _ ->
+                     false)
+            then
+                Some defaultMdTemplateAttempt1
+            elif
+                (try
+                    File.Exists(defaultMdTemplateAttempt2)
+                 with _ ->
+                     false)
+            then
+                Some defaultMdTemplateAttempt2
+            else
+                None
+
         let extraInputs =
             [ if not this.nodefaultcontent then
                   // The "extras" content goes in "."
@@ -1673,6 +1830,7 @@ type CoreBuildOptions(watch) =
         let mutable latestApiDocCodeReferenceResolver = (fun _ -> None)
         let mutable latestApiDocPhase2 = (fun _ -> ())
         let mutable latestApiDocSearchIndexEntries = [||]
+        let mutable latestApiDocOutputKind = OutputKind.Html
         let mutable latestDocContentPhase2 = (fun _ -> ())
         let mutable latestDocContentResults = Map.empty
         let mutable latestDocContentSearchIndexEntries = [||]
@@ -1689,6 +1847,24 @@ type CoreBuildOptions(watch) =
 
             File.WriteAllText(Path.Combine(rootOutputFolderAsGiven, "index.json"), indxTxt)
 
+        // Capture the bool value before it is shadowed by the generateLlmsTxt function below.
+        let generateLlmsTxtEnabled = generateLlmsTxt
+
+        let generateLlmsTxt () =
+            if generateLlmsTxtEnabled then
+                let index = Array.append latestApiDocSearchIndexEntries latestDocContentSearchIndexEntries
+                // When FsDocsGenerateLlmsTxt is enabled, markdown is always generated alongside HTML
+                // (using the bundled default markdown template if the user hasn't provided a _template.md).
+                let docContentUsesMarkdown = true
+
+                let apiDocUsesMarkdown = latestApiDocOutputKind = OutputKind.Markdown
+
+                let llmsTxt, llmsFullTxt =
+                    LlmsTxt.buildContent collectionName index docContentUsesMarkdown apiDocUsesMarkdown
+
+                File.WriteAllText(Path.Combine(rootOutputFolderAsGiven, "llms.txt"), llmsTxt)
+                File.WriteAllText(Path.Combine(rootOutputFolderAsGiven, "llms-full.txt"), llmsFullTxt)
+
         /// get the hot reload script if running in watch mode
         let getLatestWatchScript () =
             if watch then
@@ -1732,6 +1908,8 @@ type CoreBuildOptions(watch) =
 
                                 OutputKind.Html, None
 
+                    latestApiDocOutputKind <- outputKind
+
                     printfn ""
                     printfn "API docs:"
                     printfn "  generating model for %d assemblies in API docs..." apiDocInputs.Length
@@ -1823,7 +2001,25 @@ type CoreBuildOptions(watch) =
                         onError
                     )
 
-                let docModels = docContent.Convert(this.input, defaultTemplate, extraInputs)
+                let docModels =
+                    // When llms.txt generation is enabled, ensure a markdown template is available so that
+                    // .md versions of content pages are written alongside .html files, enabling
+                    // llms.txt to link to the more LLM-friendly markdown versions.
+                    // An empty template causes the processor to emit just the document content,
+                    // which is ideal for LLM consumption.
+                    let mdTemplate =
+                        if generateLlmsTxtEnabled then
+                            match defaultMdTemplate with
+                            | Some _ -> defaultMdTemplate
+                            | None ->
+                                let tempMdTemplate = Path.GetTempFileName()
+                                File.WriteAllText(tempMdTemplate, "")
+                                Some tempMdTemplate
+                        else
+                            None
+
+                    docContent.Convert(this.input, defaultTemplate, extraInputs, ?defaultMdTemplate = mdTemplate)
+
                 let actualDocModels = docModels |> List.map fst |> List.choose id
                 let extrasForSearchIndex = docContent.GetSearchIndexEntries(actualDocModels)
 
@@ -1963,6 +2159,7 @@ type CoreBuildOptions(watch) =
             // bespoke file for namespaces etc.
             let ok1 = ok1 && runDocContentPhase2 ()
             regenerateSearchIndex ()
+            generateLlmsTxt ()
             ok1 && ok2
 
         //-----------------------------------------
@@ -2024,7 +2221,8 @@ type CoreBuildOptions(watch) =
 
                             if runDocContentPhase1 () then
                                 if runDocContentPhase2 () then
-                                    regenerateSearchIndex ())
+                                    regenerateSearchIndex ()
+                                    generateLlmsTxt ())
 
                         Serve.refreshEvent.Trigger fileName
                     }
@@ -2045,7 +2243,8 @@ type CoreBuildOptions(watch) =
 
                             if runGeneratePhase1 () then
                                 if runGeneratePhase2 () then
-                                    regenerateSearchIndex ())
+                                    regenerateSearchIndex ()
+                                    generateLlmsTxt ())
 
                         Serve.refreshEvent.Trigger "full"
                     }

src/fsdocs-tool/ProjectCracker.fs

@@ -229,6 +229,7 @@ module Crack =
           FsDocsFaviconSource: string option
           FsDocsTheme: string option
           FsDocsWarnOnMissingDocs: bool
+          FsDocsGenerateLlmsTxt: bool
           FsDocsAllowExecutableProject: bool
           PackageProjectUrl: string option
           Authors: string option
@@ -262,6 +263,7 @@ module Crack =
               "FsDocsSourceFolder"
               "FsDocsSourceRepository"
               "FsDocsWarnOnMissingDocs"
+              "FsDocsGenerateLlmsTxt"
               "FsDocsAllowExecutableProject"
               "RepositoryType"
               "RepositoryBranch"
@@ -348,6 +350,7 @@ module Crack =
                   FsDocsFaviconSource = msbuildPropString "FsDocsFaviconSource"
                   FsDocsTheme = msbuildPropString "FsDocsTheme"
                   FsDocsWarnOnMissingDocs = msbuildPropBool "FsDocsWarnOnMissingDocs" |> Option.defaultValue false
+                  FsDocsGenerateLlmsTxt = msbuildPropBool "FsDocsGenerateLlmsTxt" |> Option.defaultValue true
                   FsDocsAllowExecutableProject =
                     msbuildPropBool "FsDocsAllowExecutableProject" |> Option.defaultValue false
                   UsesMarkdownComments = msbuildPropBool "UsesMarkdownComments" |> Option.defaultValue false
@@ -608,6 +611,7 @@ module Crack =
                 |> fallbackFromDirectoryProps "//RepositoryUrl"
               FsDocsTheme = projectInfos |> List.tryPick (fun info -> info.FsDocsTheme)
               FsDocsWarnOnMissingDocs = false
+              FsDocsGenerateLlmsTxt = projectInfos |> List.forall (fun i -> i.FsDocsGenerateLlmsTxt)
               FsDocsAllowExecutableProject = false
               PackageProjectUrl =
                 projectInfos
@@ -711,4 +715,4 @@ module Crack =
             |> List.choose (fun projectInfo -> projectInfo.TargetPath |> Option.map Path.GetDirectoryName)
 
         let docsParameters = parametersForProjectInfo projectInfoForDocs
-        root, collectionName, crackedProjects, paths, docsParameters
+        root, collectionName, crackedProjects, paths, docsParameters, projectInfoForDocs.FsDocsGenerateLlmsTxt

src/fsdocs-tool/fsdocs-tool.fsproj

@@ -30,6 +30,7 @@
   <ItemGroup>
     <!-- NuGet package content -->
     <Content Include="..\..\docs\_template.html" PackagePath="templates" />
+    <Content Include="..\..\docs\_template.md" PackagePath="templates" />
     <Content Include="..\..\docs\_template.tex" PackagePath="templates" />
     <Content Include="..\..\docs\_template.ipynb" PackagePath="templates" />
     <Content Include="..\..\docs\Dockerfile" PackagePath="extras" />