From 3cd91f18f812429bef9c2f59a2cad405c56f101b Mon Sep 17 00:00:00 2001 From: Matthew Podwysocki Date: Tue, 14 Apr 2026 13:33:25 -0400 Subject: [PATCH] feat: use sublevel llms.txt per product for MCP resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs.mapbox.com restructured so that llms.txt now exists at every product level (e.g. /api/llms.txt, /help/llms.txt, /mapbox-gl-js/llms.txt) and llms-full.txt contains full page content. The root llms.txt is now a pure link index, so resources that filtered it by category keyword were returning empty content. - resource://mapbox-api-reference → docs.mapbox.com/api/llms.txt (structured index of all REST APIs by service category) - resource://mapbox-guides → docs.mapbox.com/help/llms.txt (39KB Help Center index with actual guide content) - resource://mapbox-sdk-docs → docs.mapbox.com/mapbox-gl-js/llms.txt (34KB GL JS documentation index with guides, API ref, examples) - resource://mapbox-reference → root llms.txt without filtering (complete product catalog for discovering available docs) - resource://mapbox-examples → continues filtering root for playground/demo sections Add fetchCachedText() shared helper to docFetcher to consolidate the repeated fetch+cache pattern across all resource implementations. Fix toMarkdownUrl() to return null for URLs already ending in .txt/.md/.json so get_document_tool fetches llms.txt files directly without a wasted .md rewrite attempt. Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 16 ++++++++++++++++ .../MapboxApiReferenceResource.ts | 2 +- .../MapboxExamplesResource.ts | 2 +- src/utils/docFetcher.ts | 4 +--- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe1fd98..179b8b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ ## Unreleased +### Resources — use sublevel `llms.txt` per product + +docs.mapbox.com restructured its documentation so that `llms.txt` files now exist at every product level (e.g. `docs.mapbox.com/api/llms.txt`, `docs.mapbox.com/help/llms.txt`, `docs.mapbox.com/mapbox-gl-js/llms.txt`) alongside `llms-full.txt` files containing full page content. The root `docs.mapbox.com/llms.txt` is now a pure index of links to these sublevel files rather than a monolithic content file. The previous resources all filtered the root file by category keyword — now that the root contains only link lists, they were effectively returning empty or useless content. + +Updated resources to use the appropriate sublevel `llms.txt` files: + +- **`resource://mapbox-api-reference`** now fetches `docs.mapbox.com/api/llms.txt` — a clean, structured index of every Mapbox REST API grouped by service (Maps, Navigation, Search, Accounts) with links to full API reference pages +- **`resource://mapbox-guides`** now fetches `docs.mapbox.com/help/llms.txt` (39KB) — the full Mapbox Help Center index with troubleshooting guides, how-to tutorials, and walkthroughs +- **`resource://mapbox-sdk-docs`** now fetches `docs.mapbox.com/mapbox-gl-js/llms.txt` (34KB) — the GL JS documentation index listing all guides, API reference pages, and examples for the primary web mapping SDK +- **`resource://mapbox-reference`** now fetches the root `llms.txt` without filtering and returns the complete product catalog — useful for discovering what documentation exists and finding `llms.txt` URLs for any product +- **`resource://mapbox-examples`** continues to extract playground/demo/example sections from the root index (API Playgrounds, Demos & Projects) + +**`docFetcher.fetchCachedText`** — new shared helper that fetches a URL and stores it in `docCache`, used by all five resources to avoid duplicating the fetch+cache pattern. + +**`docFetcher.toMarkdownUrl`** — no longer rewrites URLs already ending in `.txt`, `.md`, or `.json`. Previously `get_document_tool` would try to fetch `llms.txt.md` before falling back; now it fetches `llms.txt` directly on the first attempt. + ### Replace Algolia search with self-contained llms.txt search `search_mapbox_docs_tool` no longer depends on the Algolia third-party service. The hosted server shares a single Algolia free-tier quota across all users, making it prone to throttling as usage grows. The new implementation searches directly against the `llms.txt` index files that now exist at every product level on docs.mapbox.com. diff --git a/src/resources/mapbox-api-reference-resource/MapboxApiReferenceResource.ts b/src/resources/mapbox-api-reference-resource/MapboxApiReferenceResource.ts index 40203d3..34e7fed 100644 --- a/src/resources/mapbox-api-reference-resource/MapboxApiReferenceResource.ts +++ b/src/resources/mapbox-api-reference-resource/MapboxApiReferenceResource.ts @@ -25,7 +25,7 @@ export class MapboxApiReferenceResource extends BaseResource { 'Mapbox REST API reference index organized by service (Maps, Navigation, Search, Accounts). ' + 'Lists all API endpoints with links to detailed reference pages covering parameters, ' + 'rate limits, authentication, and response formats (Geocoding, Directions, Static Images, ' + - 'Tilequery, Matrix, isochrone, Optimization, Styles, Uploads, Datasets, and more).'; + 'Tilequery, Matrix, Isochrone, Optimization, Styles, Uploads, Datasets, and more).'; readonly mimeType = 'text/markdown'; private httpRequest: HttpRequest; diff --git a/src/resources/mapbox-examples-resource/MapboxExamplesResource.ts b/src/resources/mapbox-examples-resource/MapboxExamplesResource.ts index fb6f6fe..9eb5278 100644 --- a/src/resources/mapbox-examples-resource/MapboxExamplesResource.ts +++ b/src/resources/mapbox-examples-resource/MapboxExamplesResource.ts @@ -27,7 +27,7 @@ export class MapboxExamplesResource extends BaseResource { readonly description = 'Mapbox interactive API playgrounds, demo applications, and code examples. ' + 'Includes playground URLs for Directions, Search Box, Static Images, ' + - 'isochrone, Matrix APIs, and demo apps for real estate, store locator, etc.'; + 'Isochrone, Matrix APIs, and demo apps for real estate, store locator, etc.'; readonly mimeType = 'text/markdown'; private httpRequest: HttpRequest; diff --git a/src/utils/docFetcher.ts b/src/utils/docFetcher.ts index d6f7cab..60eb54c 100644 --- a/src/utils/docFetcher.ts +++ b/src/utils/docFetcher.ts @@ -73,9 +73,7 @@ export async function fetchCachedText( const cached = docCache.get(url); if (cached !== null) return cached; - const response = await httpRequest(url, { - headers: { Accept: 'text/markdown, text/plain;q=0.9, */*;q=0.8' } - }); + const response = await httpRequest(url, {}); if (!response.ok) { throw new Error(`Failed to fetch ${url}: ${response.statusText}`);