Add AI agent detection via @vercel/agent-readability

apps/docs/lib/geistdocs/md-tracking.ts

@@ -1,13 +1,16 @@
+import type { DetectionMethod } from "@vercel/agent-readability";
 import { siteId } from "@/geistdocs";
 
 const PLATFORM_URL = "https://geistdocs.com/md-tracking";
 
 interface TrackMdRequestParams {
   acceptHeader: string | null;
+  /** Detection method used to identify the agent (only for agent-rewrite requests) */
+  detectionMethod?: DetectionMethod | null;
   path: string;
   referer: string | null;
-  /** How the markdown was requested: 'md-url' for direct .md URLs, 'header-negotiated' for Accept header */
-  requestType?: "md-url" | "header-negotiated";
+  /** How the markdown was requested: 'md-url' for direct .md URLs, 'header-negotiated' for Accept header, 'agent-rewrite' for detected AI agents */
+  requestType?: "md-url" | "header-negotiated" | "agent-rewrite";
   userAgent: string | null;
 }
 
@@ -21,6 +24,7 @@ export async function trackMdRequest({
   referer,
   acceptHeader,
   requestType,
+  detectionMethod,
 }: TrackMdRequestParams): Promise<void> {
   try {
     const response = await fetch(PLATFORM_URL, {
@@ -35,6 +39,7 @@ export async function trackMdRequest({
         referer,
         acceptHeader,
         requestType,
+        detectionMethod,
       }),
     });
 

apps/docs/package.json

@@ -19,6 +19,7 @@
     "@streamdown/code": "^1.0.1",
     "@streamdown/math": "^1.0.1",
     "@streamdown/mermaid": "^1.0.1",
+    "@vercel/agent-readability": "^0.2.1",
     "@vercel/analytics": "^1.6.1",
     "@vercel/speed-insights": "^1.3.1",
     "ai": "^6.0.105",
@@ -63,4 +64,4 @@
     "tw-animate-css": "^1.4.0",
     "typescript": "^5.9.3"
   }
-}
+}

apps/docs/proxy.ts

@@ -7,6 +7,7 @@ import {
 } from "next/server";
 import { i18n } from "@/lib/geistdocs/i18n";
 import { trackMdRequest } from "@/lib/geistdocs/md-tracking";
+import { generateNotFoundMarkdown, isAIAgent } from "@vercel/agent-readability";
 
 const { rewrite: rewriteLLM } = rewritePath(
   "/*path",
@@ -57,6 +58,47 @@ const proxy = (request: NextRequest, context: NextFetchEvent) => {
     }
   }
 
+  // AI agent detection — rewrite docs pages to markdown for agents
+  if (
+    (pathname === "/docs" || pathname.startsWith("/docs/")) &&
+    !pathname.includes("/llms.mdx/")
+  ) {
+    const agentResult = isAIAgent(request);
+    if (agentResult.detected && !isMarkdownPreferred(request)) {
+      const result = rewriteLLM(pathname);
+
+      if (result) {
+        context.waitUntil(
+          trackMdRequest({
+            path: pathname,
+            userAgent: request.headers.get("user-agent"),
+            referer: request.headers.get("referer"),
+            acceptHeader: request.headers.get("accept"),
+            requestType: "agent-rewrite",
+            detectionMethod: agentResult.method,
+          })
+        );
+        const response = NextResponse.rewrite(new URL(result, request.nextUrl));
+        response.headers.set("Vary", "Accept");
+        return response;
+      }
+      // Agent requested a non-existent docs URL — return helpful markdown
+      return new NextResponse(
+        generateNotFoundMarkdown(pathname, {
+          sitemapUrl: "/sitemap.md",
+          indexUrl: "/llms.txt",
+        }),
+        {
+          headers: {
+            "Content-Type": "text/markdown; charset=utf-8",
+            Vary: "Accept",
+          },
+        },
+      );
+    }
+  }
+
+
   // Handle Accept header content negotiation and track the request
   if (isMarkdownPreferred(request)) {
     const result = rewriteLLM(pathname);