diff --git a/server/middleware/canonical-redirects.global.ts b/server/middleware/canonical-redirects.global.ts index 230ab3443..86e47fd96 100644 --- a/server/middleware/canonical-redirects.global.ts +++ b/server/middleware/canonical-redirects.global.ts @@ -46,26 +46,42 @@ export default defineEventHandler(async event => { return } + // /llms.txt at root is handled by the llms-txt middleware + if (path === '/llms.txt') { + return + } + // /@org/pkg or /pkg → /package/org/pkg or /package/pkg - let pkgMatch = path.match(/^\/(?:(?@[^/]+)\/)?(?[^/@]+)$/) + // Also handles trailing /llms.txt or /llms_full.txt suffixes + let pkgMatch = path.match( + /^\/(?:(?@[^/]+)\/)?(?[^/@]+)(?\/(?:llms\.txt|llms_full\.txt))?$/, + ) if (pkgMatch?.groups) { const args = [pkgMatch.groups.org, pkgMatch.groups.name].filter(Boolean).join('/') + const suffix = pkgMatch.groups.suffix ?? '' setHeader(event, 'cache-control', cacheControl) - return sendRedirect(event, `/package/${args}` + (query ? '?' + query : ''), 301) + return sendRedirect(event, `/package/${args}${suffix}` + (query ? '?' + query : ''), 301) } // /@org/pkg/v/version or /@org/pkg@version → /package/org/pkg/v/version // /pkg/v/version or /pkg@version → /package/pkg/v/version + // Also handles trailing /llms.txt or /llms_full.txt suffixes const pkgVersionMatch = - path.match(/^\/(?:(?@[^/]+)\/)?(?[^/@]+)\/v\/(?[^/]+)$/) || - path.match(/^\/(?:(?@[^/]+)\/)?(?[^/@]+)@(?[^/]+)$/) + path.match( + /^\/(?:(?@[^/]+)\/)?(?[^/@]+)\/v\/(?[^/]+)(?\/(?:llms\.txt|llms_full\.txt))?$/, + ) || + path.match( + /^\/(?:(?@[^/]+)\/)?(?[^/@]+)@(?[^/]+)(?\/(?:llms\.txt|llms_full\.txt))?$/, + ) if (pkgVersionMatch?.groups) { const args = [pkgVersionMatch.groups.org, pkgVersionMatch.groups.name].filter(Boolean).join('/') + const versionSuffix = pkgVersionMatch.groups.suffix ?? '' setHeader(event, 'cache-control', cacheControl) return sendRedirect( event, - `/package/${args}/v/${pkgVersionMatch.groups.version}` + (query ? '?' + query : ''), + `/package/${args}/v/${pkgVersionMatch.groups.version}${versionSuffix}` + + (query ? '?' + query : ''), 301, ) } diff --git a/server/middleware/llms-txt.ts b/server/middleware/llms-txt.ts new file mode 100644 index 000000000..0ca86eb92 --- /dev/null +++ b/server/middleware/llms-txt.ts @@ -0,0 +1,105 @@ +import * as v from 'valibot' +import { PackageRouteParamsSchema } from '#shared/schemas/package' +import { handleApiError } from '#server/utils/error-handler' +import { handleLlmsTxt, handleOrgLlmsTxt, generateRootLlmsTxt } from '#server/utils/llms-txt' + +const CACHE_HEADER = 's-maxage=3600, stale-while-revalidate=86400' + +/** + * Middleware to handle ALL llms.txt / llms_full.txt routes. + * + * All llms.txt handling lives here rather than in file-based routes because + * Vercel's ISR route rules with glob patterns (e.g. `/package/ ** /llms.txt`) + * create catch-all serverless functions that interfere with Nitro's file-based + * route resolution — scoped packages and versioned paths fail to match. + * + * Handles: + * - /llms.txt (root discovery page) + * - /package/@:org/llms.txt (org package listing) + * - /package/:name/llms.txt (unscoped, latest) + * - /package/:name/llms_full.txt (unscoped, latest, full) + * - /package/@:org/:name/llms.txt (scoped, latest) + * - /package/@:org/:name/llms_full.txt (scoped, latest, full) + * - /package/:name/v/:version/llms.txt (unscoped, versioned) + * - /package/:name/v/:version/llms_full.txt (unscoped, versioned, full) + * - /package/@:org/:name/v/:version/llms.txt (scoped, versioned) + * - /package/@:org/:name/v/:version/llms_full.txt (scoped, versioned, full) + */ +export default defineEventHandler(async event => { + const path = event.path.split('?')[0] ?? '/' + + if (!path.endsWith('/llms.txt') && !path.endsWith('/llms_full.txt')) return + + const full = path.endsWith('/llms_full.txt') + const suffix = full ? '/llms_full.txt' : '/llms.txt' + + // Root /llms.txt + if (path === '/llms.txt') { + const url = getRequestURL(event) + const baseUrl = `${url.protocol}//${url.host}` + setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') + setHeader(event, 'Cache-Control', CACHE_HEADER) + return generateRootLlmsTxt(baseUrl) + } + + if (!path.startsWith('/package/')) return + + // Strip /package/ prefix and /llms[_full].txt suffix + const inner = path.slice('/package/'.length, -suffix.length) + + // Org-level: /package/@org/llms.txt (inner = "@org") + if (!full && inner.startsWith('@') && !inner.includes('/')) { + const orgName = inner.slice(1) + try { + const url = getRequestURL(event) + const baseUrl = `${url.protocol}//${url.host}` + const content = await handleOrgLlmsTxt(orgName, baseUrl) + setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') + setHeader(event, 'Cache-Control', CACHE_HEADER) + return content + } catch (error: unknown) { + handleApiError(error, { statusCode: 502, message: 'Failed to generate org llms.txt.' }) + } + } + + // Parse package name and optional version from inner path + let rawPackageName: string + let rawVersion: string | undefined + + if (inner.includes('/v/')) { + // Versioned path + if (inner.startsWith('@')) { + const match = inner.match(/^(@[^/]+\/[^/]+)\/v\/(.+)$/) + if (!match?.[1] || !match[2]) return + rawPackageName = match[1] + rawVersion = match[2] + } else { + const match = inner.match(/^([^/]+)\/v\/(.+)$/) + if (!match?.[1] || !match[2]) return + rawPackageName = match[1] + rawVersion = match[2] + } + } else { + // Latest version — inner is just the package name + rawPackageName = inner + } + + if (!rawPackageName) return + + try { + const { packageName, version } = v.parse(PackageRouteParamsSchema, { + packageName: rawPackageName, + version: rawVersion, + }) + + const content = await handleLlmsTxt(packageName, version, { includeAgentFiles: full }) + setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') + setHeader(event, 'Cache-Control', CACHE_HEADER) + return content + } catch (error: unknown) { + handleApiError(error, { + statusCode: 502, + message: `Failed to generate ${full ? 'llms_full.txt' : 'llms.txt'}.`, + }) + } +}) diff --git a/server/utils/llms-txt.ts b/server/utils/llms-txt.ts new file mode 100644 index 000000000..861ea6888 --- /dev/null +++ b/server/utils/llms-txt.ts @@ -0,0 +1,366 @@ +import type { JsDelivrFileNode, AgentFile, LlmsTxtResult } from '#shared/types' +import { NPM_MISSING_README_SENTINEL, NPM_REGISTRY } from '#shared/utils/constants' + +/** Well-known agent instruction files at the package root */ +const ROOT_AGENT_FILES: Record = { + 'CLAUDE.md': 'Claude Code', + 'AGENTS.md': 'Agent Instructions', + 'AGENT.md': 'Agent Instructions', + '.cursorrules': 'Cursor Rules', + '.windsurfrules': 'Windsurf Rules', + '.clinerules': 'Cline Rules', +} + +/** Well-known agent files inside specific directories */ +const DIRECTORY_AGENT_FILES: Record = { + '.github/copilot-instructions.md': 'GitHub Copilot', +} + +/** Directories containing rule files (match *.md inside) */ +const RULE_DIRECTORIES: Record = { + '.cursor/rules': 'Cursor Rules', + '.windsurf/rules': 'Windsurf Rules', +} + +/** + * Discover agent instruction file paths from a jsDelivr file tree. + * Scans root-level files, known subdirectory files, and rule directories. + */ +export function discoverAgentFiles(files: JsDelivrFileNode[]): string[] { + const discovered: string[] = [] + + for (const node of files) { + // Root-level well-known files + if (node.type === 'file' && node.name in ROOT_AGENT_FILES) { + discovered.push(node.name) + } + + // Directory-based files + if (node.type === 'directory') { + // .github/copilot-instructions.md + if (node.name === '.github' && node.files) { + for (const child of node.files) { + const fullPath = `.github/${child.name}` + if (child.type === 'file' && fullPath in DIRECTORY_AGENT_FILES) { + discovered.push(fullPath) + } + } + } + + // .cursor/rules/*.md and .windsurf/rules/*.md + for (const dirPath of Object.keys(RULE_DIRECTORIES)) { + const [topDir, subDir] = dirPath.split('/') + if (node.name === topDir && node.files) { + const rulesDir = node.files.find(f => f.type === 'directory' && f.name === subDir) + if (rulesDir?.files) { + for (const ruleFile of rulesDir.files) { + if (ruleFile.type === 'file' && ruleFile.name.endsWith('.md')) { + discovered.push(`${dirPath}/${ruleFile.name}`) + } + } + } + } + } + } + } + + return discovered +} + +/** + * Get the display name for an agent file path. + */ +function getDisplayName(filePath: string): string { + if (filePath in ROOT_AGENT_FILES) return ROOT_AGENT_FILES[filePath]! + if (filePath in DIRECTORY_AGENT_FILES) return DIRECTORY_AGENT_FILES[filePath]! + + for (const [dirPath, displayName] of Object.entries(RULE_DIRECTORIES)) { + if (filePath.startsWith(`${dirPath}/`)) + return `${displayName}: ${filePath.split('/').pop() ?? filePath}` + } + + return filePath +} + +/** + * Fetch agent instruction files from jsDelivr CDN. + * Fetches in parallel, gracefully skipping failures. + */ +export async function fetchAgentFiles( + packageName: string, + version: string, + filePaths: string[], +): Promise { + const results = await Promise.all( + filePaths.map(async (path): Promise => { + try { + const url = `https://cdn.jsdelivr.net/npm/${packageName}@${version}/${path}` + const response = await fetch(url) + if (!response.ok) return null + const content = await response.text() + return { path, content, displayName: getDisplayName(path) } + } catch { + return null + } + }), + ) + + return results.filter((r): r is AgentFile => r !== null) +} + +/** + * Generate llms.txt markdown content per the llmstxt.org spec. + * + * Structure: + * - H1 title with package name and version + * - Blockquote description (if available) + * - Metadata list (homepage, repository, npm) + * - README section + * - Agent Instructions section (one sub-heading per file, full mode only) + */ +export function generateLlmsTxt(result: LlmsTxtResult): string { + const lines: string[] = [] + + // Title + lines.push(`# ${result.packageName}@${result.version}`) + lines.push('') + + // Description blockquote + if (result.description) { + lines.push(`> ${result.description}`) + lines.push('') + } + + // Metadata + const meta: string[] = [] + if (result.homepage) meta.push(`- Homepage: ${result.homepage}`) + if (result.repositoryUrl) meta.push(`- Repository: ${result.repositoryUrl}`) + meta.push(`- npm: https://www.npmjs.com/package/${result.packageName}/v/${result.version}`) + lines.push(...meta) + lines.push('') + + // README + if (result.readme) { + lines.push('## README') + lines.push('') + lines.push(result.readme) + lines.push('') + } + + // Agent instructions + if (result.agentFiles.length > 0) { + lines.push('## Agent Instructions') + lines.push('') + + for (const file of result.agentFiles) { + lines.push(`### ${file.displayName} (\`${file.path}\`)`) + lines.push('') + lines.push(file.content) + lines.push('') + } + } + + return lines.join('\n').trimEnd() + '\n' +} + +/** Standard README filenames to try from jsDelivr CDN */ +const README_FILENAMES = ['README.md', 'readme.md', 'Readme.md'] + +/** Fetch README from jsDelivr CDN as fallback */ +async function fetchReadmeFromCdn(packageName: string, version: string): Promise { + for (const filename of README_FILENAMES) { + try { + const url = `https://cdn.jsdelivr.net/npm/${packageName}@${version}/${filename}` + const response = await fetch(url) + if (response.ok) return await response.text() + } catch { + // Try next + } + } + return null +} + +/** Extract README from packument data */ +function getReadmeFromPackument( + packageData: Awaited>, + requestedVersion?: string, +): string | null { + const readme = requestedVersion + ? packageData.versions[requestedVersion]?.readme + : packageData.readme + + if (readme && readme !== NPM_MISSING_README_SENTINEL) { + return readme + } + return null +} + +/** Extract a clean repository URL from packument repository field */ +function parseRepoUrl( + repository?: { type?: string; url?: string; directory?: string } | string, +): string | undefined { + if (!repository) return undefined + const url = typeof repository === 'string' ? repository : repository.url + if (!url) return undefined + return url.replace(/^git\+/, '').replace(/\.git$/, '') +} + +/** + * Orchestrates fetching all data and generating llms.txt for a package. + * + * When `includeAgentFiles` is false (default, for llms.txt), skips the file tree + * fetch and agent file discovery entirely — only returns README + metadata. + * When true (for llms_full.txt), includes agent instruction files. + */ +export async function handleLlmsTxt( + packageName: string, + requestedVersion?: string, + options?: { includeAgentFiles?: boolean }, +): Promise { + const includeAgentFiles = options?.includeAgentFiles ?? false + + const packageData = await fetchNpmPackage(packageName) + const resolvedVersion = requestedVersion ?? packageData['dist-tags']?.latest + + if (!resolvedVersion) { + throw createError({ statusCode: 404, message: 'Could not resolve package version.' }) + } + + // Extract README from packument (sync) + const readmeFromPackument = getReadmeFromPackument(packageData, requestedVersion) + + let agentFiles: AgentFile[] = [] + let cdnReadme: string | null = null + + if (includeAgentFiles) { + // Full mode: fetch file tree for agent discovery + README fallback in parallel + const [fileTreeData, readme] = await Promise.all([ + fetchFileTree(packageName, resolvedVersion), + readmeFromPackument ? null : fetchReadmeFromCdn(packageName, resolvedVersion), + ]) + cdnReadme = readme + const agentFilePaths = discoverAgentFiles(fileTreeData.files) + agentFiles = await fetchAgentFiles(packageName, resolvedVersion, agentFilePaths) + } else if (!readmeFromPackument) { + // Standard mode: only fetch README from CDN if packument lacks it + cdnReadme = await fetchReadmeFromCdn(packageName, resolvedVersion) + } + + const readme = readmeFromPackument ?? cdnReadme ?? undefined + + const result: LlmsTxtResult = { + packageName, + version: resolvedVersion, + description: packageData.description, + homepage: packageData.homepage, + repositoryUrl: parseRepoUrl(packageData.repository), + readme, + agentFiles, + } + + return generateLlmsTxt(result) +} + +// Validation for org names (matches server/api/registry/org/[org]/packages.get.ts) +const NPM_ORG_NAME_RE = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/i + +/** + * Generate llms.txt for an npm organization/scope. + * Lists all packages in the org with links to their llms.txt pages. + */ +export async function handleOrgLlmsTxt(orgName: string, baseUrl: string): Promise { + if (!orgName || orgName.length > 50 || !NPM_ORG_NAME_RE.test(orgName)) { + throw createError({ statusCode: 404, message: `Invalid org name: ${orgName}` }) + } + + const data = await $fetch>( + `${NPM_REGISTRY}/-/org/${encodeURIComponent(orgName)}/package`, + ) + + const packages = Object.keys(data).sort() + + if (packages.length === 0) { + throw createError({ statusCode: 404, message: `No packages found for @${orgName}` }) + } + + const lines: string[] = [] + + lines.push(`# @${orgName}`) + lines.push('') + lines.push(`> npm packages published under the @${orgName} scope`) + lines.push('') + lines.push(`- npm: https://www.npmjs.com/org/${orgName}`) + lines.push('') + lines.push('## Packages') + lines.push('') + + for (const pkg of packages) { + const encodedPkg = pkg.replace('/', '/') + lines.push(`- [${pkg}](${baseUrl}/package/${encodedPkg}/llms.txt)`) + } + + lines.push('') + + return lines.join('\n').trimEnd() + '\n' +} + +/** + * Generate the root /llms.txt explaining available routes. + */ +export function generateRootLlmsTxt(baseUrl: string): string { + const lines: string[] = [] + + lines.push('# npmx.dev') + lines.push('') + lines.push('> A fast, modern browser for the npm registry') + lines.push('') + lines.push('This site provides LLM-friendly documentation for npm packages.') + lines.push('') + lines.push('## Available Routes') + lines.push('') + lines.push('### Package Documentation (llms.txt)') + lines.push('') + lines.push('README and package metadata in markdown format.') + lines.push('') + lines.push(`- \`${baseUrl}/package//llms.txt\` — unscoped package (latest version)`) + lines.push( + `- \`${baseUrl}/package//v//llms.txt\` — unscoped package (specific version)`, + ) + lines.push(`- \`${baseUrl}/package/@//llms.txt\` — scoped package (latest version)`) + lines.push( + `- \`${baseUrl}/package/@//v//llms.txt\` — scoped package (specific version)`, + ) + lines.push('') + lines.push('### Full Package Documentation (llms_full.txt)') + lines.push('') + lines.push( + 'README, package metadata, and agent instruction files (CLAUDE.md, .cursorrules, etc.).', + ) + lines.push('') + lines.push(`- \`${baseUrl}/package//llms_full.txt\` — unscoped package (latest version)`) + lines.push( + `- \`${baseUrl}/package//v//llms_full.txt\` — unscoped package (specific version)`, + ) + lines.push( + `- \`${baseUrl}/package/@//llms_full.txt\` — scoped package (latest version)`, + ) + lines.push( + `- \`${baseUrl}/package/@//v//llms_full.txt\` — scoped package (specific version)`, + ) + lines.push('') + lines.push('### Organization Packages (llms.txt)') + lines.push('') + lines.push('List of all packages under an npm scope with links to their documentation.') + lines.push('') + lines.push(`- \`${baseUrl}/package/@/llms.txt\` — organization package listing`) + lines.push('') + lines.push('## Examples') + lines.push('') + lines.push(`- [nuxt llms.txt](${baseUrl}/package/nuxt/llms.txt)`) + lines.push(`- [nuxt llms_full.txt](${baseUrl}/package/nuxt/llms_full.txt)`) + lines.push(`- [@nuxt/kit llms.txt](${baseUrl}/package/@nuxt/kit/llms.txt)`) + lines.push(`- [@nuxt org packages](${baseUrl}/package/@nuxt/llms.txt)`) + lines.push('') + + return lines.join('\n').trimEnd() + '\n' +} diff --git a/shared/types/index.ts b/shared/types/index.ts index 88e28afe0..e378738e1 100644 --- a/shared/types/index.ts +++ b/shared/types/index.ts @@ -9,3 +9,4 @@ export * from './i18n-status' export * from './comparison' export * from './skills' export * from './version-downloads' +export * from './llms-txt' diff --git a/shared/types/llms-txt.ts b/shared/types/llms-txt.ts new file mode 100644 index 000000000..85bca8758 --- /dev/null +++ b/shared/types/llms-txt.ts @@ -0,0 +1,31 @@ +/** + * Agent instruction file discovered in a package + */ +export interface AgentFile { + /** Relative path within the package (e.g., "CLAUDE.md", ".github/copilot-instructions.md") */ + path: string + /** File content */ + content: string + /** Human-readable display name (e.g., "Claude Code", "GitHub Copilot") */ + displayName: string +} + +/** + * Result of gathering all data needed to generate llms.txt + */ +export interface LlmsTxtResult { + /** Package name (e.g., "nuxt" or "@nuxt/kit") */ + packageName: string + /** Resolved version (e.g., "3.12.0") */ + version: string + /** Package description from packument */ + description?: string + /** Homepage URL */ + homepage?: string + /** Repository URL */ + repositoryUrl?: string + /** README content (raw markdown) */ + readme?: string + /** Discovered agent instruction files */ + agentFiles: AgentFile[] +} diff --git a/test/unit/server/utils/llms-txt.spec.ts b/test/unit/server/utils/llms-txt.spec.ts new file mode 100644 index 000000000..fb3eafd8a --- /dev/null +++ b/test/unit/server/utils/llms-txt.spec.ts @@ -0,0 +1,335 @@ +import { describe, expect, it, vi } from 'vitest' +import type { JsDelivrFileNode, LlmsTxtResult } from '../../../../shared/types' +import { + discoverAgentFiles, + fetchAgentFiles, + generateLlmsTxt, + generateRootLlmsTxt, +} from '../../../../server/utils/llms-txt' + +describe('discoverAgentFiles', () => { + it('discovers root-level agent files', () => { + const files: JsDelivrFileNode[] = [ + { type: 'file', name: 'CLAUDE.md', size: 100 }, + { type: 'file', name: 'AGENTS.md', size: 200 }, + { type: 'file', name: 'AGENT.md', size: 50 }, + { type: 'file', name: '.cursorrules', size: 80 }, + { type: 'file', name: '.windsurfrules', size: 60 }, + { type: 'file', name: '.clinerules', size: 40 }, + { type: 'file', name: 'package.json', size: 500 }, + { type: 'file', name: 'README.md', size: 3000 }, + ] + + const result = discoverAgentFiles(files) + + expect(result).toContain('CLAUDE.md') + expect(result).toContain('AGENTS.md') + expect(result).toContain('AGENT.md') + expect(result).toContain('.cursorrules') + expect(result).toContain('.windsurfrules') + expect(result).toContain('.clinerules') + expect(result).not.toContain('package.json') + expect(result).not.toContain('README.md') + expect(result).toHaveLength(6) + }) + + it('discovers .github/copilot-instructions.md', () => { + const files: JsDelivrFileNode[] = [ + { + type: 'directory', + name: '.github', + files: [ + { type: 'file', name: 'copilot-instructions.md', size: 150 }, + { type: 'file', name: 'FUNDING.yml', size: 30 }, + ], + }, + ] + + const result = discoverAgentFiles(files) + + expect(result).toEqual(['.github/copilot-instructions.md']) + }) + + it('discovers .cursor/rules/*.md files', () => { + const files: JsDelivrFileNode[] = [ + { + type: 'directory', + name: '.cursor', + files: [ + { + type: 'directory', + name: 'rules', + files: [ + { type: 'file', name: 'coding-style.md', size: 100 }, + { type: 'file', name: 'testing.md', size: 80 }, + { type: 'file', name: 'config.json', size: 50 }, + ], + }, + ], + }, + ] + + const result = discoverAgentFiles(files) + + expect(result).toContain('.cursor/rules/coding-style.md') + expect(result).toContain('.cursor/rules/testing.md') + expect(result).not.toContain('.cursor/rules/config.json') + expect(result).toHaveLength(2) + }) + + it('discovers .windsurf/rules/*.md files', () => { + const files: JsDelivrFileNode[] = [ + { + type: 'directory', + name: '.windsurf', + files: [ + { + type: 'directory', + name: 'rules', + files: [{ type: 'file', name: 'project.md', size: 200 }], + }, + ], + }, + ] + + const result = discoverAgentFiles(files) + + expect(result).toEqual(['.windsurf/rules/project.md']) + }) + + it('returns empty array for empty file tree', () => { + expect(discoverAgentFiles([])).toEqual([]) + }) + + it('returns empty array when no agent files exist', () => { + const files: JsDelivrFileNode[] = [ + { type: 'file', name: 'package.json', size: 500 }, + { type: 'file', name: 'index.js', size: 1000 }, + { + type: 'directory', + name: 'src', + files: [{ type: 'file', name: 'main.ts', size: 200 }], + }, + ] + + expect(discoverAgentFiles(files)).toEqual([]) + }) +}) + +describe('fetchAgentFiles', () => { + it('fetches files in parallel and returns results', async () => { + const fetchMock = vi.fn().mockImplementation((url: string) => { + if (url.includes('CLAUDE.md')) { + return Promise.resolve({ ok: true, text: () => Promise.resolve('# Claude instructions') }) + } + if (url.includes('AGENTS.md')) { + return Promise.resolve({ ok: true, text: () => Promise.resolve('# Agent config') }) + } + return Promise.resolve({ ok: false }) + }) + vi.stubGlobal('fetch', fetchMock) + + try { + const result = await fetchAgentFiles('test-pkg', '1.0.0', ['CLAUDE.md', 'AGENTS.md']) + + expect(result).toHaveLength(2) + expect(result[0]).toMatchObject({ + path: 'CLAUDE.md', + content: '# Claude instructions', + displayName: 'Claude Code', + }) + expect(result[1]).toMatchObject({ + path: 'AGENTS.md', + content: '# Agent config', + displayName: 'Agent Instructions', + }) + expect(fetchMock).toHaveBeenCalledTimes(2) + } finally { + vi.unstubAllGlobals() + } + }) + + it('gracefully skips failed fetches', async () => { + const fetchMock = vi.fn().mockImplementation((url: string) => { + if (url.includes('CLAUDE.md')) { + return Promise.resolve({ ok: true, text: () => Promise.resolve('# Claude') }) + } + return Promise.resolve({ ok: false }) + }) + vi.stubGlobal('fetch', fetchMock) + + try { + const result = await fetchAgentFiles('test-pkg', '1.0.0', ['CLAUDE.md', 'missing.md']) + + expect(result).toHaveLength(1) + expect(result[0]?.path).toBe('CLAUDE.md') + } finally { + vi.unstubAllGlobals() + } + }) + + it('gracefully handles network errors', async () => { + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('Network error'))) + + try { + const result = await fetchAgentFiles('test-pkg', '1.0.0', ['CLAUDE.md']) + expect(result).toEqual([]) + } finally { + vi.unstubAllGlobals() + } + }) + + it('returns empty array for empty file paths', async () => { + const result = await fetchAgentFiles('test-pkg', '1.0.0', []) + expect(result).toEqual([]) + }) + + it('constructs correct CDN URLs for scoped packages', async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve('content'), + }) + vi.stubGlobal('fetch', fetchMock) + + try { + await fetchAgentFiles('@nuxt/kit', '1.0.0', ['CLAUDE.md']) + expect(fetchMock).toHaveBeenCalledWith( + 'https://cdn.jsdelivr.net/npm/@nuxt/kit@1.0.0/CLAUDE.md', + ) + } finally { + vi.unstubAllGlobals() + } + }) +}) + +describe('generateLlmsTxt', () => { + it('generates full output with all fields', () => { + const result: LlmsTxtResult = { + packageName: 'nuxt', + version: '3.12.0', + description: 'The Intuitive Vue Framework', + homepage: 'https://nuxt.com', + repositoryUrl: 'https://github.com/nuxt/nuxt', + readme: '# Nuxt\n\nThe Intuitive Vue Framework.', + agentFiles: [ + { + path: 'CLAUDE.md', + content: '# Claude\n\nUse Nuxt conventions.', + displayName: 'Claude Code', + }, + { path: '.cursorrules', content: 'Use composition API.', displayName: 'Cursor Rules' }, + ], + } + + const output = generateLlmsTxt(result) + + expect(output).toContain('# nuxt@3.12.0') + expect(output).toContain('> The Intuitive Vue Framework') + expect(output).toContain('- Homepage: https://nuxt.com') + expect(output).toContain('- Repository: https://github.com/nuxt/nuxt') + expect(output).toContain('- npm: https://www.npmjs.com/package/nuxt/v/3.12.0') + expect(output).toContain('## README') + expect(output).toContain('# Nuxt') + expect(output).toContain('## Agent Instructions') + expect(output).toContain('### Claude Code (`CLAUDE.md`)') + expect(output).toContain('Use Nuxt conventions.') + expect(output).toContain('### Cursor Rules (`.cursorrules`)') + expect(output).toContain('Use composition API.') + expect(output.endsWith('\n')).toBe(true) + }) + + it('generates minimal output with no optional fields', () => { + const result: LlmsTxtResult = { + packageName: 'tiny-pkg', + version: '0.1.0', + agentFiles: [], + } + + const output = generateLlmsTxt(result) + + expect(output).toContain('# tiny-pkg@0.1.0') + expect(output).toContain('- npm: https://www.npmjs.com/package/tiny-pkg/v/0.1.0') + expect(output).not.toContain('>') + expect(output).not.toContain('Homepage') + expect(output).not.toContain('Repository') + expect(output).not.toContain('## README') + expect(output).not.toContain('## Agent Instructions') + }) + + it('omits Agent Instructions section when no agent files exist', () => { + const result: LlmsTxtResult = { + packageName: 'test-pkg', + version: '1.0.0', + description: 'A test package', + readme: '# Test\n\nHello world.', + agentFiles: [], + } + + const output = generateLlmsTxt(result) + + expect(output).toContain('## README') + expect(output).not.toContain('## Agent Instructions') + }) + + it('omits README section when no readme provided', () => { + const result: LlmsTxtResult = { + packageName: 'no-readme', + version: '1.0.0', + agentFiles: [ + { path: 'AGENTS.md', content: 'Agent rules here.', displayName: 'Agent Instructions' }, + ], + } + + const output = generateLlmsTxt(result) + + expect(output).not.toContain('## README') + expect(output).toContain('## Agent Instructions') + expect(output).toContain('### Agent Instructions (`AGENTS.md`)') + }) + + it('handles scoped package names in npm URL', () => { + const result: LlmsTxtResult = { + packageName: '@nuxt/kit', + version: '1.0.0', + agentFiles: [], + } + + const output = generateLlmsTxt(result) + + expect(output).toContain('# @nuxt/kit@1.0.0') + expect(output).toContain('- npm: https://www.npmjs.com/package/@nuxt/kit/v/1.0.0') + }) +}) + +describe('generateRootLlmsTxt', () => { + it('includes all route patterns', () => { + const output = generateRootLlmsTxt('https://npmx.dev') + + expect(output).toContain('# npmx.dev') + expect(output).toContain('https://npmx.dev/package//llms.txt') + expect(output).toContain('https://npmx.dev/package//v//llms.txt') + expect(output).toContain('https://npmx.dev/package/@//llms.txt') + expect(output).toContain('https://npmx.dev/package/@//v//llms.txt') + expect(output).toContain('https://npmx.dev/package//llms_full.txt') + expect(output).toContain('https://npmx.dev/package/@/llms.txt') + }) + + it('includes example links', () => { + const output = generateRootLlmsTxt('https://npmx.dev') + + expect(output).toContain('[nuxt llms.txt](https://npmx.dev/package/nuxt/llms.txt)') + expect(output).toContain('[@nuxt org packages](https://npmx.dev/package/@nuxt/llms.txt)') + }) + + it('uses provided base URL', () => { + const output = generateRootLlmsTxt('http://localhost:3000') + + expect(output).toContain('http://localhost:3000/package//llms.txt') + expect(output).not.toContain('https://npmx.dev') + }) + + it('ends with newline', () => { + const output = generateRootLlmsTxt('https://npmx.dev') + expect(output.endsWith('\n')).toBe(true) + }) +}) diff --git a/vitest.config.ts b/vitest.config.ts index 9b96a1dbe..3be3e6f21 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -12,6 +12,7 @@ export default defineConfig({ resolve: { alias: { '#shared': `${rootDir}/shared`, + '#server': `${rootDir}/server`, }, }, test: {