diff --git a/.gitignore b/.gitignore
index 0f1fffd8970..c0a76da121f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,4 +46,11 @@ _pagefind/
# agents llms
AGENTS.md
/public/llms.txt
+/public/llms-full.txt
+/public/**/*.md
+!/public/SKILL.md
+/public/**/llms.txt
+/public/robots.txt
+/public/sitemap.xml
+/public/.well-known/
package-lock.json
diff --git a/app/layout.tsx b/app/layout.tsx
index 8d4b124174e..e5da0a3c200 100644
--- a/app/layout.tsx
+++ b/app/layout.tsx
@@ -6,6 +6,10 @@ import "nextra-theme-docs/style.css";
import "katex/dist/katex.min.css";
import { FontStyles } from "@/components/FontStyles";
+const SITE_ORIGIN = "https://docs.celestia.org";
+const SITE_DESCRIPTION =
+ "Learn, build, and operate on Celestia - the modular data availability network.";
+
// Use BASE env var (same as next.config.mjs) and ensure it's available client-side
const basePath =
process.env.NEXT_PUBLIC_BASE_PATH ||
@@ -23,10 +27,142 @@ const THEME_CONFIG = {
primarySaturation: 100,
};
export const metadata = {
- // Define your metadata here
- // For more information on metadata API, see: https://nextjs.org/docs/app/building-your-application/optimizing/metadata
+ metadataBase: new URL(SITE_ORIGIN),
+ title: {
+ default: "Celestia Documentation",
+ template: "%s - Celestia Documentation",
+ },
+ description: SITE_DESCRIPTION,
+ openGraph: {
+ type: "website",
+ siteName: "Celestia Documentation",
+ title: "Celestia Documentation",
+ description: SITE_DESCRIPTION,
+ images: [
+ {
+ url: "/Celestia-og.png",
+ width: 1200,
+ height: 630,
+ alt: "Celestia Documentation",
+ },
+ ],
+ },
+ twitter: {
+ card: "summary_large_image",
+ site: "@CelestiaOrg",
+ title: "Celestia Documentation",
+ description: SITE_DESCRIPTION,
+ images: ["/Celestia-og.png"],
+ },
+ robots: {
+ index: true,
+ follow: true,
+ googleBot: {
+ index: true,
+ follow: true,
+ },
+ },
+};
+
+const organizationJsonLd = {
+ "@context": "https://schema.org",
+ "@type": "Organization",
+ name: "Celestia",
+ url: "https://celestia.org",
+ logo: `${SITE_ORIGIN}/logo-light.svg`,
+ sameAs: [
+ "https://github.com/celestiaorg",
+ "https://x.com/CelestiaOrg",
+ "https://discord.com/invite/YsnTPcSfWQ",
+ ],
+};
+
+const websiteJsonLd = {
+ "@context": "https://schema.org",
+ "@type": "WebSite",
+ name: "Celestia Documentation",
+ url: SITE_ORIGIN,
+ description: SITE_DESCRIPTION,
+};
+
+const documentationJsonLd = {
+ "@context": "https://schema.org",
+ "@type": "TechArticle",
+ headline: "Celestia Documentation",
+ description: SITE_DESCRIPTION,
+ url: SITE_ORIGIN,
+ publisher: {
+ "@type": "Organization",
+ name: "Celestia",
+ url: "https://celestia.org",
+ },
+ about: [
+ "Celestia",
+ "data availability",
+ "modular blockchain",
+ "blobspace",
+ "node operation",
+ ],
+};
+
+const softwareApplicationJsonLd = {
+ "@context": "https://schema.org",
+ "@type": "SoftwareApplication",
+ name: "Celestia Node API",
+ applicationCategory: "DeveloperApplication",
+ operatingSystem: ["Linux", "macOS"],
+ url: `${SITE_ORIGIN}/build/rpc/node-api/`,
+ softwareHelp: `${SITE_ORIGIN}/operate/maintenance/troubleshooting/`,
+ isAccessibleForFree: true,
+ publisher: {
+ "@type": "Organization",
+ name: "Celestia",
+ url: "https://celestia.org",
+ },
};
+const faqJsonLd = {
+ "@context": "https://schema.org",
+ "@type": "FAQPage",
+ mainEntity: [
+ {
+ "@type": "Question",
+ name: "Where can I troubleshoot Celestia node issues?",
+ acceptedAnswer: {
+ "@type": "Answer",
+ text: "Use the Celestia troubleshooting documentation for common node, sync, network, and operational issues.",
+ url: `${SITE_ORIGIN}/operate/maintenance/troubleshooting/`,
+ },
+ },
+ {
+ "@type": "Question",
+ name: "Where can I find the Celestia Node API reference?",
+ acceptedAnswer: {
+ "@type": "Answer",
+ text: "Use the Celestia Node API reference for RPC methods and the OpenRPC specification for machine-readable API details.",
+ url: `${SITE_ORIGIN}/build/rpc/node-api/`,
+ },
+ },
+ {
+ "@type": "Question",
+ name: "Where can I learn how to run a Celestia light node?",
+ acceptedAnswer: {
+ "@type": "Answer",
+ text: "Start with the light node quickstart guide, then use the troubleshooting page if setup, sync, or connectivity issues appear.",
+ url: `${SITE_ORIGIN}/operate/data-availability/light-node/quickstart/`,
+ },
+ },
+ ],
+};
+
+const jsonLd = [
+ organizationJsonLd,
+ websiteJsonLd,
+ documentationJsonLd,
+ softwareApplicationJsonLd,
+ faqJsonLd,
+];
+
const banner = (
Welcome to our new docs! 🎉
);
@@ -83,11 +219,29 @@ export default async function RootLayout({
href={withBasePath("/favicons/favicon-16x16.png")}
/>
-
-
+
+
+
+ {jsonLd.map((data, index) => (
+
+ ))}
{
const trimmedExpr = expression.trim();
+ if (!/^(mainnetVersions|mochaVersions|arabicaVersions|constants)(?:\[['"][^'"]+['"]\]|\.\w+)$/.test(trimmedExpr)) {
+ return match;
+ }
+
const resolved = resolveExpression(trimmedExpr);
if (resolved !== null) {
@@ -149,10 +156,22 @@ const header = [
'',
'> Official documentation for Celestia, the modular blockchain powering unstoppable apps with full-stack control.',
'',
- 'These docs are built with Next.js + Nextra and exported statically. Links below point to the raw MDX sources in `main` so tools and LLMs can ingest clean text.',
+ 'These docs are built with Next.js + Nextra and exported statically. Links below point to canonical markdown pages so tools and LLMs can ingest clean text.',
+ '',
+ '## Agent instructions',
+ '',
+ '- Prefer the `.md` pages linked here for retrieval and citations.',
+ '- Use `llms-full.txt` when a single-file context snapshot is needed.',
+ '- Use `/build/llms.txt`, `/learn/llms.txt`, and `/operate/llms.txt` for section-specific context.',
+ '- Celestia does not currently publish an official MCP server for this documentation site.',
'',
'## Related resources',
'',
+ '- Full LLM context: https://docs.celestia.org/llms-full.txt',
+ '- Agent skill: https://docs.celestia.org/SKILL.md',
+ '- Agent skills index: https://docs.celestia.org/.well-known/agent-skills/index.json',
+ '- API catalog: https://docs.celestia.org/.well-known/api-catalog',
+ '- Node API OpenRPC spec: https://docs.celestia.org/specs/openrpc-v0.28.4.json',
'- CIPs (Celestia Improvement Proposals): https://cips.celestia.org',
];
@@ -163,7 +182,52 @@ const titleize = (segment) =>
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
.join(' ');
-const formatItem = (path, label) => `- [${label}](${RAW_BASE}${path})`;
+const toPosixPath = (filePath) => filePath.split(path.sep).join('/');
+
+const markdownPathForFile = (file) => {
+ let outputPath = toPosixPath(file).replace(/^app\//, '').replace(/\.mdx$/, '.md');
+
+ if (outputPath.endsWith('/page.md')) {
+ outputPath = outputPath.replace(/\/page\.md$/, '.md');
+ } else if (outputPath === 'page.md') {
+ outputPath = 'index.md';
+ }
+
+ return outputPath;
+};
+
+const htmlPathForFile = (file) => {
+ const rel = toPosixPath(file).replace(/^app\//, '');
+
+ if (rel === 'page.mdx') return '/';
+ if (!rel.endsWith('/page.mdx')) return null;
+
+ return `/${rel.replace(/\/page\.mdx$/, '/')}`;
+};
+
+const canonicalUrl = (pathname) => `${SITE_ORIGIN}${pathname}`;
+
+const canonicalMarkdownUrl = (file) => `${SITE_ORIGIN}/${markdownPathForFile(file)}`;
+
+const formatItem = (file, label) => `- [${label}](${canonicalMarkdownUrl(file)})`;
+
+const escapeXml = (value) =>
+ value
+ .replaceAll('&', '&')
+ .replaceAll('<', '<')
+ .replaceAll('>', '>')
+ .replaceAll('"', '"')
+ .replaceAll("'", ''');
+
+const extractTitle = (content, fallback) => {
+ const heading = content.match(/^#\s+(.+)$/m);
+ return heading ? heading[1].trim() : fallback;
+};
+
+const sha256 = async (file) => {
+ const content = await fs.readFile(file);
+ return createHash('sha256').update(content).digest('hex');
+};
const walkPages = async (dir) => {
const pages = [];
@@ -179,15 +243,18 @@ const walkPages = async (dir) => {
return pages;
};
-const buildSections = async () => {
- const files = (await walkPages('app')).sort();
-
+const buildSections = (files) => {
const grouped = new Map();
for (const file of files) {
- const rel = file.replace(/^app\//, '');
+ const rel = toPosixPath(file).replace(/^app\//, '');
const parts = rel.split('/');
- if (parts.length < 2) continue; // skip unexpected roots
+ if (parts.length < 2) {
+ const label = rel === 'page.mdx' ? 'Home' : titleize(rel.replace(/\.mdx$/, ''));
+ if (!grouped.has('Overview')) grouped.set('Overview', []);
+ grouped.get('Overview').push({ file, label });
+ continue;
+ }
const [top, second, ...rest] = parts;
// Section title: "Top" or "Top: Second"
@@ -199,39 +266,33 @@ const buildSections = async () => {
const label = labelSegments.map(titleize).join(' / ');
if (!grouped.has(sectionTitle)) grouped.set(sectionTitle, []);
- grouped.get(sectionTitle).push({ path: file, label });
+ grouped.get(sectionTitle).push({ file, label });
}
// Sort sections and items for stability
- const sortedSections = [...grouped.entries()].sort(([a], [b]) => a.localeCompare(b));
+ const sortedSections = [...grouped.entries()].sort(([a], [b]) => {
+ if (a === 'Overview') return -1;
+ if (b === 'Overview') return 1;
+ return a.localeCompare(b);
+ });
return sortedSections.map(([title, items]) => ({
title,
- items: items.sort((a, b) => a.path.localeCompare(b.path)),
+ items: items.sort((a, b) => a.file.localeCompare(b.file)),
}));
};
-const generateMarkdownFiles = async (outputBase) => {
+const generateMarkdownFiles = async (outputBase, files) => {
console.log('🤖 Generating LLM-ready markdown files...');
- // Find all MDX files
- const files = await walkPages('app');
-
console.log(`Found ${files.length} MDX files to convert`);
console.log(`Writing markdown files to: ${outputBase}/`);
+ const pages = [];
+
for (const file of files) {
const content = await fs.readFile(file, 'utf-8');
const cleanedContent = cleanMdxContent(content);
-
- // Determine output path
- let outputPath = file.replace(/^app\//, '').replace(/\.mdx$/, '.md');
-
- // Handle page.mdx files - they should become index.md
- if (outputPath.endsWith('/page.md')) {
- outputPath = outputPath.replace(/\/page\.md$/, '.md');
- } else if (outputPath === 'page.md') {
- outputPath = 'index.md';
- }
+ const outputPath = markdownPathForFile(file);
const outputFullPath = path.join(outputBase, outputPath);
@@ -242,34 +303,233 @@ const generateMarkdownFiles = async (outputBase) => {
// Write the cleaned markdown file
await fs.writeFile(outputFullPath, cleanedContent, 'utf-8');
console.log(`✅ Generated: ${outputPath}`);
+
+ pages.push({
+ file,
+ markdownPath: outputPath,
+ markdownUrl: canonicalMarkdownUrl(file),
+ htmlPath: htmlPathForFile(file),
+ content: cleanedContent,
+ title: extractTitle(cleanedContent, titleize(path.basename(path.dirname(file)))),
+ });
}
-};
-const main = async () => {
- // Determine output directory once - use 'out' if it exists (during build), otherwise 'public' (during dev)
- const outputBase = await fs.access('out').then(() => 'out').catch(() => 'public');
+ return pages;
+};
- // Generate individual markdown files
- await generateMarkdownFiles(outputBase);
+const writeTextFile = async (outputBase, filePath, content) => {
+ const outputFullPath = path.join(outputBase, filePath);
+ await fs.mkdir(path.dirname(outputFullPath), { recursive: true });
+ await fs.writeFile(outputFullPath, content, 'utf8');
+};
- // Also generate the index file
- const sections = await buildSections();
+const generateLlmsTxt = async (outputBase, sections, filePath = 'llms.txt') => {
const lines = [...header, ''];
sections.forEach((section) => {
if (!section.items.length) return;
lines.push(`## ${section.title}`, '');
- section.items.forEach((item) => lines.push(formatItem(item.path, item.label)));
+ section.items.forEach((item) => lines.push(formatItem(item.file, item.label)));
lines.push('');
});
const output = lines.join('\n').replace(/\n{3,}/g, '\n\n').trimEnd() + '\n';
+ await writeTextFile(outputBase, filePath, output);
+};
- // Write llms.txt to the appropriate directory
- await fs.mkdir(outputBase, { recursive: true });
- await fs.writeFile(path.join(outputBase, 'llms.txt'), output, 'utf8');
+const generateSectionLlmsTxt = async (outputBase, sections) => {
+ for (const topLevel of ['build', 'learn', 'operate']) {
+ const sectionPrefix = `${titleize(topLevel)}: `;
+ const sectionItems = sections.filter((section) => section.title.startsWith(sectionPrefix));
+ const lines = [
+ `# Celestia ${titleize(topLevel)} documentation`,
+ '',
+ `> Section-specific LLM index for the ${titleize(topLevel)} area of the Celestia docs.`,
+ '',
+ '- Full docs index: https://docs.celestia.org/llms.txt',
+ '- Full LLM context: https://docs.celestia.org/llms-full.txt',
+ '',
+ ];
+
+ sectionItems.forEach((section) => {
+ lines.push(`## ${section.title.replace(sectionPrefix, '')}`, '');
+ section.items.forEach((item) => lines.push(formatItem(item.file, item.label)));
+ lines.push('');
+ });
+
+ await writeTextFile(outputBase, `${topLevel}/llms.txt`, lines.join('\n').replace(/\n{3,}/g, '\n\n').trimEnd() + '\n');
+ }
+};
- console.log(`llms.txt generated (${outputBase}/)`);
- console.log('✨ LLM-ready markdown generation complete!');
+const generateLlmsFullTxt = async (outputBase, pages) => {
+ const lines = [
+ '# Celestia documentation full context',
+ '',
+ '> Full cleaned markdown export of the Celestia documentation for LLM and agent ingestion.',
+ '',
+ ];
+
+ pages
+ .filter((page) => page.content)
+ .sort((a, b) => a.markdownPath.localeCompare(b.markdownPath))
+ .forEach((page) => {
+ lines.push('---');
+ lines.push(`Title: ${page.title}`);
+ lines.push(`URL: ${page.markdownUrl}`);
+ lines.push(`Source: ${toPosixPath(page.file)}`);
+ lines.push('---');
+ lines.push('');
+ lines.push(page.content);
+ lines.push('');
+ });
+
+ await writeTextFile(outputBase, 'llms-full.txt', lines.join('\n').replace(/\n{3,}/g, '\n\n').trimEnd() + '\n');
+};
+
+const generateSitemap = async (outputBase, pages) => {
+ const urls = pages
+ .map((page) => page.htmlPath)
+ .filter(Boolean)
+ .sort()
+ .map((pathname) => ` \n ${escapeXml(canonicalUrl(pathname))}\n `);
+
+ const sitemap = [
+ '',
+ '',
+ ...urls,
+ '',
+ '',
+ ].join('\n');
+
+ await writeTextFile(outputBase, 'sitemap.xml', sitemap);
+};
+
+const generateRobotsTxt = async (outputBase) => {
+ const aiBots = [
+ 'GPTBot',
+ 'OAI-SearchBot',
+ 'ClaudeBot',
+ 'Claude-Web',
+ 'Google-Extended',
+ 'PerplexityBot',
+ 'CCBot',
+ ];
+
+ const lines = [
+ 'User-agent: *',
+ 'Allow: /',
+ '',
+ ...aiBots.flatMap((bot) => [`User-agent: ${bot}`, 'Allow: /', '']),
+ 'Content-Signal: ai-train=yes, search=yes, ai-input=yes',
+ `Sitemap: ${SITE_ORIGIN}/sitemap.xml`,
+ '',
+ ];
+
+ await writeTextFile(outputBase, 'robots.txt', lines.join('\n'));
+};
+
+const generateAgentSkillsIndex = async (outputBase) => {
+ const skillPath = '/.well-known/agent-skills/celestia/SKILL.md';
+ const skillHash = await sha256('public/SKILL.md');
+ const index = {
+ $schema: 'https://schemas.agentskills.io/discovery/0.2.0/schema.json',
+ skills: [
+ {
+ name: 'celestia',
+ type: 'skill-md',
+ description: 'Route Celestia requests to the correct repo and apply canonical blob submit/retrieve guidance with docs guardrails.',
+ url: `${SITE_ORIGIN}${skillPath}`,
+ digest: `sha256:${skillHash}`,
+ },
+ ],
+ };
+
+ await fs.mkdir(path.join(outputBase, '.well-known/agent-skills/celestia'), { recursive: true });
+ await fs.copyFile('public/SKILL.md', path.join(outputBase, '.well-known/agent-skills/celestia/SKILL.md'));
+ await writeTextFile(outputBase, '.well-known/agent-skills/index.json', `${JSON.stringify(index, null, 2)}\n`);
+};
+
+const generateApiCatalog = async (outputBase) => {
+ const linkset = {
+ linkset: [
+ {
+ anchor: SITE_ORIGIN,
+ 'service-desc': [
+ {
+ href: `${SITE_ORIGIN}${LATEST_OPENRPC_SPEC}`,
+ type: 'application/json',
+ title: 'Celestia Node API OpenRPC specification',
+ },
+ ],
+ 'service-doc': [
+ {
+ href: `${SITE_ORIGIN}/build/rpc/node-api/`,
+ type: 'text/html',
+ title: 'Celestia Node API documentation',
+ },
+ {
+ href: `${SITE_ORIGIN}/build/rpc/clients/`,
+ type: 'text/html',
+ title: 'Celestia RPC clients',
+ },
+ ],
+ describedby: [
+ {
+ href: `${SITE_ORIGIN}/llms.txt`,
+ type: 'text/plain',
+ title: 'Celestia docs LLM index',
+ },
+ {
+ href: `${SITE_ORIGIN}/llms-full.txt`,
+ type: 'text/plain',
+ title: 'Celestia docs full LLM context',
+ },
+ {
+ href: `${SITE_ORIGIN}/SKILL.md`,
+ type: 'text/markdown',
+ title: 'Celestia agent skill',
+ },
+ ],
+ status: [
+ {
+ href: `${SITE_ORIGIN}/operate/maintenance/troubleshooting/`,
+ type: 'text/html',
+ title: 'Celestia troubleshooting documentation',
+ },
+ ],
+ 'source-code': [
+ {
+ href: GITHUB_REPO,
+ type: 'text/html',
+ title: 'Celestia docs repository',
+ },
+ ],
+ },
+ ],
+ };
+
+ await writeTextFile(outputBase, '.well-known/api-catalog', `${JSON.stringify(linkset, null, 2)}\n`);
+};
+
+const main = async () => {
+ // Determine output directory once - use 'out' if it exists (during build), otherwise 'public' (during dev)
+ const outputBase = await fs.access('out').then(() => 'out').catch(() => 'public');
+ const files = (await walkPages('app')).sort();
+
+ // Generate individual markdown files
+ const pages = await generateMarkdownFiles(outputBase, files);
+
+ const sections = buildSections(files);
+ await fs.mkdir(outputBase, { recursive: true });
+ await generateLlmsTxt(outputBase, sections);
+ await generateSectionLlmsTxt(outputBase, sections);
+ await generateLlmsFullTxt(outputBase, pages);
+ await generateSitemap(outputBase, pages);
+ await generateRobotsTxt(outputBase);
+ await generateAgentSkillsIndex(outputBase);
+ await generateApiCatalog(outputBase);
+
+ console.log(`Agent-ready assets generated (${outputBase}/)`);
+ console.log('✨ LLM-ready markdown and discovery asset generation complete!');
};
main().catch((err) => {