|
| 1 | +import fs from 'node:fs'; |
| 2 | +import path from 'node:path'; |
| 3 | +import {fileURLToPath} from 'node:url'; |
| 4 | + |
| 5 | +const websiteDirectory = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); |
| 6 | +const repositoryDirectory = path.resolve(websiteDirectory, '..'); |
| 7 | +const generatedDocsDirectory = path.join(websiteDirectory, '.site-docs'); |
| 8 | +const staticDirectory = path.join(websiteDirectory, 'static'); |
| 9 | +const markdownDirectory = path.join(staticDirectory, 'llms'); |
| 10 | +const truthTableSource = path.join(websiteDirectory, 'data', 'ser-truth-table.json'); |
| 11 | +const publicBaseUrl = new URL('https://scriptedevents.github.io/ScriptedEventsReloaded/'); |
| 12 | + |
| 13 | +function collectMarkdownFiles(directory, result = []) { |
| 14 | + for (const entry of fs.readdirSync(directory, {withFileTypes: true})) { |
| 15 | + const filename = path.join(directory, entry.name); |
| 16 | + if (entry.isDirectory()) collectMarkdownFiles(filename, result); |
| 17 | + else if (entry.isFile() && entry.name.endsWith('.md')) result.push(filename); |
| 18 | + } |
| 19 | + return result; |
| 20 | +} |
| 21 | + |
| 22 | +function posixRelative(filename) { |
| 23 | + return path.relative(generatedDocsDirectory, filename).replaceAll('\\', '/'); |
| 24 | +} |
| 25 | + |
| 26 | +function parseFrontMatter(markdown) { |
| 27 | + const normalized = markdown.replaceAll('\r\n', '\n'); |
| 28 | + const match = normalized.match(/^---\n([\s\S]*?)\n---(?:\n|$)/); |
| 29 | + if (!match) return {attributes: {}, body: normalized}; |
| 30 | + |
| 31 | + const attributes = {}; |
| 32 | + for (const line of match[1].split('\n')) { |
| 33 | + const separator = line.indexOf(':'); |
| 34 | + if (separator === -1) continue; |
| 35 | + const key = line.slice(0, separator).trim(); |
| 36 | + const rawValue = line.slice(separator + 1).trim(); |
| 37 | + try { |
| 38 | + attributes[key] = JSON.parse(rawValue); |
| 39 | + } catch { |
| 40 | + attributes[key] = rawValue; |
| 41 | + } |
| 42 | + } |
| 43 | + return {attributes, body: normalized.slice(match[0].length).trimStart()}; |
| 44 | +} |
| 45 | + |
| 46 | +function titleFrom(relativeFilename, attributes, body) { |
| 47 | + if (attributes.title) return String(attributes.title); |
| 48 | + const heading = body.match(/^#\s+(.+)$/m)?.[1]; |
| 49 | + if (heading) return heading.trim(); |
| 50 | + return path.posix.basename(relativeFilename, '.md') |
| 51 | + .replaceAll('-', ' ') |
| 52 | + .replace(/\b\w/g, character => character.toUpperCase()); |
| 53 | +} |
| 54 | + |
| 55 | +function canonicalRoute(relativeFilename, attributes) { |
| 56 | + if (attributes.slug !== undefined) return String(attributes.slug).replace(/^\/+/, ''); |
| 57 | + return relativeFilename.replace(/\.md$/, '/'); |
| 58 | +} |
| 59 | + |
| 60 | +function markdownUrl(relativeFilename) { |
| 61 | + return new URL(`llms/${relativeFilename}`, publicBaseUrl).href; |
| 62 | +} |
| 63 | + |
| 64 | +function canonicalUrl(relativeFilename, attributes) { |
| 65 | + return new URL(canonicalRoute(relativeFilename, attributes), publicBaseUrl).href; |
| 66 | +} |
| 67 | + |
| 68 | +function escapedLinkTitle(title) { |
| 69 | + return title.replaceAll('\\', '\\\\').replaceAll('[', '\\[').replaceAll(']', '\\]'); |
| 70 | +} |
| 71 | + |
| 72 | +function slug(value) { |
| 73 | + return String(value) |
| 74 | + .toLocaleLowerCase() |
| 75 | + .replace(/[^a-z0-9]+/g, '-') |
| 76 | + .replace(/^-|-$/g, ''); |
| 77 | +} |
| 78 | + |
| 79 | +function sectionFor(relativeFilename) { |
| 80 | + if ( |
| 81 | + relativeFilename === 'README.md' |
| 82 | + || relativeFilename === 'SUMMARY.md' |
| 83 | + || relativeFilename.startsWith('getting-started/') |
| 84 | + || relativeFilename.startsWith('guides/') |
| 85 | + || relativeFilename.startsWith('tutorial/') |
| 86 | + ) return 'Learn and guides'; |
| 87 | + if (relativeFilename === 'language-specification.md' || relativeFilename.startsWith('language/')) { |
| 88 | + return 'Language reference'; |
| 89 | + } |
| 90 | + if (relativeFilename.startsWith('reference/')) return 'Generated reference'; |
| 91 | + if (relativeFilename.startsWith('examples/')) return 'Build-validated examples'; |
| 92 | + if (relativeFilename.startsWith('developer/')) return 'Developer documentation'; |
| 93 | + return 'Additional documentation'; |
| 94 | +} |
| 95 | + |
| 96 | +if (!fs.existsSync(generatedDocsDirectory)) { |
| 97 | + throw new Error('Missing .site-docs. Run the website preparation step before building LLM artifacts.'); |
| 98 | +} |
| 99 | + |
| 100 | +const pageSources = collectMarkdownFiles(generatedDocsDirectory) |
| 101 | + .map(filename => ({filename, relativeFilename: posixRelative(filename)})); |
| 102 | +const summaryFilename = path.join(repositoryDirectory, 'docs', 'SUMMARY.md'); |
| 103 | +pageSources.push({ |
| 104 | + filename: summaryFilename, |
| 105 | + relativeFilename: 'SUMMARY.md', |
| 106 | + canonicalUrlOverride: markdownUrl('SUMMARY.md'), |
| 107 | + markdownOverride: fs.readFileSync(summaryFilename, 'utf8') |
| 108 | + .replaceAll('../language_specification.md', 'language-specification.md'), |
| 109 | +}); |
| 110 | + |
| 111 | +const existingPagePaths = new Set(pageSources.map(page => page.relativeFilename)); |
| 112 | +const exampleSourceDirectory = path.join(repositoryDirectory, 'Example Scripts'); |
| 113 | +for (const entry of fs.readdirSync(exampleSourceDirectory, {recursive: true, withFileTypes: true})) { |
| 114 | + if (!entry.isFile() || !/\.(?:ser|txt)$/i.test(entry.name)) continue; |
| 115 | + const filename = path.join(entry.parentPath, entry.name); |
| 116 | + const examplePath = path.relative(exampleSourceDirectory, filename).replaceAll('\\', '/'); |
| 117 | + const exampleName = examplePath.replace(/\.(?:ser|txt)$/i, ''); |
| 118 | + const relativeFilename = `examples/${slug(exampleName)}.md`; |
| 119 | + if (existingPagePaths.has(relativeFilename)) continue; |
| 120 | + |
| 121 | + const rawUrl = `https://raw.githubusercontent.com/ScriptedEvents/ScriptedEventsReloaded/main/Example%20Scripts/${examplePath.split('/').map(encodeURIComponent).join('/')}`; |
| 122 | + const script = fs.readFileSync(filename, 'utf8').trimEnd(); |
| 123 | + pageSources.push({ |
| 124 | + filename, |
| 125 | + relativeFilename, |
| 126 | + canonicalUrlOverride: markdownUrl(relativeFilename), |
| 127 | + markdownOverride: [ |
| 128 | + `# ${exampleName}`, |
| 129 | + '', |
| 130 | + `[Download the raw \`${entry.name}\` file](${rawUrl})`, |
| 131 | + '', |
| 132 | + '## Complete script', |
| 133 | + '', |
| 134 | + `\`\`\`ser title="${entry.name}"\n${script}\n\`\`\``, |
| 135 | + '', |
| 136 | + 'This file is compiled during the SER build and is safe to use as a current-syntax learning example.', |
| 137 | + '', |
| 138 | + ].join('\n'), |
| 139 | + }); |
| 140 | + existingPagePaths.add(relativeFilename); |
| 141 | +} |
| 142 | + |
| 143 | +const pages = pageSources |
| 144 | + .map(({filename, relativeFilename, canonicalUrlOverride, markdownOverride}) => { |
| 145 | + const markdown = markdownOverride ?? fs.readFileSync(filename, 'utf8'); |
| 146 | + const {attributes, body} = parseFrontMatter(markdown); |
| 147 | + return { |
| 148 | + filename, |
| 149 | + relativeFilename, |
| 150 | + markdown, |
| 151 | + body, |
| 152 | + title: titleFrom(relativeFilename, attributes, body), |
| 153 | + canonicalUrl: canonicalUrlOverride || canonicalUrl(relativeFilename, attributes), |
| 154 | + markdownUrl: markdownUrl(relativeFilename), |
| 155 | + section: sectionFor(relativeFilename), |
| 156 | + }; |
| 157 | + }) |
| 158 | + .sort((left, right) => left.relativeFilename.localeCompare(right.relativeFilename)); |
| 159 | + |
| 160 | +fs.rmSync(markdownDirectory, {recursive: true, force: true}); |
| 161 | +for (const page of pages) { |
| 162 | + const target = path.join(markdownDirectory, ...page.relativeFilename.split('/')); |
| 163 | + fs.mkdirSync(path.dirname(target), {recursive: true}); |
| 164 | + fs.writeFileSync(target, page.markdown); |
| 165 | +} |
| 166 | + |
| 167 | +const sectionOrder = [ |
| 168 | + 'Learn and guides', |
| 169 | + 'Language reference', |
| 170 | + 'Generated reference', |
| 171 | + 'Build-validated examples', |
| 172 | + 'Developer documentation', |
| 173 | + 'Additional documentation', |
| 174 | +]; |
| 175 | +const indexParts = [ |
| 176 | + '# ScriptedEventsReloaded documentation', |
| 177 | + '', |
| 178 | + '> Stable, machine-readable documentation generated from the same sources as the public SER documentation site.', |
| 179 | + '', |
| 180 | + '## Complete corpus and structured data', |
| 181 | + '', |
| 182 | + `- [Full documentation corpus](${new URL('llms-full.txt', publicBaseUrl).href})`, |
| 183 | + `- [SER truth table](${new URL('data/ser-truth-table.json', publicBaseUrl).href})`, |
| 184 | +]; |
| 185 | + |
| 186 | +for (const section of sectionOrder) { |
| 187 | + const sectionPages = pages.filter(page => page.section === section); |
| 188 | + if (!sectionPages.length) continue; |
| 189 | + indexParts.push('', `## ${section}`, ''); |
| 190 | + for (const page of sectionPages) { |
| 191 | + indexParts.push(`- [${escapedLinkTitle(page.title)}](${page.markdownUrl})`); |
| 192 | + } |
| 193 | +} |
| 194 | + |
| 195 | +const fullParts = [ |
| 196 | + '# ScriptedEventsReloaded full documentation', |
| 197 | + '', |
| 198 | + '> Generated from the current SER tutorials, guides, language specification, reference catalog, and build-validated examples.', |
| 199 | +]; |
| 200 | +for (const page of pages) { |
| 201 | + fullParts.push( |
| 202 | + '', |
| 203 | + '---', |
| 204 | + '', |
| 205 | + `<!-- BEGIN DOCUMENT: ${page.relativeFilename} -->`, |
| 206 | + '', |
| 207 | + `Canonical source URL: [${page.canonicalUrl}](${page.canonicalUrl})`, |
| 208 | + '', |
| 209 | + `Markdown source: [${page.markdownUrl}](${page.markdownUrl})`, |
| 210 | + '', |
| 211 | + ); |
| 212 | + if (!/^#\s+/m.test(page.body)) fullParts.push(`# ${page.title}`, ''); |
| 213 | + fullParts.push(page.body.trimEnd(), '', `<!-- END DOCUMENT: ${page.relativeFilename} -->`); |
| 214 | +} |
| 215 | + |
| 216 | +fs.writeFileSync(path.join(staticDirectory, 'llms.txt'), `${indexParts.join('\n')}\n`); |
| 217 | +fs.writeFileSync(path.join(staticDirectory, 'llms-full.txt'), `${fullParts.join('\n')}\n`); |
| 218 | + |
| 219 | +JSON.parse(fs.readFileSync(truthTableSource, 'utf8')); |
| 220 | +const publicDataDirectory = path.join(staticDirectory, 'data'); |
| 221 | +fs.mkdirSync(publicDataDirectory, {recursive: true}); |
| 222 | +fs.copyFileSync(truthTableSource, path.join(publicDataDirectory, 'ser-truth-table.json')); |
| 223 | + |
| 224 | +console.log(`Generated llms.txt, llms-full.txt, ${pages.length} Markdown assets, and the SER truth table.`); |
0 commit comments