diff --git a/.github/workflows/validate-content.yaml b/.github/workflows/validate-content.yaml new file mode 100644 index 00000000..e74c00c5 --- /dev/null +++ b/.github/workflows/validate-content.yaml @@ -0,0 +1,53 @@ +name: Validate blog content + +# Runs the same checks the publishing tools apply, so a hand-written post is +# held to the same rules as a generated one. +# +# The site is built first: with a build present, link checking resolves against +# the paths the site actually serves instead of inferring them from the content +# tree. + +on: + pull_request: + paths: + - 'content/**' + - 'data/taxonomy.yaml' + - 'data/community-links.yaml' + - 'hack/mcp/**' + - '.github/workflows/validate-content.yaml' + +jobs: + validate: + runs-on: ubuntu-latest + env: + HUGO_VERSION: 0.164.0 + steps: + - name: Install Hugo CLI + run: | + wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ + && sudo dpkg -i ${{ runner.temp }}/hugo.deb + - name: Install Dart Sass + run: sudo snap install dart-sass + - name: Checkout + uses: actions/checkout@v7 + with: + submodules: recursive + fetch-depth: 0 + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: | + npm install + python3 -m pip install --quiet PyYAML Pillow + - name: Run publishing tool tests + run: python3 hack/mcp/test_mcp.py + - name: Build site + run: ./hack/download_openapi.sh && hugo --gc --minify + - name: Validate content + run: python3 hack/mcp/server.py --check diff --git a/.gitignore b/.gitignore index 55782a61..baa04e01 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,7 @@ static/docs/*/cozystack-api/ # Claude Code local settings .claude/ + +# Python bytecode from hack/ tooling +__pycache__/ +*.pyc diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 00000000..a37d8cfd --- /dev/null +++ b/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "cozystack-website": { + "command": "python3", + "args": ["hack/mcp/server.py"] + } + } +} diff --git a/README.md b/README.md index f8a87015..423b8ce0 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,24 @@ brew install hugo ```bash hugo serve ``` + +## Publishing tools + +`hack/mcp/` holds two tools for the blog: one that creates a post, one that +checks posts. They share a single implementation, so a generated post and a +hand-written one are held to the same rules — closed taxonomy vocabularies, +resolvable internal links, a present description, and an Open Graph card social +parsers can actually render. + +Check content before opening a pull request: + +```bash +python3 hack/mcp/server.py --check +``` + +The same command runs in CI. `.mcp.json` registers the tools as an MCP server, +so an MCP-capable client picks them up from a checkout with no separate +installation. + +See [`hack/mcp/README.md`](hack/mcp/README.md) for the tool reference, what the +checks cover, and what the tools deliberately leave alone. diff --git a/data/community-links.yaml b/data/community-links.yaml new file mode 100644 index 00000000..552e9b36 --- /dev/null +++ b/data/community-links.yaml @@ -0,0 +1,18 @@ +# The closing "Join the community" section appended to blog posts. +# +# Single source of truth: changing a link here changes it for every post +# published from now on, instead of requiring edits across published posts. +# Kept in data/ so templates can read it too if that ever becomes useful. + +heading: Join the community + +links: + - text: Cozystack on GitHub + url: https://github.com/cozystack/cozystack + - text: Telegram group + url: https://t.me/cozystack + - text: Slack group + url: https://kubernetes.slack.com/archives/C06L3CPRVN1 + note: "(get an invite at [slack.kubernetes.io](https://slack.kubernetes.io))" + - text: Community Meeting Calendar + url: https://calendar.google.com/calendar?cid=ZTQzZDIxZTVjOWI0NWE5NWYyOGM1ZDY0OWMyY2IxZTFmNDMzZTJlNjUzYjU2ZGJiZGE3NGNhMzA2ZjBkMGY2OEBncm91cC5jYWxlbmRhci5nb29nbGUuY29t diff --git a/hack/mcp/README.md b/hack/mcp/README.md new file mode 100644 index 00000000..8c0f4964 --- /dev/null +++ b/hack/mcp/README.md @@ -0,0 +1,118 @@ +# Publishing tools + +Two tools for publishing to the blog: one that creates a post, one that checks +posts. They share a single implementation, so a generated post and a +hand-written one are held to the same rules. + +Writing a markdown file into the right directory is not the hard part. Keeping +every post consistent with rules that live scattered across documentation, +templates and habit is. Each check here exists because the mistake it prevents +has already been made in this repository. + +## Running the checker + +```bash +python3 hack/mcp/server.py --check # every blog post +python3 hack/mcp/server.py --check --path content/en/blog/some-post.md +``` + +Exits non-zero when any post has an error, so it works as a CI step. + +Link checking has two modes. If `public/` holds a build, links are resolved +against it, which is exact — those are the paths the site actually serves, and +an unresolved link is an error. Without a build, URLs are inferred from the +content tree, which is only approximate: Hugo derives them through permalinks, +per-page aliases and version directories. In that mode an unresolved link is a +warning, so the checker never fails on its own guesswork. Build first for a +strict run: + +```bash +hugo --gc --minify && python3 hack/mcp/server.py --check +``` + +## Running the tests + +```bash +python3 hack/mcp/test_mcp.py +``` + +Each test builds a throwaway site in a temporary directory. Nothing touches the +real content tree. + +## Using it as an MCP server + +`.mcp.json` in the repository root registers the server, so an MCP-capable +client picks it up from a checkout with no separate installation. It speaks MCP +over stdio as line-delimited JSON-RPC. + +### publish_post + +Creates a post from markdown: writes a page bundle when images are supplied and +a plain file otherwise, copies the images beside the markdown, appends the +standard community section, validates the result, and commits to a branch. +Validation runs before anything is written — and if a later step fails, whatever +was created is removed, so a failed publish never leaves debris behind. + +Required: `title`, `description`, `author`, `body`, `article_types`, `topics`. +Optional: `images`, `doc_links`, `slug`, `date`, `branch`, `commit`. + +Metadata is expected ready-made. Turning a Google Doc or a raw draft into +markdown and choosing sensible taxonomy terms is the calling agent's job; this +server only lays the result out correctly and refuses what breaks the rules. + +Images are copied unchanged. Resizing and AVIF or WebP conversion belong to +Hugo, which processes bundle resources natively and encodes AVIF as of 0.162 — +there is no reason to keep a second implementation of that here. + +### validate + +The same checks with nothing written. Pass `path` for a single post, omit it for +the whole blog. + +## What the checks cover + +**Markdown only.** `.html` content is refused. Hugo denies `text/html` content +by default, as the fix for an XSS vulnerability, and this repository carries no +such files any more; one added by hand would break the build again. + +**Taxonomy.** Terms must come from `data/taxonomy.yaml`, and the two axes must +stay separate — a genre in `topics` or a subject in `article_types` is an error. +The vocabularies are closed on purpose: a term invented while writing produces a +taxonomy page with one entry, which reads as thin content. One post once carried +an image filename among its topics, which is what an unchecked list eventually +yields. + +**Structure.** `slug` matches the bundle directory, the date in the directory +matches the front matter, posts with local images live in a bundle, and a bundle +without assets is flagged as pointless. + +**Open Graph card.** The first entry in `images` must exist, be raster, and be +roughly 1200×630. SVG, AVIF and WebP are refused for the card specifically: +Telegram, LinkedIn and other parsers do not render them in `og:image`, and the +Telegram preview is the reason the card exists. AVIF and WebP remain fine in the +article body. + +**Description.** Required, since it feeds both the meta description and the +JSON-LD `BlogPosting`. An empty one yields an empty field in structured data and +no useful search snippet. + +**Links.** Internal links must resolve, and a link pinning a docs version other +than the current one is flagged as something that will age out. Links into +`/docs/next/` are refused — that trunk is excluded from production builds. + +## What these tools deliberately do not do + +**Generate meta tags.** The SEO and structured-data setup already lives in +`layouts/partials/hooks/head-end.html`: canonical URLs, `noindex` for superseded +docs versions, JSON-LD for the organization, the site and every blog post, plus +`robots.txt`, `llms.txt` and Open Graph tags from Docsy. Emitting any of that +here would only conflict with it. The job is to guarantee the quality of the +fields those templates read. + +**Touch documentation.** `content/*/docs/**` is out of scope. Versioning, the +`next/` trunk and pages generated from upstream belong to the release pipeline +in `cozystack/cozystack`. + +**Touch translations.** The localization pipeline has its own review gates. + +**Parse arbitrary formats.** Input is markdown. diff --git a/hack/mcp/core.py b/hack/mcp/core.py new file mode 100644 index 00000000..308767bd --- /dev/null +++ b/hack/mcp/core.py @@ -0,0 +1,291 @@ +"""The single write path. + +Every write goes through publish(): assemble in memory, validate, write, and +roll back if anything fails. A tool that errors out or is interrupted leaves +the repository exactly as it was, so a failed publish never needs manual +cleanup. +""" + +from __future__ import annotations + +import datetime as dt +import re +import shutil +import subprocess +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +import frontmatter +import validate + +BLOG_DIR = Path("content") / "en" / "blog" + +# Community links live in one file so that changing one does not mean editing +# thirty published posts after the fact. +COMMUNITY_DATA = Path("data") / "community-links.yaml" + +COMMUNITY_HEADING = "Join the community" + +SLUG_RE = re.compile(r"[^a-z0-9]+") + + +class PublishError(Exception): + """Raised when a publish cannot proceed. Nothing has been written.""" + + +@dataclass +class PublishResult: + path: Path + slug: str + branch: str | None = None + commit: str | None = None + warnings: list[str] = field(default_factory=list) + copied_images: list[str] = field(default_factory=list) + + +def slugify(title: str) -> str: + return SLUG_RE.sub("-", title.lower()).strip("-") + + +def community_block(root: Path) -> str: + """Render the closing community section from data. + + Returns an empty string when the data file is absent, so that a repository + without it still publishes rather than failing on a cosmetic section. + """ + path = root / COMMUNITY_DATA + if not path.exists(): + return "" + + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + links = data.get("links") or [] + if not links: + return "" + + lines = [f"## {data.get('heading', COMMUNITY_HEADING)}", ""] + for item in links: + text = item.get("text", "") + url = item.get("url", "") + note = item.get("note", "") + entry = f"- [{text}]({url})" if url else f"- {text}" + if note: + entry += f" {note}" + lines.append(entry) + return "\n".join(lines) + "\n" + + +def documentation_block(links: list[dict]) -> str: + """Render the optional documentation section.""" + if not links: + return "" + lines = ["## Documentation", ""] + for item in links: + lines.append(f"- [{item.get('text', '')}]({item.get('url', '')})") + return "\n".join(lines) + "\n" + + +def assemble_body( + root: Path, body: str, doc_links: list[dict] | None = None +) -> str: + """Append the standard closing sections to an article body.""" + parts = [body.rstrip("\n")] + + docs = documentation_block(doc_links or []) + if docs and "## Documentation" not in body: + parts.append(docs.rstrip("\n")) + + community = community_block(root) + if community and COMMUNITY_HEADING not in body: + parts.append(community.rstrip("\n")) + + return "\n\n".join(parts) + "\n" + + +def publish( + root: Path, + title: str, + description: str, + author: str, + body: str, + article_types: list[str], + topics: list[str], + images: list[str] | None = None, + slug: str | None = None, + date: str | None = None, + doc_links: list[dict] | None = None, + branch: str | None = None, + commit: bool = True, +) -> PublishResult: + """Create a blog post. Validates before writing and rolls back on failure. + + images are paths on disk; the first one becomes the Open Graph card. They + are copied into the bundle unchanged: resizing and format conversion are + left to Hugo, which processes bundle resources natively and, since 0.162, + encodes AVIF. + """ + images = images or [] + slug = slug or slugify(title) + date = date or dt.date.today().isoformat() + + if not re.match(r"^\d{4}-\d{2}-\d{2}$", date): + raise PublishError(f"date must be YYYY-MM-DD, got '{date}'") + if not slug: + raise PublishError("could not derive a slug; pass one explicitly") + + site = validate.Site(root) + _reject_unknown_terms(site, article_types, topics) + + is_bundle = bool(images) + if is_bundle: + target_dir = root / BLOG_DIR / f"{date}-{slug}" + target = target_dir / "index.md" + else: + target_dir = None + target = root / BLOG_DIR / f"{date}-{slug}.md" + + if target.exists(): + raise PublishError(f"{target.relative_to(root)} already exists") + if target_dir is not None and target_dir.exists(): + raise PublishError(f"{target_dir.relative_to(root)} already exists") + + data = { + "title": title, + "slug": slug, + "date": date, + "author": author, + "description": description, + "article_types": list(article_types), + "topics": list(topics), + } + if images: + data["images"] = [Path(images[0]).name] + + content = frontmatter.dump(data, assemble_body(root, body, doc_links)) + + created: list[Path] = [] + try: + if target_dir is not None: + target_dir.mkdir(parents=True) + created.append(target_dir) + target.write_text(content, encoding="utf-8") + created.append(target) + + copied = [] + for src in images: + source = Path(src).expanduser() + if not source.exists(): + raise PublishError(f"image not found: {source}") + dest = target.parent / source.name + shutil.copy2(source, dest) + created.append(dest) + copied.append(source.name) + + report = validate.validate_post(target, validate.Site(root)) + if not report.ok: + raise PublishError( + "validation failed:\n" + "\n".join(f" - {e}" for e in report.errors) + ) + + result = PublishResult( + path=target.relative_to(root), + slug=slug, + warnings=report.warnings, + copied_images=copied, + ) + + if commit: + result.branch, result.commit = _commit( + root, target, target_dir, branch or f"blog/{slug}", title + ) + + return result + + except Exception: + _rollback(created) + raise + + +def _reject_unknown_terms( + site: validate.Site, article_types: list[str], topics: list[str] +) -> None: + """Fail before touching disk when taxonomy terms are wrong.""" + if not site.article_types and not site.topics: + return + + problems = [] + for term in article_types: + if term in site.topics: + problems.append(f"'{term}' is a subject; it belongs in topics") + elif term not in site.article_types: + problems.append( + f"'{term}' is not a known article type " + f"({', '.join(sorted(site.article_types))})" + ) + for term in topics: + if term in site.article_types: + problems.append(f"'{term}' is a genre; it belongs in article_types") + elif term not in site.topics: + problems.append( + f"'{term}' is not a known topic; add it to data/taxonomy.yaml " + "if the subject genuinely recurs" + ) + if problems: + raise PublishError("\n".join(f" - {p}" for p in problems)) + + +def _rollback(created: list[Path]) -> None: + """Undo whatever the failed publish managed to create.""" + for path in reversed(created): + try: + if path.is_dir(): + shutil.rmtree(path) + elif path.exists(): + path.unlink() + except OSError: + # Nothing useful to do here; the exception being handled upstream + # is the one worth reporting. + pass + + +def _git(root: Path, *args: str) -> str: + proc = subprocess.run( + ["git", *args], + cwd=root, + capture_output=True, + text=True, + ) + if proc.returncode != 0: + raise PublishError(f"git {' '.join(args)} failed: {proc.stderr.strip()}") + return proc.stdout.strip() + + +def _commit( + root: Path, + target: Path, + target_dir: Path | None, + branch: str, + title: str, +) -> tuple[str, str]: + """Put the new post on its own branch and commit it. + + Never commits onto the default branch: blog posts arrive through pull + requests. + """ + current = _git(root, "rev-parse", "--abbrev-ref", "HEAD") + if current in ("main", "master"): + _git(root, "checkout", "-b", branch) + else: + branch = current + + paths = [str((target_dir or target).relative_to(root))] + _git(root, "add", *paths) + _git( + root, + "commit", + "--signoff", + "-m", + f"feat(blog): {title}", + ) + return branch, _git(root, "rev-parse", "--short", "HEAD") diff --git a/hack/mcp/frontmatter.py b/hack/mcp/frontmatter.py new file mode 100644 index 00000000..f7616907 --- /dev/null +++ b/hack/mcp/frontmatter.py @@ -0,0 +1,115 @@ +"""Reading and writing YAML front matter of Hugo content files. + +Kept deliberately small: the publishing tools need to inspect a handful of +keys and to emit front matter in the shape the existing blog posts use, not +to model everything Hugo accepts. +""" + +from __future__ import annotations + +import yaml + +DELIMITER = "---" + +# Key order used when writing front matter, mirroring the existing blog posts. +# Keys not listed here follow, alphabetically. +KEY_ORDER = [ + "title", + "slug", + "date", + "author", + "description", + "images", + "article_types", + "topics", +] + + +class FrontMatterError(Exception): + """Raised when a content file has no parseable front matter.""" + + +def split(text: str) -> tuple[str, str]: + """Split a content file into its raw front matter and body. + + Returns (front_matter_text, body). Raises FrontMatterError when the file + does not open with a delimiter or the closing delimiter is missing. + """ + if not text.startswith(DELIMITER): + raise FrontMatterError("file does not start with '---'") + + # Search for the closing delimiter on its own line. + rest = text[len(DELIMITER) :] + marker = f"\n{DELIMITER}" + end = rest.find(marker) + if end == -1: + raise FrontMatterError("closing '---' not found") + + fm = rest[:end] + body = rest[end + len(marker) :] + return fm.lstrip("\n"), body.lstrip("\n") + + +def load(text: str) -> tuple[dict, str]: + """Parse a content file into (front matter mapping, body).""" + fm_text, body = split(text) + try: + data = yaml.safe_load(fm_text) or {} + except yaml.YAMLError as exc: + raise FrontMatterError(f"front matter is not valid YAML: {exc}") from exc + if not isinstance(data, dict): + raise FrontMatterError("front matter is not a mapping") + return data, body + + +class _Dumper(yaml.SafeDumper): + """Indents sequences, so lists match the style of the existing posts.""" + + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + + +# Written unquoted so Hugo parses them as dates rather than strings. +_UNQUOTED_KEYS = {"date"} + + +def dump(data: dict, body: str) -> str: + """Render front matter and body back into a content file. + + The output is shaped to match the hand-written posts: keys in the usual + order, double-quoted scalars, indented lists, and no line wrapping. That + last one matters beyond aesthetics — a wrapped title reaches the templates + with the fold in it, which then shows up in og:title and in the JSON-LD + headline. + """ + ordered = {} + for key in KEY_ORDER: + if key in data: + ordered[key] = data[key] + for key in sorted(data): + if key not in ordered: + ordered[key] = data[key] + + lines = [] + for key, value in ordered.items(): + if isinstance(value, list): + lines.append(f"{key}:") + for item in value: + lines.append(f" - {_scalar(item)}") + elif key in _UNQUOTED_KEYS: + lines.append(f"{key}: {value}") + else: + lines.append(f"{key}: {_scalar(value)}") + + fm = "\n".join(lines) + body = body.rstrip("\n") + return f"{DELIMITER}\n{fm}\n{DELIMITER}\n\n{body}\n" + + +def _scalar(value) -> str: + """Render one scalar as a double-quoted YAML string on a single line.""" + if isinstance(value, bool) or value is None or isinstance(value, (int, float)): + return yaml.safe_dump(value, default_flow_style=True).strip().rstrip("...").strip() + text = str(value) + escaped = text.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' diff --git a/hack/mcp/server.py b/hack/mcp/server.py new file mode 100644 index 00000000..817338f6 --- /dev/null +++ b/hack/mcp/server.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""MCP server for publishing to cozystack.io. + +Speaks MCP over stdio as line-delimited JSON-RPC 2.0. The protocol is small +enough to implement directly, which keeps this in line with the rest of hack/: +no dependency beyond PyYAML, and Pillow only for reading image dimensions. + +Two tools: + + publish_post create a blog post from markdown plus images, validate it, + and commit it to a branch + validate run the same checks without writing anything, over one post + or the whole blog + +Run it directly to talk MCP on stdin/stdout, or with --check to run the +validator as a CI job. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import core # noqa: E402 +import validate # noqa: E402 + +PROTOCOL_VERSION = "2025-06-18" +SERVER_NAME = "cozystack-website-publish" +SERVER_VERSION = "0.1.0" + + +def repo_root() -> Path: + """The repository this server operates on: two levels up from hack/mcp.""" + return Path(__file__).resolve().parent.parent.parent + + +TOOLS = [ + { + "name": "publish_post", + "description": ( + "Create a blog post on cozystack.io from markdown. Writes the file " + "(a page bundle when images are given, a plain file otherwise), " + "copies the images beside it, appends the standard community " + "section, validates the result and commits it to a branch. " + "Refuses to write anything if validation fails. Images are copied " + "unchanged: resizing and AVIF/WebP conversion are Hugo's job. The " + "first image becomes the Open Graph card and must be PNG or JPEG " + "near 1200x630, because social parsers do not render SVG, AVIF or " + "WebP as previews." + ), + "inputSchema": { + "type": "object", + "properties": { + "title": {"type": "string", "description": "Post title"}, + "description": { + "type": "string", + "description": ( + "Meta description, also used in JSON-LD BlogPosting. " + "Aim for 50-200 characters." + ), + }, + "author": {"type": "string", "description": "Author name"}, + "body": { + "type": "string", + "description": ( + "Article body in markdown, without front matter and " + "without the community section" + ), + }, + "article_types": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Genre terms from data/taxonomy.yaml: announcement, " + "case, how-to, news, release, tech-article" + ), + }, + "topics": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Subject terms from data/taxonomy.yaml, e.g. platform, " + "kubernetes, storage, security" + ), + }, + "images": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Paths to images on disk. The first is the Open Graph " + "card. Omit for a post without illustrations." + ), + }, + "doc_links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "url": {"type": "string"}, + }, + }, + "description": ( + "Optional links for a Documentation section. Each URL " + "is checked against the content tree." + ), + }, + "slug": { + "type": "string", + "description": "Override the slug derived from the title", + }, + "date": { + "type": "string", + "description": "Publication date as YYYY-MM-DD, defaults to today", + }, + "branch": { + "type": "string", + "description": "Branch name, defaults to blog/", + }, + "commit": { + "type": "boolean", + "description": "Commit the post. Default true.", + }, + }, + "required": [ + "title", + "description", + "author", + "body", + "article_types", + "topics", + ], + }, + }, + { + "name": "validate", + "description": ( + "Run the publishing checks without writing anything. Given a path, " + "checks that one post; given nothing, checks every post in the " + "blog. Verifies front matter, that taxonomy terms come from the " + "vocabularies and stay on their own axis, that internal links " + "resolve to real pages, that the description is present and the " + "Open Graph card is a raster image of roughly the right shape." + ), + "inputSchema": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": ( + "Repository-relative path to one post. Omit to check " + "the whole blog." + ), + } + }, + }, + }, +] + + +def tool_publish_post(root: Path, args: dict) -> str: + result = core.publish( + root=root, + title=args["title"], + description=args["description"], + author=args["author"], + body=args["body"], + article_types=args["article_types"], + topics=args["topics"], + images=args.get("images"), + slug=args.get("slug"), + date=args.get("date"), + doc_links=args.get("doc_links"), + branch=args.get("branch"), + commit=args.get("commit", True), + ) + + lines = [f"Published {result.path}"] + if result.copied_images: + lines.append(f"Images: {', '.join(result.copied_images)}") + if result.branch: + lines.append(f"Branch: {result.branch} ({result.commit})") + if result.warnings: + lines.append("") + lines.append("Warnings:") + lines.extend(f" - {w}" for w in result.warnings) + return "\n".join(lines) + + +def tool_validate(root: Path, args: dict) -> str: + site = validate.Site(root) + rel = args.get("path") + + if rel: + path = (root / rel).resolve() + if not path.exists(): + return f"{rel}: no such file" + reports = {rel: validate.validate_post(path, site)} + else: + reports = validate.validate_tree(site) + + if not reports: + return "No posts found." + + errors = {p: r for p, r in reports.items() if r.errors} + warned = {p: r for p, r in reports.items() if r.warnings and not r.errors} + + lines = [] + for path, report in errors.items(): + lines.append(f"FAIL {path}") + lines.extend(f" error: {e}" for e in report.errors) + lines.extend(f" warn: {w}" for w in report.warnings) + for path, report in warned.items(): + lines.append(f"WARN {path}") + lines.extend(f" warn: {w}" for w in report.warnings) + + summary = ( + f"{len(reports)} post(s): {len(errors)} with errors, " + f"{len(warned)} with warnings only" + ) + if lines: + return summary + "\n\n" + "\n".join(lines) + return summary + "\nAll checks passed." + + +HANDLERS = { + "publish_post": tool_publish_post, + "validate": tool_validate, +} + + +def handle(request: dict, root: Path) -> dict | None: + """Handle one JSON-RPC request. Returns None for notifications.""" + method = request.get("method") + request_id = request.get("id") + + if method == "initialize": + return _result( + request_id, + { + "protocolVersion": PROTOCOL_VERSION, + "capabilities": {"tools": {}}, + "serverInfo": {"name": SERVER_NAME, "version": SERVER_VERSION}, + }, + ) + + if method in ("notifications/initialized", "initialized"): + return None + + if method == "ping": + return _result(request_id, {}) + + if method == "tools/list": + return _result(request_id, {"tools": TOOLS}) + + if method == "tools/call": + params = request.get("params") or {} + name = params.get("name") + handler = HANDLERS.get(name) + if handler is None: + return _error(request_id, -32602, f"unknown tool: {name}") + try: + text = handler(root, params.get("arguments") or {}) + return _result( + request_id, {"content": [{"type": "text", "text": text}]} + ) + except Exception as exc: + # Tool failures are reported in-band so the caller can react, + # rather than as protocol errors. + return _result( + request_id, + { + "content": [{"type": "text", "text": f"{type(exc).__name__}: {exc}"}], + "isError": True, + }, + ) + + if request_id is None: + return None + return _error(request_id, -32601, f"unknown method: {method}") + + +def _result(request_id, payload: dict) -> dict: + return {"jsonrpc": "2.0", "id": request_id, "result": payload} + + +def _error(request_id, code: int, message: str) -> dict: + return { + "jsonrpc": "2.0", + "id": request_id, + "error": {"code": code, "message": message}, + } + + +def serve(root: Path, stdin=sys.stdin, stdout=sys.stdout) -> None: + for line in stdin: + line = line.strip() + if not line: + continue + try: + request = json.loads(line) + except json.JSONDecodeError: + continue + response = handle(request, root) + if response is not None: + stdout.write(json.dumps(response) + "\n") + stdout.flush() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="run the validator over the whole blog and exit non-zero on errors", + ) + parser.add_argument( + "--path", help="with --check, validate a single post instead" + ) + args = parser.parse_args() + + root = repo_root() + + if args.check: + output = tool_validate(root, {"path": args.path} if args.path else {}) + print(output) + return 1 if "FAIL" in output else 0 + + serve(root) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hack/mcp/test_mcp.py b/hack/mcp/test_mcp.py new file mode 100644 index 00000000..6a137e0f --- /dev/null +++ b/hack/mcp/test_mcp.py @@ -0,0 +1,548 @@ +#!/usr/bin/env python3 +"""Tests for the publishing tools. + +Run with: python3 hack/mcp/test_mcp.py + +Each test builds a throwaway site in a temporary directory, so nothing here +touches the real content tree. No test framework, matching the rest of hack/. +""" + +from __future__ import annotations + +import json +import io +import shutil +import sys +import tempfile +import textwrap +import traceback +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import core +import frontmatter +import server +import validate + +TAXONOMY = """ +article_types: + - announcement + - how-to + - news + - release +topics: + - kubernetes + - platform + - storage +""" + +COMMUNITY = """ +heading: Join the community +links: + - text: Cozystack on GitHub + url: https://github.com/cozystack/cozystack +""" + + +def make_site(tmp: Path, built: bool = True) -> Path: + """Build a minimal site: taxonomy, community links, one existing page. + + With built=True a public/ tree is created too, which puts link checking in + its exact mode. Without it, unresolved links are warnings rather than + errors, and that path is covered by its own test. + """ + (tmp / "data").mkdir(parents=True) + (tmp / "data" / "taxonomy.yaml").write_text(TAXONOMY, encoding="utf-8") + (tmp / "data" / "community-links.yaml").write_text(COMMUNITY, encoding="utf-8") + (tmp / "hugo.yaml").write_text(' latest_version_id: "v1.6"\n', encoding="utf-8") + + blog = tmp / "content" / "en" / "blog" + blog.mkdir(parents=True) + + docs = tmp / "content" / "en" / "docs" / "v1.6" / "storage" + docs.mkdir(parents=True) + (docs / "_index.md").write_text( + "---\ntitle: Storage\n---\n\nbody\n", encoding="utf-8" + ) + + if built: + served = tmp / "public" + (served).mkdir() + (served / "index.html").write_text("", encoding="utf-8") + page = served / "docs" / "v1.6" / "storage" + page.mkdir(parents=True) + (page / "index.html").write_text("", encoding="utf-8") + return tmp + + +def write_post(root: Path, name: str, front: str, body: str = "Text.") -> Path: + path = root / "content" / "en" / "blog" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"---\n{textwrap.dedent(front).strip()}\n---\n\n{body}\n", encoding="utf-8") + return path + + +VALID_FRONT = """ +title: A Post About Storage +slug: a-post +date: 2026-08-03 +author: Someone +description: A description long enough to be a useful search snippet for readers. +article_types: + - how-to +topics: + - storage +""" + + +# --- frontmatter ------------------------------------------------------------ + + +def test_frontmatter_roundtrip(): + text = "---\ntitle: Hi\ndate: 2026-01-01\n---\n\nBody here.\n" + data, body = frontmatter.load(text) + assert data["title"] == "Hi", data + assert body.strip() == "Body here.", body + + +def test_frontmatter_missing_delimiter(): + try: + frontmatter.load("no front matter here") + except frontmatter.FrontMatterError: + return + raise AssertionError("expected FrontMatterError") + + +def test_frontmatter_never_wraps_long_scalars(): + """A wrapped title reaches the templates with the fold in it, which then + shows up in og:title and in the JSON-LD headline.""" + long_title = ( + "Blockstor: a LINSTOR-compatible storage system for Kubernetes, " + "written from scratch in Go" + ) + out = frontmatter.dump({"title": long_title, "date": "2026-08-04"}, "body") + assert f'title: "{long_title}"' in out, out + # The date must stay unquoted so Hugo parses it as a date. + assert "date: 2026-08-04\n" in out, out + # And it must survive a round trip unchanged. + data, _ = frontmatter.load(out) + assert data["title"] == long_title, data + + +def test_frontmatter_indents_lists(): + out = frontmatter.dump({"topics": ["storage", "platform"]}, "body") + assert ' - "storage"' in out, out + data, _ = frontmatter.load(out) + assert data["topics"] == ["storage", "platform"], data + + +def test_frontmatter_key_order(): + out = frontmatter.dump({"topics": ["a"], "title": "T", "zzz": 1}, "body") + lines = [l for l in out.splitlines() if l and not l.startswith("-")] + assert lines[0].startswith("title:"), out + assert "topics:" in out and "zzz:" in out, out + + +# --- validators ------------------------------------------------------------- + + +def test_valid_post_passes(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + path = write_post(root, "2026-08-03-a-post.md", VALID_FRONT) + report = validate.validate_post(path, validate.Site(root)) + assert report.ok, report.errors + + +def test_html_content_rejected(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + path = root / "content" / "en" / "blog" / "post.html" + path.write_text("---\ntitle: X\n---\n\nbody\n", encoding="utf-8") + report = validate.validate_post(path, validate.Site(root)) + assert not report.ok, "html must be rejected" + assert "markdown" in report.errors[0], report.errors + + +def test_unknown_taxonomy_term_rejected(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + front = VALID_FRONT.replace("- storage", "- invented-topic") + path = write_post(root, "2026-08-03-a-post.md", front) + report = validate.validate_post(path, validate.Site(root)) + assert not report.ok, "unknown term must be rejected" + assert any("invented-topic" in e for e in report.errors), report.errors + + +def test_term_on_wrong_axis_rejected(): + """A genre in topics, or a subject in article_types, must be caught.""" + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + front = VALID_FRONT.replace("- storage", "- release") + path = write_post(root, "2026-08-03-a-post.md", front) + report = validate.validate_post(path, validate.Site(root)) + assert not report.ok, "genre in topics must be rejected" + assert any("genre" in e for e in report.errors), report.errors + + front2 = VALID_FRONT.replace("- how-to", "- storage") + path2 = write_post(root, "2026-08-03-b-post.md", front2) + report2 = validate.validate_post(path2, validate.Site(root)) + assert not report2.ok, "subject in article_types must be rejected" + assert any("subject" in e for e in report2.errors), report2.errors + + +def test_image_filename_in_topics_rejected(): + """The mistake that motivated the closed vocabulary.""" + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + front = VALID_FRONT.replace("- storage", '- "cozystack-v1.3.0.png"') + path = write_post(root, "2026-08-03-a-post.md", front) + report = validate.validate_post(path, validate.Site(root)) + assert not report.ok, "image filename must not pass as a topic" + + +def test_missing_description_rejected(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + front = "\n".join( + l for l in VALID_FRONT.strip().splitlines() if not l.startswith("description") + ) + path = write_post(root, "2026-08-03-a-post.md", front) + report = validate.validate_post(path, validate.Site(root)) + assert not report.ok, "missing description must be rejected" + + +def test_slug_directory_mismatch_rejected(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + bundle = root / "content" / "en" / "blog" / "2026-08-03-wrong-name" + bundle.mkdir(parents=True) + (bundle / "index.md").write_text( + f"---\n{VALID_FRONT.strip()}\n---\n\nText.\n", encoding="utf-8" + ) + (bundle / "card.png").write_bytes(b"x") + report = validate.validate_post(bundle / "index.md", validate.Site(root)) + assert not report.ok, "slug/directory mismatch must be rejected" + assert any("directory" in e for e in report.errors), report.errors + + +def test_broken_internal_link_rejected(): + """With a build present, an unserved link is an error.""" + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + body = "See [storage](https://cozystack.io/docs/v1/storage/) for details." + path = write_post(root, "2026-08-03-a-post.md", VALID_FRONT, body) + report = validate.validate_post(path, validate.Site(root)) + assert not report.ok, "link to a nonexistent page must be rejected" + assert any("not served" in e for e in report.errors), report.errors + + +def test_broken_link_is_warning_without_build(): + """Without a build the checker must not fail on its own inference.""" + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp), built=False) + body = "See [storage](/docs/v1/storage/) for details." + path = write_post(root, "2026-08-03-a-post.md", VALID_FRONT, body) + site = validate.Site(root) + assert not site.exact_links, "no build means approximate mode" + report = validate.validate_post(path, site) + assert report.ok, report.errors + assert any("could not be resolved" in w for w in report.warnings), report.warnings + + +def test_good_internal_link_passes(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + body = "See [storage](/docs/v1.6/storage/) for details." + path = write_post(root, "2026-08-03-a-post.md", VALID_FRONT, body) + report = validate.validate_post(path, validate.Site(root)) + assert report.ok, report.errors + + +def test_link_to_next_trunk_rejected(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + body = "See [storage](/docs/next/storage/)." + path = write_post(root, "2026-08-03-a-post.md", VALID_FRONT, body) + report = validate.validate_post(path, validate.Site(root)) + assert not report.ok, "link into the unreleased trunk must be rejected" + + +def test_svg_og_card_rejected(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + bundle = root / "content" / "en" / "blog" / "2026-08-03-a-post" + bundle.mkdir(parents=True) + front = VALID_FRONT.strip() + '\nimages:\n - "card.svg"' + (bundle / "index.md").write_text( + f"---\n{front}\n---\n\nText.\n", encoding="utf-8" + ) + (bundle / "card.svg").write_text("", encoding="utf-8") + report = validate.validate_post(bundle / "index.md", validate.Site(root)) + assert not report.ok, "SVG must not be accepted as an OG card" + assert any("SVG" in e for e in report.errors), report.errors + + +def test_missing_og_card_file_rejected(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + bundle = root / "content" / "en" / "blog" / "2026-08-03-a-post" + bundle.mkdir(parents=True) + front = VALID_FRONT.strip() + '\nimages:\n - "absent.png"' + (bundle / "index.md").write_text( + f"---\n{front}\n---\n\nText.\n", encoding="utf-8" + ) + report = validate.validate_post(bundle / "index.md", validate.Site(root)) + assert not report.ok, "declared card that does not exist must be rejected" + + +def test_plain_file_with_images_rejected(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + front = VALID_FRONT.strip() + '\nimages:\n - "card.png"' + path = write_post(root, "2026-08-03-a-post.md", front) + report = validate.validate_post(path, validate.Site(root)) + assert not report.ok, "images on a plain file must be rejected" + assert any("bundle" in e for e in report.errors), report.errors + + +# --- publishing ------------------------------------------------------------- + + +def publish_args(**overrides): + args = dict( + title="A Post About Storage", + description="A description long enough to be a useful search snippet.", + author="Someone", + body="Some text about storage.", + article_types=["how-to"], + topics=["storage"], + date="2026-08-03", + commit=False, + ) + args.update(overrides) + return args + + +def test_publish_plain_post(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + result = core.publish(root=root, **publish_args()) + path = root / result.path + assert path.exists(), result.path + text = path.read_text(encoding="utf-8") + assert "Join the community" in text, "community section must be appended" + assert "Cozystack on GitHub" in text, text + + +def test_publish_bundle_with_image(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + card = Path(tmp) / "card.png" + _write_png(card, 1200, 630) + result = core.publish(root=root, images=[str(card)], **publish_args()) + assert result.path.name == "index.md", result.path + assert (root / result.path.parent / "card.png").exists() + data, _ = frontmatter.load((root / result.path).read_text(encoding="utf-8")) + assert data["images"] == ["card.png"], data + + +def test_publish_rejects_unknown_term_before_writing(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + try: + core.publish(root=root, **publish_args(topics=["invented"])) + except core.PublishError: + blog = root / "content" / "en" / "blog" + assert not any(blog.iterdir()), "nothing must be written on failure" + return + raise AssertionError("expected PublishError") + + +def test_publish_rolls_back_on_missing_image(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + try: + core.publish(root=root, images=["/nonexistent/x.png"], **publish_args()) + except core.PublishError: + blog = root / "content" / "en" / "blog" + assert not any(blog.iterdir()), "bundle must be removed on failure" + return + raise AssertionError("expected PublishError") + + +def test_publish_rolls_back_on_validation_failure(): + """A body with a broken link must leave nothing behind.""" + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + try: + core.publish( + root=root, + **publish_args(body="See [x](/docs/v1.6/absent/)."), + ) + except core.PublishError: + blog = root / "content" / "en" / "blog" + assert not any(blog.iterdir()), "file must be removed on failure" + return + raise AssertionError("expected PublishError") + + +def test_publish_refuses_duplicate(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + core.publish(root=root, **publish_args()) + try: + core.publish(root=root, **publish_args()) + except core.PublishError: + return + raise AssertionError("expected PublishError on duplicate") + + +def test_community_section_not_duplicated(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + body = "Text.\n\n## Join the community\n\n- already here\n" + result = core.publish(root=root, **publish_args(body=body)) + text = (root / result.path).read_text(encoding="utf-8") + assert text.count("Join the community") == 1, text + + +def test_doc_links_validated(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + try: + core.publish( + root=root, + **publish_args( + doc_links=[{"text": "Absent", "url": "/docs/v1.6/absent/"}] + ), + ) + except core.PublishError: + return + raise AssertionError("expected PublishError for an unresolvable doc link") + + +def test_slugify(): + assert core.slugify("Hello, World!") == "hello-world" + assert core.slugify("Cozystack 1.6: What's New") == "cozystack-1-6-what-s-new" + + +# --- server protocol -------------------------------------------------------- + + +def test_initialize_and_tools_list(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + init = server.handle( + {"jsonrpc": "2.0", "id": 1, "method": "initialize"}, root + ) + assert init["result"]["serverInfo"]["name"] == server.SERVER_NAME, init + + listing = server.handle( + {"jsonrpc": "2.0", "id": 2, "method": "tools/list"}, root + ) + names = {t["name"] for t in listing["result"]["tools"]} + assert names == {"publish_post", "validate"}, names + + +def test_notification_returns_nothing(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + out = server.handle( + {"jsonrpc": "2.0", "method": "notifications/initialized"}, root + ) + assert out is None, out + + +def test_tool_error_reported_in_band(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + response = server.handle( + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": "publish_post", + "arguments": publish_args(topics=["invented"]), + }, + }, + root, + ) + assert response["result"].get("isError"), response + assert "invented" in response["result"]["content"][0]["text"] + + +def test_unknown_tool(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + response = server.handle( + { + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": {"name": "nope", "arguments": {}}, + }, + root, + ) + assert "error" in response, response + + +def test_serve_loop_roundtrip(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + stdin = io.StringIO( + json.dumps({"jsonrpc": "2.0", "id": 1, "method": "tools/list"}) + "\n" + ) + stdout = io.StringIO() + server.serve(root, stdin=stdin, stdout=stdout) + response = json.loads(stdout.getvalue().strip()) + assert response["id"] == 1, response + assert "tools" in response["result"], response + + +def test_validate_tool_over_tree(): + with tempfile.TemporaryDirectory() as tmp: + root = make_site(Path(tmp)) + write_post(root, "2026-08-03-good.md", VALID_FRONT) + write_post( + root, + "2026-08-03-bad.md", + VALID_FRONT.replace("- storage", "- invented"), + ) + out = server.tool_validate(root, {}) + assert "FAIL" in out, out + assert "2 post(s)" in out, out + + +# --- helpers ---------------------------------------------------------------- + + +def _write_png(path: Path, width: int, height: int) -> None: + from PIL import Image + + Image.new("RGB", (width, height), "white").save(path) + + +def main() -> int: + tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")] + failed = [] + for test in tests: + try: + test() + print(f" ok {test.__name__}") + except Exception: + failed.append(test.__name__) + print(f" FAIL {test.__name__}") + traceback.print_exc() + + print(f"\n{len(tests) - len(failed)}/{len(tests)} passed") + if failed: + print("failed: " + ", ".join(failed)) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hack/mcp/validate.py b/hack/mcp/validate.py new file mode 100644 index 00000000..501146c3 --- /dev/null +++ b/hack/mcp/validate.py @@ -0,0 +1,561 @@ +"""Validation rules for blog content. + +These rules are the point of the whole tool. Writing a markdown file into the +right directory is trivial; what is not trivial is keeping every post +consistent with rules that live in a mix of documentation, templates and +habit. Each rule here encodes one such rule, and each one exists because the +mistake it prevents has been made in this repository already. + +The same checks back the MCP server and the CI job, so a hand-written post and +a generated one are held to identical standards. +""" + +from __future__ import annotations + +import datetime as dt +import re +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +import frontmatter + +# Raster formats acceptable as an Open Graph card. AVIF and WebP are excluded +# on purpose: Telegram, LinkedIn and several other parsers do not render them +# in og:image, and the Telegram preview is the main reason the card exists. +OG_FORMATS = {".png", ".jpg", ".jpeg"} + +# Open Graph cards are expected to be close to this. Not enforced exactly — +# a slightly different crop is fine, a square avatar is not. +OG_TARGET = (1200, 630) +OG_TOLERANCE = 0.25 + +# Descriptions feed both the meta description and the JSON-LD BlogPosting. +# Search engines truncate well before 200 characters, and an empty value +# produces an empty field in structured data. +DESCRIPTION_MIN = 50 +DESCRIPTION_MAX = 200 + +BUNDLE_DIR_RE = re.compile(r"^(\d{4}-\d{2}-\d{2})-(.+)$") + +# Internal links we can resolve against the content tree. +INTERNAL_LINK_RE = re.compile( + r"\[[^\]]*\]\(\s*(?:https?://cozystack\.io)?(/[^)\s]*)" +) + +# Hugo strips the version segment from docs URLs at build time only for the +# configured versions; a link naming a version other than the current one goes +# stale at the next release. +DOCS_VERSION_RE = re.compile(r"^/docs/(v[\w.]+|next)/") + + +@dataclass +class Report: + """Outcome of a validation run.""" + + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + return not self.errors + + def error(self, message: str) -> None: + self.errors.append(message) + + def warn(self, message: str) -> None: + self.warnings.append(message) + + def merge(self, other: "Report") -> None: + self.errors.extend(other.errors) + self.warnings.extend(other.warnings) + + +class Site: + """The repository under validation. + + Holds the things every rule needs to consult: the taxonomy vocabularies, + the current docs version and the set of pages that exist. + + Link resolution has two modes. When a build output is present in public/ it + is used as the source of truth, which is exact — those are the paths the + site actually serves. Without it, resolution falls back to inferring URLs + from the content tree, which is only approximate: Hugo derives URLs through + permalinks, per-page aliases and version directories, and reproducing all of + that faithfully is not worth it. In the approximate mode an unresolved link + is reported as a warning rather than an error, so the checker never blocks + on its own guesswork. + """ + + def __init__(self, root: Path): + self.root = root + self._pages: set[str] | None = None + self._built: set[str] | None = None + taxonomy = self._load_taxonomy() + self.article_types: set[str] = set(taxonomy.get("article_types", [])) + self.topics: set[str] = set(taxonomy.get("topics", [])) + self.latest_version: str = self._load_latest_version() + + def _load_taxonomy(self) -> dict: + path = self.root / "data" / "taxonomy.yaml" + if not path.exists(): + return {} + return yaml.safe_load(path.read_text(encoding="utf-8")) or {} + + def _load_latest_version(self) -> str: + path = self.root / "hugo.yaml" + if not path.exists(): + return "" + # A targeted read rather than a full YAML parse: hugo.yaml is large and + # only this one key matters here. + for line in path.read_text(encoding="utf-8").splitlines(): + match = re.match(r"\s*latest_version_id:\s*\"?([\w.]+)\"?", line) + if match: + return match.group(1) + return "" + + @property + def pages(self) -> set[str]: + """Everything a link may legitimately point at, without leading slash. + + Covers three kinds of target, because a link in a post can address any + of them: + + - pages from the content tree, by file path; + - the same blog posts by their permalink, which is dated rather than + matching the directory name (``permalinks.blog`` in hugo.yaml); + - files under static/, which are copied to the site root verbatim. + """ + if self._pages is None: + self._pages = self._collect_pages() | self._collect_static() + return self._pages + + def _collect_pages(self) -> set[str]: + found: set[str] = set() + content = self.root / "content" + if not content.exists(): + return found + + for path in content.rglob("*.md"): + rel = path.relative_to(content) + parts = list(rel.parts) + if not parts: + continue + # content//... — the language segment is not part of the URL + # for the default language, and localized trees mirror the same + # structure, so it is dropped for resolution purposes. + parts = parts[1:] + if not parts: + continue + name = parts[-1] + if name in ("_index.md", "index.md"): + parts = parts[:-1] + else: + parts[-1] = name[: -len(".md")] + if not parts: + continue + found.add("/".join(parts)) + found.update(self._blog_permalinks(parts)) + found.update(self._aliases(path)) + return found + + @staticmethod + def _aliases(path: Path) -> set[str]: + """Alias URLs declared in a page's front matter. + + Sections have moved between docs versions — storage used to live under + operations/ — and the old URLs keep working through aliases. Ignoring + them makes live links look broken. + """ + try: + head = path.read_text(encoding="utf-8", errors="replace")[:4096] + except OSError: + return set() + if "aliases:" not in head: + return set() + + found: set[str] = set() + in_block = False + for line in head.splitlines(): + if re.match(r"^aliases:\s*$", line): + in_block = True + continue + if in_block: + item = re.match(r"^\s+-\s+(.+?)\s*$", line) + if item: + found.add(item.group(1).strip().strip("\"'").strip("/")) + continue + break + inline = re.match(r"^aliases:\s*\[(.+)\]\s*$", line) + if inline: + for raw in inline.group(1).split(","): + found.add(raw.strip().strip("\"'").strip("/")) + return found + + @staticmethod + def _blog_permalinks(parts: list[str]) -> set[str]: + """Map a blog path to the permalinks it is reachable at. + + permalinks.blog in hugo.yaml is /:section/:year/:month/:slug/ — there is + no day segment, so content/en/blog/2024-04-05-some-slug/ is served from + /blog/2024/04/some-slug/. Verified against the live site: the shape with + a day in it returns 404, so it is not accepted here. + """ + if len(parts) != 2 or parts[0] != "blog": + return set() + match = re.match(r"^(\d{4})-(\d{2})-(\d{2})-(.+)$", parts[1]) + if not match: + return set() + year, month, _day, slug = match.groups() + return {f"blog/{year}/{month}/{slug}"} + + def _collect_static(self) -> set[str]: + found: set[str] = set() + static = self.root / "static" + if not static.exists(): + return found + for path in static.rglob("*"): + if path.is_file(): + found.add(str(path.relative_to(static))) + return found + + @property + def built(self) -> set[str] | None: + """Paths served by a build in public/, or None when there is no build.""" + if self._built is None: + public = self.root / "public" + if not (public / "index.html").exists(): + return None + served: set[str] = set() + for path in public.rglob("*"): + if not path.is_file(): + continue + rel = path.relative_to(public) + served.add(str(rel)) + if rel.name == "index.html": + served.add(str(rel.parent).strip(".")) + self._built = served + return self._built + + @property + def exact_links(self) -> bool: + """Whether link resolution is exact, i.e. backed by a build.""" + return self.built is not None + + def resolves(self, url: str) -> bool: + """Whether a site-relative URL addresses something that exists. + + Beyond exact matches this accepts version-agnostic docs links such as + /docs/components/. Those serve the current version and return 200 on + the live site, so treating them as broken would be wrong — even though + no file sits at that literal path. + """ + candidate = url.strip("/") + if not candidate: + return True + + built = self.built + if built is not None: + return candidate in built or f"{candidate}/index.html" in built + + if candidate in self.pages: + return True + + if candidate.startswith("docs/") and not DOCS_VERSION_RE.match(url): + tail = candidate[len("docs/") :] + if self.latest_version: + if f"docs/{self.latest_version}/{tail}" in self.pages: + return True + # Fall back to any version providing the page: the unversioned URL + # is served from whichever version is current, and that moves. + suffix = f"/{tail}" + return any( + page.startswith("docs/") and page.endswith(suffix) + for page in self.pages + ) + + return False + + +def validate_post(path: Path, site: Site) -> Report: + """Validate a single blog post, given the path to its markdown file.""" + report = Report() + + if path.suffix != ".md": + report.error( + f"{path.name}: content must be markdown. Hugo denies text/html " + "content by default since it fixed the XSS in html content files, " + "and this repository carries no .html content any more" + ) + return report + + try: + data, body = frontmatter.load(path.read_text(encoding="utf-8")) + except frontmatter.FrontMatterError as exc: + report.error(f"{path.name}: {exc}") + return report + + report.merge(_check_structure(path, data)) + report.merge(_check_taxonomy(path, data, site)) + report.merge(_check_description(path, data)) + report.merge(_check_og_image(path, data)) + report.merge(_check_links(path, body, site)) + return report + + +def _check_structure(path: Path, data: dict) -> Report: + report = Report() + name = path.name + + for key in ("title", "date", "author"): + if not data.get(key): + report.error(f"{name}: '{key}' is required") + + slug = data.get("slug") + is_bundle = path.name == "index.md" + + if is_bundle: + match = BUNDLE_DIR_RE.match(path.parent.name) + if not match: + report.error( + f"{path.parent.name}: bundle directory must be named " + "YYYY-MM-DD-" + ) + else: + dir_date, dir_slug = match.groups() + if slug and slug != dir_slug: + report.error( + f"{path.parent.name}: 'slug' is '{slug}' but the directory " + f"says '{dir_slug}'" + ) + fm_date = data.get("date") + if isinstance(fm_date, dt.date): + fm_date = fm_date.isoformat() + if fm_date and str(fm_date)[:10] != dir_date: + report.error( + f"{path.parent.name}: 'date' is {fm_date} but the " + f"directory says {dir_date}" + ) + # A bundle exists to hold assets; one without any is a plain file + # wearing a costume. + assets = [ + p + for p in path.parent.iterdir() + if p.is_file() and p.name != "index.md" + ] + if not assets: + report.warn( + f"{path.parent.name}: page bundle holds no assets — a plain " + "markdown file would do" + ) + else: + # Only locally-hosted images require a bundle. Older posts point at a + # remote CDN, and there is nothing to sit beside the markdown then. + local = [ + str(i) + for i in (data.get("images") or []) + if not str(i).startswith(("http://", "https://")) + ] + if local: + report.error( + f"{name}: post declares local images but is a plain file; posts " + "with images belong in a page bundle so the assets sit beside " + "them" + ) + + return report + + +def _check_taxonomy(path: Path, data: dict, site: Site) -> Report: + report = Report() + name = path.name + + if not site.article_types and not site.topics: + report.warn( + "data/taxonomy.yaml not found — taxonomy terms cannot be checked" + ) + return report + + types = data.get("article_types") or [] + topics = data.get("topics") or [] + + if not types: + report.error(f"{name}: 'article_types' is required") + if not topics: + report.error(f"{name}: 'topics' is required") + + for term in types: + if term in site.topics: + report.error( + f"{name}: '{term}' is a subject, not a genre — it belongs in " + "'topics'" + ) + elif term not in site.article_types: + report.error( + f"{name}: '{term}' is not in the article_types vocabulary. " + f"Known: {', '.join(sorted(site.article_types))}" + ) + + for term in topics: + if term in site.article_types: + report.error( + f"{name}: '{term}' is a genre, not a subject — it belongs in " + "'article_types'" + ) + elif term not in site.topics: + report.error( + f"{name}: '{term}' is not in the topics vocabulary. Add it to " + "data/taxonomy.yaml if the subject genuinely recurs" + ) + + return report + + +def _check_description(path: Path, data: dict) -> Report: + report = Report() + name = path.name + description = (data.get("description") or "").strip() + + if not description: + report.error( + f"{name}: 'description' is required — it feeds both the meta " + "description and the JSON-LD BlogPosting" + ) + return report + + if len(description) < DESCRIPTION_MIN: + report.warn( + f"{name}: description is {len(description)} characters; under " + f"{DESCRIPTION_MIN} rarely earns a useful snippet" + ) + elif len(description) > DESCRIPTION_MAX: + report.warn( + f"{name}: description is {len(description)} characters; search " + f"results truncate well before {DESCRIPTION_MAX}" + ) + + return report + + +def _check_og_image(path: Path, data: dict) -> Report: + report = Report() + name = path.name + images = data.get("images") or [] + + if not images: + # Legitimate: the site default card is used instead. Worth saying out + # loud, because a post with a good illustration and no card gets a + # generic preview in Telegram and Slack. + report.warn( + f"{name}: no 'images' — social previews fall back to the site " + "default card" + ) + return report + + card = str(images[0]) + if card.startswith("http://") or card.startswith("https://"): + report.warn(f"{name}: Open Graph card is remote, not checked: {card}") + return report + + card_path = path.parent / card + if not card_path.exists(): + report.error(f"{name}: Open Graph card '{card}' does not exist") + return report + + suffix = card_path.suffix.lower() + if suffix == ".svg": + report.error( + f"{name}: '{card}' is SVG. Social parsers do not render SVG in " + "og:image — leave 'images' unset to fall back to the site default, " + "or add a raster card" + ) + return report + if suffix not in OG_FORMATS: + report.error( + f"{name}: '{card}' is {suffix}; an Open Graph card must be one of " + f"{', '.join(sorted(OG_FORMATS))}. AVIF and WebP are fine in the " + "article body but are not rendered as previews" + ) + return report + + report.merge(_check_og_dimensions(name, card, card_path)) + return report + + +def _check_og_dimensions(name: str, card: str, card_path: Path) -> Report: + report = Report() + try: + from PIL import Image + except ImportError: + report.warn(f"{name}: Pillow unavailable, '{card}' dimensions unchecked") + return report + + try: + with Image.open(card_path) as img: + width, height = img.size + except Exception as exc: # pragma: no cover - depends on broken files + report.error(f"{name}: cannot read '{card}': {exc}") + return report + + target_ratio = OG_TARGET[0] / OG_TARGET[1] + ratio = width / height if height else 0 + if abs(ratio - target_ratio) / target_ratio > OG_TOLERANCE: + report.warn( + f"{name}: '{card}' is {width}×{height}; Open Graph cards are " + f"expected near {OG_TARGET[0]}×{OG_TARGET[1]} and other shapes get " + "cropped unpredictably" + ) + return report + + +def _check_links(path: Path, body: str, site: Site) -> Report: + report = Report() + name = path.name + + for target in INTERNAL_LINK_RE.findall(body): + url = target.split("#")[0].split("?")[0] + if not url or url == "/": + continue + + version_match = DOCS_VERSION_RE.match(url) + if version_match: + version = version_match.group(1) + if version == "next": + report.error( + f"{name}: link to '{url}' points at the unreleased docs " + "trunk, which is excluded from production builds" + ) + continue + if site.latest_version and version != site.latest_version: + report.warn( + f"{name}: link to '{url}' pins docs version {version} " + f"while the current one is {site.latest_version}; it will " + "age out" + ) + + if not site.resolves(url): + if site.exact_links: + report.error(f"{name}: link to '{url}' is not served by the build") + else: + report.warn( + f"{name}: link to '{url}' could not be resolved from the " + "content tree. Build the site and re-run for an exact check" + ) + + return report + + +def validate_tree(site: Site, section: str = "blog") -> dict[str, Report]: + """Validate every post in a section. Returns path -> report.""" + results: dict[str, Report] = {} + base = site.root / "content" / "en" / section + if not base.exists(): + return results + + for path in sorted(base.rglob("*.md")): + if path.name.startswith("_"): + continue + rel = str(path.relative_to(site.root)) + results[rel] = validate_post(path, site) + return results