diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..03154b7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,90 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + checks: + name: Checks / Node ${{ matrix.node-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: + - 20.x + - 22.x + - 24.x + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: ${{ matrix.node-version }} + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Check formatting + run: pnpm format:check + + - name: Lint + run: pnpm lint + + - name: Typecheck + run: pnpm typecheck + + - name: Test + run: pnpm test + + - name: Build + run: pnpm build + + pandoc-integration: + name: Pandoc Integration + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: 22.x + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + + - name: Setup Pandoc + uses: pandoc/actions/setup@v1 + with: + version: 3.10.1 + + - name: Verify Pandoc + run: pandoc --version + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run Pandoc-backed tests + run: pnpm test + env: + POLYDOC_REQUIRE_PANDOC: "1" diff --git a/README.md b/README.md index 6adcc63..7968913 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,9 @@ taking on a CLI. The original Polydoc design work scoped a local-first Markdown-to-Word and Markdown-to-Google Docs workflow. This package keeps only the reusable library boundary from that work. CLI commands, project manifests, watch mode, OAuth user -experience, transports, and sidecar storage stay outside this repository. +experience, remote service integrations, and sidecar storage stay outside this +repository. The library does include the shared transport contract and a local +file transport for consumers that need deterministic DOCX writes. ## API @@ -45,6 +47,53 @@ console.log(SUPPORTED_PANDOC_MAJOR); // 3 write those bytes to disk, upload them to a transport, or pass them to another library. +## Transports + +Transports are side-effecting adapters that publish DOCX bytes for a canonical +document ID and return a stable destination handle for later consumers. + +```ts +import { + LocalFileTransport, + convertMarkdownToDocx, +} from "@agentic-tooling/polydoc-core"; + +const docx = await convertMarkdownToDocx({ + markdown, + referenceDocxPath: "./reference.docx", +}); + +const transport = new LocalFileTransport({ rootDir: "./generated-docx" }); +const destination = await transport.upload("teamwiki/basic-note", docx); + +console.log(destination.destinationId); // absolute path to generated-docx/teamwiki/basic-note.docx +console.log(destination.path); // same path, for local-file consumers +``` + +`LocalFileTransport` writes or overwrites one deterministic `.docx` destination +per canonical ID, creating parent directories as needed. By default, +`teamwiki/basic-note` maps to `/teamwiki/basic-note.docx`. A custom +`mapCanonicalId` option can map opaque IDs, such as `urn:teamwiki:note:123`, to +a relative destination under the same root. + +The local file transport validates paths lexically before writing: + +- Canonical IDs must be non-empty trimmed identifiers and must not contain NUL + bytes. +- Mapped destinations must be non-empty relative paths. +- Parent-directory segments, absolute paths, Windows drive syntax, backslashes, + colons, Windows-invalid filename characters, control characters, empty path + segments, Windows-reserved device names, path segments ending in dot or space, + and NUL bytes are rejected in mapped destinations. +- The resolved destination must stay under the configured `rootDir`. + +This boundary check prevents accidental lexical path escape. It does not resolve +symlinks or claim protection against a writable root that already contains +hostile symlinks. + +Transport failures throw `TransportError` with a stable `code`, actionable +`guidance`, and the original `cause` when filesystem or mapper operations fail. + ## Pandoc Contract This package shells out to the system `pandoc` binary through `execa` with an @@ -103,7 +152,7 @@ Pandoc binary/version are expected to produce identical bytes. ## Requirements - Node.js 20 or newer -- pnpm 11.9.0 +- pnpm 10.34.5 - Pandoc 3.x for conversion ## Development @@ -130,7 +179,14 @@ pnpm format:check ``` Pandoc integration tests are included in `pnpm test`. They skip cleanly when a -supported Pandoc binary is unavailable. +supported Pandoc binary is unavailable. CI has a dedicated Pandoc-backed job +that installs Pandoc 3.10.1 and requires the integration path to be available, +while the regular Node matrix can still run without Pandoc. + +GitHub Actions runs blocking checks for pushes to `main` and pull requests: +format check, lint, typecheck, tests, and build across supported Node majors +20.x, 22.x, and 24.x. Dependency installation uses the checked-in lockfile with +`pnpm install --frozen-lockfile`. Format files: diff --git a/package.json b/package.json index 730fee9..cafc47d 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "type": "module", "license": "MIT", "author": "Logan Lindquist Land", - "packageManager": "pnpm@11.9.0", + "packageManager": "pnpm@10.34.5", "engines": { "node": ">=20" }, @@ -45,7 +45,7 @@ "lint": "biome check .", "format": "biome format --write .", "format:check": "biome format .", - "check": "pnpm lint && pnpm typecheck && pnpm test && pnpm build", + "check": "biome check . && tsc -p tsconfig.json --noEmit && vitest run && tsc -p tsconfig.build.json", "prepack": "pnpm build" }, "devDependencies": { diff --git a/src/index.ts b/src/index.ts index fb7c736..0180476 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,3 +24,12 @@ export { PandocError, SUPPORTED_PANDOC_MAJOR, } from "./pandoc.js"; +export type { + LocalFileDestinationMapper, + LocalFileTransportDestination, + LocalFileTransportOptions, + Transport, + TransportErrorCode, + TransportUploadResult, +} from "./transport.js"; +export { LocalFileTransport, TransportError } from "./transport.js"; diff --git a/src/transport.ts b/src/transport.ts new file mode 100644 index 0000000..18c32e1 --- /dev/null +++ b/src/transport.ts @@ -0,0 +1,281 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve, sep, win32 } from "node:path"; + +export type TransportErrorCode = + | "TRANSPORT_ROOT_INVALID" + | "TRANSPORT_ID_INVALID" + | "TRANSPORT_DESTINATION_INVALID" + | "TRANSPORT_DESTINATION_OUTSIDE_ROOT" + | "TRANSPORT_WRITE_FAILED"; + +export interface TransportUploadResult { + readonly destinationId: string; +} + +export interface Transport { + upload(canonicalId: string, docx: Uint8Array): Promise; +} + +export interface LocalFileTransportDestination extends TransportUploadResult { + readonly kind: "local-file"; + readonly path: string; +} + +export type LocalFileDestinationMapper = (canonicalId: string) => string; + +export interface LocalFileTransportOptions { + readonly rootDir: string; + readonly mapCanonicalId?: LocalFileDestinationMapper; +} + +const WINDOWS_RESERVED_DEVICE_BASENAME = /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?$/i; +const WINDOWS_INVALID_FILENAME_CHARS = /[<>"|?*]/; + +export class TransportError extends Error { + readonly code: TransportErrorCode; + readonly guidance: string; + + constructor( + code: TransportErrorCode, + message: string, + guidance: string, + options: { readonly cause?: unknown } = {}, + ) { + const fullMessage = `${message} ${guidance}`; + + if ("cause" in options) { + super(fullMessage, { cause: options.cause }); + } else { + super(fullMessage); + } + + this.name = "TransportError"; + this.code = code; + this.guidance = guidance; + } +} + +export class LocalFileTransport implements Transport { + readonly rootDir: string; + readonly #mapCanonicalId: LocalFileDestinationMapper; + + constructor(options: LocalFileTransportOptions) { + if (options.rootDir.trim() === "") { + throw new TransportError( + "TRANSPORT_ROOT_INVALID", + "A local file transport rootDir is required.", + "Pass rootDir pointing at the directory where generated DOCX files may be written.", + ); + } + + this.rootDir = resolve(options.rootDir); + this.#mapCanonicalId = options.mapCanonicalId ?? ((canonicalId) => canonicalId); + } + + resolveDestination(canonicalId: string): LocalFileTransportDestination { + validateCanonicalId(canonicalId); + + const mappedDestination = mapCanonicalId(canonicalId, this.#mapCanonicalId); + const relativeDestination = normalizeRelativeDestination(mappedDestination); + const destinationPath = resolve(this.rootDir, ensureDocxExtension(relativeDestination)); + const relativeToRoot = relative(this.rootDir, destinationPath); + + if ( + relativeToRoot === "" || + relativeToRoot === ".." || + relativeToRoot.startsWith(`..${sep}`) || + isAbsolute(relativeToRoot) || + win32.isAbsolute(relativeToRoot) + ) { + throw new TransportError( + "TRANSPORT_DESTINATION_OUTSIDE_ROOT", + `Canonical ID ${JSON.stringify(canonicalId)} maps outside the local file transport root.`, + "Return a relative DOCX destination under rootDir from mapCanonicalId.", + ); + } + + return { + kind: "local-file", + destinationId: destinationPath, + path: destinationPath, + }; + } + + async upload(canonicalId: string, docx: Uint8Array): Promise { + const destination = this.resolveDestination(canonicalId); + const bytes = Buffer.from(docx); + + try { + await mkdir(dirname(destination.path), { recursive: true }); + await writeFile(destination.path, bytes); + } catch (cause) { + throw new TransportError( + "TRANSPORT_WRITE_FAILED", + `Failed to write DOCX for canonical ID ${JSON.stringify(canonicalId)}.`, + "Check that rootDir is writable and that the destination parent can be created.", + { cause }, + ); + } + + return destination; + } +} + +function mapCanonicalId(canonicalId: string, mapper: LocalFileDestinationMapper): string { + try { + const mappedDestination = mapper(canonicalId); + + if (typeof mappedDestination !== "string") { + throw new TypeError("Local file destination mapper must return a string."); + } + + return mappedDestination; + } catch (cause) { + if (cause instanceof TransportError) { + throw cause; + } + + throw new TransportError( + "TRANSPORT_DESTINATION_INVALID", + `Local file transport mapping failed for canonical ID ${JSON.stringify(canonicalId)}.`, + "Fix mapCanonicalId so it returns a safe relative DOCX destination.", + { cause }, + ); + } +} + +function validateCanonicalId(canonicalId: string): void { + if (canonicalId.trim() === "") { + throw new TransportError( + "TRANSPORT_ID_INVALID", + "A non-empty canonical ID is required for transport upload.", + "Pass the stable document ID used by the source system.", + ); + } + + if (canonicalId !== canonicalId.trim()) { + throw new TransportError( + "TRANSPORT_ID_INVALID", + "Canonical IDs must not include leading or trailing whitespace.", + "Normalize the document ID before uploading.", + ); + } + + if (canonicalId.includes("\0")) { + throw new TransportError( + "TRANSPORT_ID_INVALID", + "Canonical IDs must not contain NUL bytes.", + "Pass the stable document ID used by the source system.", + ); + } +} + +function normalizeRelativeDestination(destination: string): string { + if (destination.trim() === "") { + throw new TransportError( + "TRANSPORT_DESTINATION_INVALID", + "Local file transport mapping returned an empty destination.", + "Return a relative DOCX destination under rootDir from mapCanonicalId.", + ); + } + + if (destination !== destination.trim()) { + throw new TransportError( + "TRANSPORT_DESTINATION_INVALID", + "Local file transport destinations must not include leading or trailing whitespace.", + "Normalize mapped destinations before returning them from mapCanonicalId.", + ); + } + + validateRelativePathText( + destination, + "TRANSPORT_DESTINATION_INVALID", + "local file transport destination", + ); + + return destination; +} + +function validateRelativePathText( + value: string, + code: "TRANSPORT_ID_INVALID" | "TRANSPORT_DESTINATION_INVALID", + label: string, +): void { + if (value.includes("\0")) { + throw invalidRelativePathError(code, label, "must not contain NUL bytes"); + } + + if (containsWindowsInvalidControlCharacter(value)) { + throw invalidRelativePathError(code, label, "must not contain control characters"); + } + + if (value.includes("\\")) { + throw invalidRelativePathError(code, label, "must use forward slashes, not backslashes"); + } + + if (value.includes(":")) { + throw invalidRelativePathError(code, label, "must not contain colon characters"); + } + + if (WINDOWS_INVALID_FILENAME_CHARS.test(value)) { + throw invalidRelativePathError(code, label, 'must not contain < > " | ? or * characters'); + } + + if (value.startsWith("/") || value.startsWith("//") || win32.isAbsolute(value)) { + throw invalidRelativePathError(code, label, "must be a relative path"); + } + + for (const segment of value.split("/")) { + if (segment === "" || segment === "." || segment === "..") { + throw invalidRelativePathError( + code, + label, + "must not contain empty, current-directory, or parent-directory path segments", + ); + } + + if (segment.endsWith(".") || segment.endsWith(" ")) { + throw invalidRelativePathError( + code, + label, + "must not contain path segments ending in a dot or space", + ); + } + + if (WINDOWS_RESERVED_DEVICE_BASENAME.test(segment)) { + throw invalidRelativePathError( + code, + label, + "must not use Windows-reserved device names as path segments", + ); + } + } +} + +function invalidRelativePathError( + code: "TRANSPORT_ID_INVALID" | "TRANSPORT_DESTINATION_INVALID", + label: string, + reason: string, +): TransportError { + return new TransportError( + code, + `The ${label} ${reason}.`, + "Use a stable relative ID such as docs/handbook without traversal, platform separators, or drive syntax.", + ); +} + +function ensureDocxExtension(destination: string): string { + return destination.toLowerCase().endsWith(".docx") ? destination : `${destination}.docx`; +} + +function containsWindowsInvalidControlCharacter(value: string): boolean { + for (const character of value) { + const codePoint = character.codePointAt(0); + + if (codePoint !== undefined && codePoint <= 31) { + return true; + } + } + + return false; +} diff --git a/tests/index.test.ts b/tests/index.test.ts index fb9f8fe..0c9a030 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -1,6 +1,6 @@ -import { chmod, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { strFromU8, unzipSync } from "fflate"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -11,6 +11,7 @@ import { convertMarkdownToDocx, DEFAULT_SOURCE_DATE_EPOCH, doctor, + LocalFileTransport, PandocError, type PandocRunner, SUPPORTED_PANDOC_MAJOR, @@ -23,6 +24,7 @@ const fixtureReferenceDocxPath = "tests/fixtures/reference/reference.docx"; const realPandocProbe = await doctor(); const canRepresentUnreadableFiles = process.platform !== "win32" && process.getuid?.() !== 0; +const requirePandocIntegration = process.env.POLYDOC_REQUIRE_PANDOC === "1"; const tempDirectories: string[] = []; const unsupportedMajorOptionRegression: ConvertMarkdownToDocxOptions = { @@ -537,15 +539,226 @@ describe.skipIf(realPandocProbe.ok)("Pandoc integration skip behavior", () => { }); }); +describe.skipIf(!requirePandocIntegration)("Pandoc integration requirement", () => { + it("has a supported Pandoc binary when CI requires integration coverage", () => { + expect(realPandocProbe).toMatchObject({ ok: true }); + }); +}); + +describe("local file transport", () => { + it("creates a deterministic DOCX destination for a canonical ID", async () => { + const rootDir = await createTempDirectory(); + const transport = new LocalFileTransport({ rootDir }); + const result = await transport.upload("teamwiki/basic-note", new TextEncoder().encode("docx")); + + expect(result).toEqual({ + kind: "local-file", + destinationId: join(rootDir, "teamwiki", "basic-note.docx"), + path: join(rootDir, "teamwiki", "basic-note.docx"), + }); + expect(await readFile(result.path, "utf8")).toBe("docx"); + }); + + it("overwrites the same path instead of creating duplicate destinations", async () => { + const rootDir = await createTempDirectory(); + const transport = new LocalFileTransport({ rootDir }); + + const first = await transport.upload("notes/update", new TextEncoder().encode("first")); + const second = await transport.upload("notes/update", new TextEncoder().encode("second")); + + expect(second).toEqual(first); + expect(await readFile(first.path, "utf8")).toBe("second"); + }); + + it("resolves safe nested mapped destinations under the configured root", async () => { + const rootDir = await createTempDirectory(); + const transport = new LocalFileTransport({ + rootDir, + mapCanonicalId: (canonicalId) => `published/${canonicalId}/index`, + }); + + const result = transport.resolveDestination("handbook/intro"); + + expect(result).toEqual({ + kind: "local-file", + destinationId: join(rootDir, "published", "handbook", "intro", "index.docx"), + path: join(rootDir, "published", "handbook", "intro", "index.docx"), + }); + }); + + it("accepts opaque canonical IDs when a custom mapper returns a safe destination", async () => { + const rootDir = await createTempDirectory(); + const seenIds: string[] = []; + const transport = new LocalFileTransport({ + rootDir, + mapCanonicalId: (canonicalId) => { + seenIds.push(canonicalId); + return "opaque/teamwiki-note-123"; + }, + }); + + const result = await transport.upload( + "urn:teamwiki:note:123", + new TextEncoder().encode("docx"), + ); + + expect(seenIds).toEqual(["urn:teamwiki:note:123"]); + expect(result.path).toBe(join(rootDir, "opaque", "teamwiki-note-123.docx")); + expect(await readFile(result.path, "utf8")).toBe("docx"); + }); + + it("does not treat relative paths starting with '..' characters as parent traversal", async () => { + const rootDir = await createTempDirectory(); + const transport = new LocalFileTransport({ rootDir }); + + const result = await transport.upload("..safe/note", new TextEncoder().encode("docx")); + + expect(result.path).toBe(join(rootDir, "..safe", "note.docx")); + expect(await readFile(result.path, "utf8")).toBe("docx"); + }); + + it("preserves an explicit .docx extension from the mapped destination", async () => { + const rootDir = await createTempDirectory(); + const transport = new LocalFileTransport({ + rootDir, + mapCanonicalId: () => "exports/final.DOCX", + }); + + const result = await transport.upload("final", new TextEncoder().encode("docx")); + + expect(result.path).toBe(join(rootDir, "exports", "final.DOCX")); + }); + + it.each(["", " ", " leading", "trailing ", "bad\0id"])( + "rejects invalid canonical ID %j", + async (canonicalId) => { + const transport = new LocalFileTransport({ rootDir: await createTempDirectory() }); + + await expect(transport.upload(canonicalId, new Uint8Array())).rejects.toMatchObject({ + name: "TransportError", + code: "TRANSPORT_ID_INVALID", + }); + }, + ); + + it.each([ + "../secret", + "team/../secret", + "team//secret", + "./secret", + "/absolute", + "C:/absolute", + "C:relative", + String.raw`team\secret`, + ])("rejects default-mapped canonical ID %j as an unsafe destination", async (canonicalId) => { + const transport = new LocalFileTransport({ rootDir: await createTempDirectory() }); + + await expect(transport.upload(canonicalId, new Uint8Array())).rejects.toMatchObject({ + name: "TransportError", + code: "TRANSPORT_DESTINATION_INVALID", + }); + }); + + it.each([ + "../outside", + "safe/../../outside", + "/absolute", + "C:/absolute", + "C:relative", + String.raw`safe\outside`, + "safe//outside", + "bad\0destination", + "CON.docx", + "COM1", + "name?", + "name*", + "trailing.", + "nested/trailing ", + "AUX", + "LPT9.txt", + "control\u001Fchar", + ])("rejects unsafe mapped destination %j", (mappedDestination) => { + const transport = new LocalFileTransport({ + rootDir: resolve("/tmp/polydoc-root"), + mapCanonicalId: () => mappedDestination, + }); + + expect(() => transport.resolveDestination("safe")).toThrow( + expect.objectContaining({ + name: "TransportError", + code: "TRANSPORT_DESTINATION_INVALID", + }), + ); + }); + + it("wraps mapper failures with an actionable typed error and original cause", async () => { + const cause = new Error("mapper exploded"); + const transport = new LocalFileTransport({ + rootDir: await createTempDirectory(), + mapCanonicalId: () => { + throw cause; + }, + }); + + await expect(transport.upload("opaque:id", new Uint8Array())).rejects.toMatchObject({ + name: "TransportError", + code: "TRANSPORT_DESTINATION_INVALID", + guidance: expect.stringContaining("mapCanonicalId"), + cause, + }); + }); + + it("snapshots DOCX bytes before asynchronous writes can observe caller mutation", async () => { + const rootDir = await createTempDirectory(); + const transport = new LocalFileTransport({ rootDir }); + const bytes = new TextEncoder().encode("before"); + + const upload = transport.upload("snapshot", bytes); + bytes.fill("x".charCodeAt(0)); + const result = await upload; + + expect(await readFile(result.path, "utf8")).toBe("before"); + }); + + it("wraps write failures with an actionable typed error and original cause", async () => { + const rootDir = join(await createTempDirectory(), "file-root"); + await writeFile(rootDir, "not a directory"); + const transport = new LocalFileTransport({ rootDir }); + + await expect(transport.upload("doc", new Uint8Array([1, 2, 3]))).rejects.toMatchObject({ + name: "TransportError", + code: "TRANSPORT_WRITE_FAILED", + guidance: expect.stringContaining("rootDir"), + cause: expect.objectContaining({ code: expect.any(String) }), + }); + }); + + it("rejects an empty local root", () => { + expect(() => new LocalFileTransport({ rootDir: " " })).toThrow( + expect.objectContaining({ + name: "TransportError", + code: "TRANSPORT_ROOT_INVALID", + }), + ); + }); +}); + async function createTempDocx(name: string): Promise { - const tempDirectory = await mkdtemp(join(tmpdir(), "polydoc-core-test-")); - tempDirectories.push(tempDirectory); + const tempDirectory = await createTempDirectory(); const docxPath = join(tempDirectory, name); + await mkdir(join(docxPath, ".."), { recursive: true }); await writeFile(docxPath, "fake docx"); return docxPath; } +async function createTempDirectory(): Promise { + const tempDirectory = await mkdtemp(join(tmpdir(), "polydoc-core-test-")); + tempDirectories.push(tempDirectory); + + return tempDirectory; +} + function createSuccessfulDoctorRunner(): PandocRunner { return vi.fn(async () => ({ stdout: "pandoc 3.10",