diff --git a/README.md b/README.md index ebcdd56..f62bd61 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # polydoc-core `polydoc-core` is the reusable TypeScript library layer for Polydoc: deterministic -Markdown-to-DOCX conversion plus hook contracts for consumers that need their own -Markdown transforms. It is intended to be consumed by the standalone `polydoc` -tool and by TeamWiki workflows that need the same conversion behavior without -taking on a CLI. +Markdown-to-DOCX conversion, a reported DOCX-to-Markdown reverse conversion, plus +hook contracts for consumers that need their own Markdown transforms. It is +intended to be consumed by the standalone `polydoc` tool and by TeamWiki +workflows that need the same conversion behavior without taking on a CLI. The original Polydoc design work scoped a local-first Markdown-to-Word and Markdown-to-Google Docs workflow. This package keeps only the reusable library @@ -365,6 +365,150 @@ The forward DOCX writer intentionally does not pass `--wrap=none` or `--markdown-headings=atx`; those Pandoc options affect textual Markdown output, not DOCX generation. +## Reverse Conversion + +`convertDocxToMarkdown()` converts a Word document back into clean +GitHub-Flavored Markdown, and reports what it could not bring with it. + +```ts +import { convertDocxToMarkdown } from "@agentic-tooling/polydoc-core"; + +const { markdown, report } = await convertDocxToMarkdown({ + docx: "./out/word/handbook-intro.docx", // bytes or a path + postprocessors: [ + // Restore what the DOCX could not carry, from your own sidecar. + (source) => { + let restored = source; + + // Note replaceAll: passing a string to replace() rewrites only the first + // match. A real sidecar should record source offsets rather than bare + // names, because "Handbook" as a wikilink target is indistinguishable + // from "Handbook" written as ordinary prose. + for (const target of new Set(sidecar.wikilinks)) { + restored = restored.replaceAll(target, `[[${target}]]`); + } + + return `${sidecar.frontmatter}${restored}`; + }, + ], +}); + +if (report.hasUnmappableContent) { + for (const item of report.unmappable) { + console.warn(`${item.kind} (${item.count}): ${item.summary}`); + } +} +``` + +Reverse conversion uses: + +```txt +pandoc --from docx --to gfm --track-changes --output +``` + +`trackChanges` defaults to `accept`, which resolves Word revision marks into the +clean Markdown a reviewed import wants. `reject` restores the pre-edit text, and +`all` keeps insertions, deletions, and comments as inline HTML spans. The value +is always passed explicitly rather than relying on Pandoc's default, and the +resolution is recorded on the report. + +Input is validated before Pandoc runs. Bytes that are not a ZIP archive fail +with `DOCX_INPUT_INVALID`, and a ZIP with no main document part fails with +`DOCX_ARCHIVE_INVALID`, so a `.doc`, a PDF, or a renamed archive never reaches +the converter. + +Because reverse conversion is the one path that takes bytes from outside, the +archive reader is bounded in four independent ways. Each raises +`DOCX_ARCHIVE_INVALID`: + +- `DOCX_MAX_TOTAL_BYTES` (128 MB) caps the **total** decompressed across every + part read from one archive. This is the bound that actually holds, because the + set of parts read is not fixed — headers and footers are enumerated from the + archive — so a per-part limit alone is not an aggregate limit. +- `DOCX_MAX_PART_BYTES` (64 MB) caps any single part. A crafted `.docx` can + reach a compression ratio near 300:1. +- `DOCX_MAX_ARCHIVE_ENTRIES` (4096) caps how many entries are walked, and + `DOCX_MAX_HEADER_FOOTER_PARTS` (64) caps header and footer parts specifically. +- A part whose inflated size disagrees with the size its archive entry declares + is rejected rather than analyzed, because the decompressor sizes its output + buffer from that declaration and would otherwise hand back a silently + truncated fragment — producing a clean report for a document Pandoc reads in + full. + +The limits are deliberately not configurable. A hardening limit a consumer can +raise is one that hostile input can argue them into raising. + +### The round trip is lossy + +`md → docx → md` does not return the note you started with. Pandoc's DOCX reader +silently discards several kinds of Word content, so this package inspects the +DOCX archive itself and reports each one. Verified against Pandoc 3.x: + +| Report `kind` | What happens | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | +| `tracked-change` | Insertions and deletions are resolved per `trackChanges`; the revision marks and their authors are gone. Tracked *moves* count as both. Formatting-only revisions are lost in every mode, including `all`. | +| `comment` | Word comments are dropped entirely unless `trackChanges` is `all`. | +| `header-footer` | Headers and footers are ignored entirely, along with anything parked in them. | +| `text-box` | A text box is reached only through the legacy VML representation. One carrying an `mc:Fallback`, which is what Word 2010 and newer writes, is inlined as an ordinary body paragraph — the text survives but the framing and position do not. One without a fallback is dropped along with its text. | +| `external-content` | `altChunk` imports, which embed another HTML or DOCX document, are ignored. | +| `smart-art` | SmartArt is dropped, including every node label. | +| `chart` | Charts are dropped, including titles and axis labels. | +| `embedded-media` | Images are referenced by path but their bytes are never written, so the Markdown links dangle. | +| `embedded-object` | Embedded OLE objects are dropped with no placeholder. | + +Counters run across the main document part plus `footnotes.xml` and +`endnotes.xml`, whose text does flow into the Markdown; `report.scannedParts` +names exactly what was inspected. The main document part is resolved from the +package relationships rather than assumed to be `word/document.xml`. + +On top of that, plain Markdown-level detail does not survive a DOCX round trip: +YAML frontmatter is gone, `[[wikilinks]]` come back as backslash-escaped literal +text, callout markers are escaped and their internal line breaks are folded, and +paragraphs are re-wrapped at Pandoc's default column width. + +Restoring that layer is the consumer's job, through `postprocessors` and a +sidecar recorded at publish time. `tests/fixtures/golden/roundtrip/basic-note/` +records exactly what the round trip produces for the publish fixture. + +### What the report does not cover + +An empty `report.unmappable` means none of the kinds above were found. It is +**not** a proof of lossless conversion. Known gaps, by design: + +- Page and section layout: manual page breaks, columns, and section properties. +- Direct formatting with no Markdown equivalent, such as highlighting, + character spacing, and colored text. +- Content in a part outside `report.scannedParts`, including + `word/glossary/document.xml`, where Word stores AutoText and building blocks. +- VML WordArt, whose text lives in a `` attribute + rather than in an element, so element counters cannot see it. +- A header or footer whose content is entirely non-textual. The + `header-footer` item fires only when a part contains at least one `w:t`, which + keeps empty Word boilerplate out of the report at this cost. +- An OLE object that is linked rather than embedded, which leaves no + `word/embeddings/` part to detect. + +Detection is a structural scan of the archive, so a construct Word can produce +but this library does not know about will not be reported. + +### Check the report before writing anything back + +**`report.unmappable` must be checked before this Markdown is written over a +source note.** Every entry is content that exists in the Word document and does +not exist in the Markdown; writing the result back unreviewed destroys it. + +The report is returned, never thrown, and there is deliberately no `strict` +option. The intended consumer is a review step with a human in the loop, which +decides what to do — the library will not make that call by refusing to convert. +A reasonable consumer treats a non-empty `report.unmappable` as a prompt, shows +the summaries, and only then offers to apply the diff. + +Reverse import is Word-only by design. Google Docs is publish-only: a Google +round trip would go through Drive's own DOCX export on top of Pandoc's DOCX +reader, which is lossy twice over, so this package does not import, diff, or +reverse-convert Google Docs. The same applies to SharePoint, which is a +transport for Word files rather than a second document format. + ## Hooks Forward conversion accepts Markdown preprocessors: @@ -382,10 +526,16 @@ const docxBytes = await convertMarkdownToDocx({ Preprocessors run sequentially before Pandoc receives the Markdown. They receive a context object with `phase: "preprocess"` and `targetFormat: "docx"`. -The package also exports `MarkdownPostprocessor` and -`applyMarkdownPostprocessors()` for future reverse or textual Markdown pipelines. -They do not run during Markdown-to-DOCX conversion because that pipeline returns -DOCX bytes, not Markdown text. +Reverse conversion accepts Markdown postprocessors, which run sequentially over +the Markdown Pandoc produced and receive a context object with +`phase: "postprocess"` and `targetFormat: "markdown"`. This is where a consumer +restores frontmatter, wikilinks, and anything else its sidecar recorded at +publish time. + +Postprocessors do not run during Markdown-to-DOCX conversion, because that +pipeline returns DOCX bytes rather than Markdown text. `MarkdownPostprocessor` +and `applyMarkdownPostprocessors()` are also exported directly for consumers +that want to run the same hooks outside a conversion. ## Determinism diff --git a/package.json b/package.json index 0d504b1..15fdc1e 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,6 @@ "@biomejs/biome": "^2.5.5", "@types/node": "^20.19.43", "@vitest/coverage-v8": "^4.1.10", - "fflate": "^0.8.3", "google-auth-library": "10.5.0", "typescript": "^7.0.2", "vitest": "^4.1.10" @@ -60,7 +59,8 @@ "dependencies": { "@azure/msal-node": "5.4.2", "@googleapis/drive": "20.2.0", - "execa": "^8.0.1" + "execa": "^8.0.1", + "fflate": "0.8.3" }, "peerDependencies": { "google-auth-library": "^10" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e3b2a20..a98f643 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: execa: specifier: ^8.0.1 version: 8.0.1 + fflate: + specifier: 0.8.3 + version: 0.8.3 devDependencies: '@biomejs/biome': specifier: ^2.5.5 @@ -27,9 +30,6 @@ importers: '@vitest/coverage-v8': specifier: ^4.1.10 version: 4.1.10(vitest@4.1.10) - fflate: - specifier: ^0.8.3 - version: 0.8.3 google-auth-library: specifier: 10.5.0 version: 10.5.0 diff --git a/src/docx.ts b/src/docx.ts new file mode 100644 index 0000000..ce9d558 --- /dev/null +++ b/src/docx.ts @@ -0,0 +1,960 @@ +/** + * Pure DOCX archive inspection: no Pandoc, no filesystem access. + * + * Pandoc's DOCX reader silently discards several kinds of Word content. The + * fidelity contract for reverse conversion is that nothing is dropped without + * being surfaced, so the archive is inspected directly and every known loss is + * reported back to the caller alongside the Markdown. + * + * Every behavior asserted in a summary string below was verified by feeding + * Pandoc 3.10 a DOCX containing that construct and reading the Markdown it + * produced. Notably, the DrawingML/VML text box split is not the intuitive one: + * see `analyzeTextBoxes`. + * + * ## Why this scans XML with regular expressions instead of parsing it + * + * This is deliberate, and it should stay this way: + * + * - These are structural presence and count checks ("is there a `w:ins` + * anywhere"), not queries that need a tree. + * - WordprocessingML entity-escapes `<` in text content, so an open-tag pattern + * can never collide with user-authored text. A `` does not appear unescaped inside an attribute + * value. That is legal XML but Word does not emit it. + * - Namespaces declared inline on `mc:AlternateContent`, rather than on the + * document root, change what Pandoc does without changing what this reports: + * Pandoc drops such a text box while the report still calls it inlined. The + * `text-box` item still fires, so the caller is still told to review it and + * only the prose misdescribes the outcome. Word declares on the root. + * - On unbalanced XML an unclosed region never terminates, so its content falls + * through into the `analyzeTextBoxes` remainder and can be counted twice. + * Malformed input only; a size-mismatched archive is rejected before this. + */ + +import { unzipSync } from "fflate"; + +import { PandocError } from "./errors.js"; + +/** Package relationship part naming the main document part. */ +export const DOCX_PACKAGE_RELS_PART = "_rels/.rels"; +/** Used when the package relationships do not name a main document part. */ +export const DEFAULT_DOCX_DOCUMENT_PART = "word/document.xml"; +/** Archive prefix holding embedded images and other media. */ +export const DOCX_MEDIA_PREFIX = "word/media/"; +/** Archive prefix holding embedded OLE objects. */ +export const DOCX_EMBEDDINGS_PREFIX = "word/embeddings/"; +/** Archive prefix holding SmartArt graphics. */ +export const DOCX_DIAGRAMS_PREFIX = "word/diagrams/"; +/** Archive prefix holding charts. */ +export const DOCX_CHARTS_PREFIX = "word/charts/"; + +/** + * Largest single part this inspector will decompress. + * + * Word's own main document part is low single-digit megabytes even for a large + * manuscript. The cap exists because a hostile archive can reach a ratio near + * 300:1, so a few hundred kilobytes of input would otherwise inflate into + * hundreds of megabytes of resident memory. + */ +export const DOCX_MAX_PART_BYTES = 64 * 1024 * 1024; + +/** + * Largest total this inspector will decompress across every part of one + * archive. + * + * The per-part cap alone is not a memory bound. The set of parts read is not + * fixed — headers and footers are enumerated from the archive — so an archive + * can stay under the per-part cap on every entry and still force gigabytes of + * decompression in aggregate. This budget is what actually holds, and it holds + * regardless of which part kind is enumerated next. + * + * Note that this bounds bytes decompressed, not peak resident memory, which + * runs roughly 2.4x higher: the inflated `Uint8Array`s and the UTF-16 strings + * decoded from them coexist, on top of the caller's copy of the input. An + * archive sitting just under this budget peaks around 300 MB. + */ +export const DOCX_MAX_TOTAL_BYTES = 128 * 1024 * 1024; + +/** Largest number of archive entries this inspector will walk. */ +export const DOCX_MAX_ARCHIVE_ENTRIES = 4096; + +/** + * Largest number of header and footer parts this inspector will read. Word + * writes at most six per section, so anything near this is already pathological. + */ +export const DOCX_MAX_HEADER_FOOTER_PARTS = 64; + +/** ZIP local file header, the first four bytes of every DOCX. */ +const ZIP_LOCAL_FILE_HEADER = [0x50, 0x4b, 0x03, 0x04] as const; + +const OFFICE_DOCUMENT_RELATIONSHIP = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument"; + +/** + * How Pandoc resolves Word revision marks. + * + * `accept` produces the clean Markdown a reviewed import wants. `all` keeps + * insertions, deletions, and comments as HTML spans instead of dropping them. + */ +export type DocxTrackChangesMode = "accept" | "reject" | "all"; + +export const DOCX_TRACK_CHANGES_MODES: readonly DocxTrackChangesMode[] = [ + "accept", + "reject", + "all", +]; + +export const DEFAULT_DOCX_TRACK_CHANGES: DocxTrackChangesMode = "accept"; + +export type DocxUnmappableKind = + | "chart" + | "comment" + | "embedded-media" + | "embedded-object" + | "external-content" + | "header-footer" + | "smart-art" + | "text-box" + | "tracked-change"; + +export interface DocxUnmappableItem { + /** Stable machine-readable discriminator for this class of loss. */ + readonly kind: DocxUnmappableKind; + /** Human-readable explanation of what was found and what Pandoc did with it. */ + readonly summary: string; + /** How many of this item the archive contains. */ + readonly count: number; + /** Archive part names, for the kinds that map to concrete files. */ + readonly entries?: readonly string[]; +} + +export interface DocxTrackedChangeCounts { + /** `w:ins` and `w:moveTo` elements. */ + readonly insertions: number; + /** `w:del` and `w:moveFrom` elements. */ + readonly deletions: number; + /** `w:rPrChange` and friends: revisions that restyle rather than edit text. */ + readonly formattingChanges: number; +} + +export interface DocxImportReport { + /** + * Everything the conversion could not carry into Markdown faithfully. + * Empty for a document Pandoc can represent completely. + */ + readonly unmappable: readonly DocxUnmappableItem[]; + /** Convenience flag: `unmappable.length > 0`. */ + readonly hasUnmappableContent: boolean; + /** The `--track-changes` value the conversion ran with. */ + readonly trackChanges: DocxTrackChangesMode; + /** + * Revision marks counted across `scannedParts`. + * + * `insertions` and `deletions` count revision-mark elements, which includes + * paragraph-mark revisions that Word writes on every Enter pressed with + * tracking on and that cost nothing in Markdown. Formatting-only revisions + * are not revision marks and are counted separately as `formattingChanges`. + */ + readonly trackedChanges: DocxTrackedChangeCounts; + /** The main document part, resolved from the package relationships. */ + readonly documentPart: string; + /** + * Every part the structural counters ran over. Content in a part outside this + * list is covered by a whole-part item, such as `header-footer`, or is not + * inspected at all. + */ + readonly scannedParts: readonly string[]; +} + +interface DocxArchive { + readonly documentPart: string; + readonly scannedParts: readonly string[]; + /** Concatenated XML of every scanned content part, exactly as stored. */ + readonly contentXml: string; + readonly commentsXml: string | undefined; + readonly headerFooterParts: readonly string[]; + readonly mediaEntries: readonly string[]; + readonly embeddingEntries: readonly string[]; + readonly diagramEntries: readonly string[]; + readonly chartEntries: readonly string[]; +} + +interface TextBoxCounts { + readonly inlined: number; + readonly dropped: number; +} + +interface ElementRegion { + readonly start: number; + readonly end: number; + readonly inner: string; +} + +interface ArchiveScan { + /** Running total of declared bytes decompressed from one archive. */ + bytes: number; + /** CRC-32 per entry, read from the archive's own central directory. */ + readonly checksums: ReadonlyMap; +} + +function validateDocxTrackChanges(value: unknown): DocxTrackChangesMode { + if (typeof value === "string" && isTrackChangesMode(value)) { + return value; + } + + throw new PandocError( + "DOCX_TRACK_CHANGES_INVALID", + `trackChanges must be one of ${DOCX_TRACK_CHANGES_MODES.join(", ")}.`, + "Pass trackChanges as accept to resolve Word revision marks into clean Markdown, reject to restore the pre-edit text, or all to keep insertions, deletions, and comments as inline spans.", + ); +} + +/** + * Inspects a DOCX archive and reports everything reverse conversion cannot + * represent in Markdown. Throws when the bytes are not a readable Word + * document, so garbage never reaches Pandoc. + */ +export function buildDocxImportReport( + docx: Uint8Array, + trackChanges: DocxTrackChangesMode = DEFAULT_DOCX_TRACK_CHANGES, +): DocxImportReport { + const mode = validateDocxTrackChanges(trackChanges); + const archive = readDocxArchive(docx); + // `mc:Fallback` restates the preceding `mc:Choice` in a legacy representation + // rather than adding content, so counting both would double-report one edit. + // Text boxes are the exception and are analyzed on the unstripped XML, + // because for them the fallback is the branch Pandoc actually reads. + const contentXml = stripFallbackRegions(archive.contentXml); + const trackedChanges: DocxTrackedChangeCounts = { + insertions: countElements(contentXml, "ins") + countElements(contentXml, "moveTo"), + deletions: countElements(contentXml, "del") + countElements(contentXml, "moveFrom"), + formattingChanges: countFormattingRevisions(contentXml), + }; + const scope = describeParts(archive.scannedParts); + const unmappable = [ + buildTrackedChangeItem(trackedChanges, mode, scope), + buildCommentItem(archive.commentsXml, mode), + buildHeaderFooterItem(archive.headerFooterParts), + buildTextBoxItem(analyzeTextBoxes(archive.contentXml), scope), + buildExternalContentItem(countElements(contentXml, "altChunk"), scope), + buildSmartArtItem(archive.diagramEntries), + buildChartItem(archive.chartEntries), + buildMediaItem(archive.mediaEntries), + buildEmbeddedObjectItem(archive.embeddingEntries), + ].filter((item): item is DocxUnmappableItem => item !== undefined); + + return { + unmappable, + hasUnmappableContent: unmappable.length > 0, + trackChanges: mode, + trackedChanges, + documentPart: archive.documentPart, + scannedParts: archive.scannedParts, + }; +} + +function readDocxArchive(docx: Uint8Array): DocxArchive { + validateDocxBytes(docx); + + const names: string[] = []; + // One scan spans both passes, so the total decompressed for an archive is + // bounded no matter how the work is split across them. + const scan: ArchiveScan = { bytes: 0, checksums: readCentralDirectoryChecksums(docx) }; + // The first pass parses only the central directory and inflates the tiny + // package relationships part, so the main document part can be resolved by + // name before anything large is decompressed. + const rels = inflateParts(docx, names, (name) => name === DOCX_PACKAGE_RELS_PART, scan); + const relsXml = rels[DOCX_PACKAGE_RELS_PART]; + const resolved = resolveDocumentPart(relsXml, names); + + if (!names.includes(resolved.part)) { + throw new PandocError( + "DOCX_ARCHIVE_INVALID", + `The input is a ZIP archive with no readable main document part: ${ + relsXml === undefined + ? `it has no ${DOCX_PACKAGE_RELS_PART} and no ${DEFAULT_DOCX_DOCUMENT_PART}` + : `its package relationships name no officeDocument target present in the archive, and there is no ${DEFAULT_DOCX_DOCUMENT_PART}` + }.`, + "Pass a Word .docx file. Other Office formats, OpenDocument files, and plain ZIP archives are not supported.", + ); + } + + const documentPart = resolved.part; + const directory = documentPart.slice(0, documentPart.lastIndexOf("/") + 1); + const commentsPart = `${directory}comments.xml`; + const headerFooterPattern = new RegExp(`^${escapeRegExp(directory)}(?:header|footer)\\d*\\.xml$`); + // Footnote and endnote text does flow into the Markdown, so revisions inside + // them belong in the same counters as the body. + const notesParts = [`${directory}footnotes.xml`, `${directory}endnotes.xml`].filter((part) => + names.includes(part), + ); + const headerFooterCandidates = names.filter((name) => headerFooterPattern.test(name)).sort(); + + if (headerFooterCandidates.length > DOCX_MAX_HEADER_FOOTER_PARTS) { + throw new PandocError( + "DOCX_ARCHIVE_INVALID", + `The DOCX archive declares ${headerFooterCandidates.length} header and footer parts, over the ${DOCX_MAX_HEADER_FOOTER_PARTS} part limit.`, + "Pass a Word .docx file. Word writes at most six header and footer parts per section, so a count this high indicates a corrupt or hostile archive.", + ); + } + + const scannedParts = [documentPart, ...notesParts]; + const wanted = new Set([...scannedParts, commentsPart, ...headerFooterCandidates]); + const parts = inflateParts(docx, undefined, (name) => wanted.has(name), scan); + + return { + documentPart, + scannedParts, + contentXml: scannedParts.map((part) => parts[part] ?? "").join("\n"), + commentsXml: parts[commentsPart], + // A header or footer with no text is Word boilerplate, not lost content. + headerFooterParts: headerFooterCandidates.filter((part) => + containsElement(parts[part] ?? "", "t"), + ), + mediaEntries: filterEntries(names, DOCX_MEDIA_PREFIX), + embeddingEntries: filterEntries(names, DOCX_EMBEDDINGS_PREFIX), + diagramEntries: filterEntries(names, DOCX_DIAGRAMS_PREFIX), + chartEntries: filterEntries(names, DOCX_CHARTS_PREFIX), + }; +} + +/** + * Decompresses the parts `wanted` selects, optionally recording every entry + * name into `names`. The filter callback runs once per archive entry, so it + * doubles as the inventory and as the place to enforce the archive limits + * before any decompression happens. + */ +function inflateParts( + docx: Uint8Array, + names: string[] | undefined, + wanted: (name: string) => boolean, + scan: ArchiveScan, +): Record { + const declaredSizes = new Map(); + let entryCount = 0; + let files: Record; + + try { + files = unzipSync(docx, { + filter: (file) => { + entryCount += 1; + + if (entryCount > DOCX_MAX_ARCHIVE_ENTRIES) { + throw new PandocError( + "DOCX_ARCHIVE_INVALID", + `The DOCX archive declares more than ${DOCX_MAX_ARCHIVE_ENTRIES} entries.`, + "Pass a Word .docx file. An archive this large is not a document Word produced.", + ); + } + + if (names !== undefined && !file.name.endsWith("/")) { + names.push(file.name); + } + + if (!wanted(file.name)) { + return false; + } + + // `originalSize` comes from the central directory, and fflate allocates + // exactly that many bytes to inflate into, so it bounds the allocation + // even when an archive lies about it. A lie in the other direction is + // caught after inflation, below. + if (file.originalSize > DOCX_MAX_PART_BYTES) { + throw new PandocError( + "DOCX_ARCHIVE_INVALID", + `The DOCX part ${file.name} declares ${file.originalSize} bytes, over the ${DOCX_MAX_PART_BYTES} byte limit.`, + "Pass a Word .docx file. A part this large indicates a corrupt or hostile archive rather than a document.", + ); + } + + scan.bytes += file.originalSize; + + if (scan.bytes > DOCX_MAX_TOTAL_BYTES) { + throw new PandocError( + "DOCX_ARCHIVE_INVALID", + `The DOCX parts this inspector must read declare more than ${DOCX_MAX_TOTAL_BYTES} bytes in total.`, + "Pass a Word .docx file. Many individually plausible parts adding up to this much data indicate a corrupt or hostile archive.", + ); + } + + declaredSizes.set(file.name, file.originalSize); + + return true; + }, + }); + } catch (cause) { + // The limit checks above throw through fflate, so they must not be remapped + // into the generic unreadable-archive error. + if (cause instanceof PandocError) { + throw cause; + } + + throw new PandocError( + "DOCX_ARCHIVE_INVALID", + "The DOCX input could not be read as a ZIP archive.", + "Pass an intact .docx file. A truncated download or a file rewritten by a text editor produces this error.", + { cause }, + ); + } + + const decoded: Record = {}; + const decoder = new TextDecoder(); + + for (const [name, bytes] of Object.entries(files)) { + // fflate inflates a deflate entry into a buffer pre-sized from the central + // directory, so a part declaring less than it really holds is silently + // truncated and analyzing the fragment would produce a clean report for a + // document Pandoc reads in full. Comparing lengths cannot catch that: the + // declared size is what sized the buffer, so it always matches. The CRC is + // the archive's own integrity mechanism and catches both directions. + // A stored entry is sliced by its compressed size, so an understated + // uncompressed size leaves the content intact but the index inconsistent. + // That is caught here; it is a no-op for a deflate entry, where this + // comparison is against the very number that sized the buffer. + if (bytes.length !== declaredSizes.get(name)) { + throw new PandocError( + "DOCX_ARCHIVE_INVALID", + `The DOCX part ${name} holds ${bytes.length} bytes but its archive entry declares ${declaredSizes.get(name)}.`, + "Pass an intact .docx file. A size mismatch means the archive index disagrees with its contents, so no report over it could be trusted.", + ); + } + + const expected = scan.checksums.get(name); + + if (expected === undefined || expected !== crc32(bytes)) { + throw new PandocError( + "DOCX_ARCHIVE_INVALID", + `The DOCX part ${name} does not match the CRC-32 recorded in the archive index.`, + "Pass an intact .docx file. A checksum mismatch means the archive index disagrees with its contents, so the part may be truncated and no report over it could be trusted.", + ); + } + + decoded[name] = decoder.decode(bytes); + } + + return decoded; +} + +/** + * Reads the CRC-32 of every entry from the archive's central directory. + * + * fflate keeps its own `crc` helper internal and exposes no checksum on the + * filter callback, so the directory is walked here. The traversal mirrors + * fflate's own end-of-central-directory discovery, including its ZIP64 handling, + * so the two agree on which records exist. + */ +function readCentralDirectoryChecksums(docx: Uint8Array): ReadonlyMap { + const checksums = new Map(); + + if (docx.byteLength < 22) { + return checksums; + } + + const view = new DataView(docx.buffer, docx.byteOffset, docx.byteLength); + let end = docx.byteLength - 22; + + while (view.getUint32(end, true) !== 0x0605_4b50) { + if (end === 0 || docx.byteLength - end > 65_558) { + return checksums; + } + + end -= 1; + } + + let entries = view.getUint16(end + 8, true); + let offset = view.getUint32(end + 16, true); + + if (end >= 20 && view.getUint32(end - 20, true) === 0x0706_4b50) { + const zip64End = view.getUint32(end - 12, true); + + if (zip64End + 52 <= docx.byteLength && view.getUint32(zip64End, true) === 0x0606_4b50) { + entries = view.getUint32(zip64End + 32, true); + offset = view.getUint32(zip64End + 48, true); + } + } + + const decoder = new TextDecoder(); + + for (let index = 0; index < entries; index += 1) { + if (offset + 46 > docx.byteLength || view.getUint32(offset, true) !== 0x0201_4b50) { + break; + } + + const nameLength = view.getUint16(offset + 28, true); + const nameStart = offset + 46; + + if (nameStart + nameLength > docx.byteLength) { + break; + } + + checksums.set( + decoder.decode(docx.subarray(nameStart, nameStart + nameLength)), + view.getUint32(offset + 16, true), + ); + offset = + nameStart + + nameLength + + view.getUint16(offset + 30, true) + + view.getUint16(offset + 32, true); + } + + return checksums; +} + +const CRC32_TABLE = /* @__PURE__ */ (() => { + const table = new Int32Array(256); + + for (let index = 0; index < 256; index += 1) { + let value = index; + + for (let bit = 0; bit < 8; bit += 1) { + value = (value & 1) === 1 ? (value >>> 1) ^ 0xedb8_8320 : value >>> 1; + } + + table[index] = value; + } + + return table; +})(); + +/** Standard ZIP CRC-32. */ +function crc32(bytes: Uint8Array): number { + let checksum = -1; + + for (let index = 0; index < bytes.length; index += 1) { + checksum = (checksum >>> 8) ^ (CRC32_TABLE[(checksum ^ (bytes[index] ?? 0)) & 0xff] ?? 0); + } + + return (checksum ^ -1) >>> 0; +} + +/** + * Resolves the main document part from the package relationships. + * + * The name is not fixed by the specification. `word/document2.xml` occurs in + * the wild and Pandoc converts it happily, so assuming `word/document.xml` + * would reject valid Word documents. + */ +function resolveDocumentPart( + relsXml: string | undefined, + names: readonly string[], +): { part: string; fromRels: boolean } { + // The element may carry a namespace prefix, so match on the local name. + for (const tag of relsXml?.match(/<(?:[A-Za-z_][\w.-]*:)?Relationship(?=[\s/>])[^>]*>/g) ?? []) { + if (readAttribute(tag, "Type") !== OFFICE_DOCUMENT_RELATIONSHIP) { + continue; + } + + // Package-relative targets may be written absolute or explicitly relative. + const target = (readAttribute(tag, "Target") ?? "").replace(/^\/+/, "").replace(/^\.\//, ""); + + if (target !== "" && names.includes(target)) { + return { part: target, fromRels: true }; + } + } + + return { part: DEFAULT_DOCX_DOCUMENT_PART, fromRels: false }; +} + +function validateDocxBytes(docx: Uint8Array): void { + if (!(docx instanceof Uint8Array)) { + throw new PandocError( + "DOCX_INPUT_INVALID", + "The DOCX input must be a Uint8Array of file bytes.", + "Pass DOCX bytes, or a path to a .docx file for the library to read.", + ); + } + + if (docx.byteLength === 0) { + throw new PandocError( + "DOCX_INPUT_INVALID", + "The DOCX input is empty.", + "Pass a non-empty .docx file. An empty file usually means an interrupted download or an unwritten export.", + ); + } + + if (!ZIP_LOCAL_FILE_HEADER.every((byte, index) => docx[index] === byte)) { + throw new PandocError( + "DOCX_INPUT_INVALID", + "The DOCX input does not start with a ZIP local file header, so it is not a .docx file.", + "Pass a real Word .docx file. Legacy .doc, RTF, and PDF files are different formats and are not supported.", + ); + } +} + +function buildTrackedChangeItem( + counts: DocxTrackedChangeCounts, + mode: DocxTrackChangesMode, + scope: string, +): DocxUnmappableItem | undefined { + const revisions = counts.insertions + counts.deletions; + + // `all` keeps insertions and deletions as inline spans, so only the + // formatting revisions it cannot represent remain a loss in that mode. + if (mode === "all") { + if (counts.formattingChanges === 0) { + return undefined; + } + + return { + kind: "tracked-change", + count: counts.formattingChanges, + summary: `${plural( + counts.formattingChanges, + "formatting revision", + )} in ${scope}. Pandoc ran --track-changes=all, which preserves insertions and deletions as inline spans but has no representation for a formatting-only revision, so the superseded formatting is gone.`, + }; + } + + if (revisions + counts.formattingChanges === 0) { + return undefined; + } + + const resolution = + mode === "accept" + ? "insertions were kept as ordinary text and deletions were discarded" + : "deletions were restored as ordinary text and insertions were discarded"; + const formattingNote = + counts.formattingChanges === 0 + ? "" + : ` A further ${plural( + counts.formattingChanges, + "formatting revision", + )} discarded the superseded formatting.`; + + return { + kind: "tracked-change", + count: revisions + counts.formattingChanges, + summary: `${plural(revisions, "tracked change")} in ${scope} (${plural( + counts.insertions, + "insertion", + )}, ${plural( + counts.deletions, + "deletion", + )}, counting moved text as both). Pandoc ran --track-changes=${mode}, so ${resolution}; the revision marks and their authors are not represented in the Markdown.${formattingNote}`, + }; +} + +function buildCommentItem( + commentsXml: string | undefined, + mode: DocxTrackChangesMode, +): DocxUnmappableItem | undefined { + if (commentsXml === undefined) { + return undefined; + } + + // Pandoc writes an empty `` part into every DOCX it generates, + // so the part existing proves nothing. Only comment bodies count. + const count = countElements(commentsXml, "comment"); + + if (count === 0 || mode === "all") { + return undefined; + } + + return { + kind: "comment", + count, + summary: `${plural(count, "Word comment")}. Pandoc only emits comment text with --track-changes=all; this conversion ran --track-changes=${mode}, so the comments were dropped and appear nowhere in the Markdown.`, + }; +} + +function buildHeaderFooterItem(parts: readonly string[]): DocxUnmappableItem | undefined { + if (parts.length === 0) { + return undefined; + } + + return { + kind: "header-footer", + count: parts.length, + entries: parts, + summary: `${plural( + parts.length, + "header or footer part", + "header and footer parts", + )} containing text. Pandoc's DOCX reader ignores headers and footers entirely, so running titles, page furniture, and any content parked there are absent from the Markdown.`, + }; +} + +function buildTextBoxItem(counts: TextBoxCounts, scope: string): DocxUnmappableItem | undefined { + const total = counts.inlined + counts.dropped; + + if (total === 0) { + return undefined; + } + + return { + kind: "text-box", + count: total, + summary: `${plural(total, "text box", "text boxes")} in ${scope} (${ + counts.inlined + } inlined as ordinary body paragraphs, losing their text-box framing and position; ${ + counts.dropped + } dropped outright). Pandoc 3.x reaches text box content only through the legacy VML representation: a DrawingML text box carrying an mc:Fallback, which is what Word 2010 and newer writes, is read from that fallback, while one without a fallback is discarded along with its text.`, + }; +} + +function buildExternalContentItem(count: number, scope: string): DocxUnmappableItem | undefined { + if (count === 0) { + return undefined; + } + + return { + kind: "external-content", + count, + summary: `${plural( + count, + "altChunk import", + )} in ${scope}. An altChunk embeds another document, usually HTML or a nested DOCX, for Word to merge in. Pandoc's DOCX reader ignores it, so all of that text is missing from the Markdown.`, + }; +} + +function buildSmartArtItem(entries: readonly string[]): DocxUnmappableItem | undefined { + if (entries.length === 0) { + return undefined; + } + + // One SmartArt graphic spans several parts; the data part is the one holding + // the node text, so it is what makes the graphic countable. + const count = countMatchingEntries(entries, /\/data\d*\.xml$/); + + return { + kind: "smart-art", + count, + entries, + summary: `${plural( + count, + "SmartArt graphic", + )} under ${DOCX_DIAGRAMS_PREFIX}. Pandoc's DOCX reader drops SmartArt entirely, and its node labels are ordinary prose, so every word in the diagram is missing from the Markdown.`, + }; +} + +function buildChartItem(entries: readonly string[]): DocxUnmappableItem | undefined { + if (entries.length === 0) { + return undefined; + } + + const count = countMatchingEntries(entries, /\/chart\d*\.xml$/); + + return { + kind: "chart", + count, + entries, + summary: `${plural( + count, + "chart", + )} under ${DOCX_CHARTS_PREFIX}. Pandoc's DOCX reader drops charts entirely, including their titles and axis labels. A chart also stores its data as a separate embedded workbook, which is reported on its own.`, + }; +} + +function buildMediaItem(entries: readonly string[]): DocxUnmappableItem | undefined { + if (entries.length === 0) { + return undefined; + } + + return { + kind: "embedded-media", + count: entries.length, + entries, + summary: `${plural( + entries.length, + "embedded media file", + )} under ${DOCX_MEDIA_PREFIX}. Pandoc references media by path, such as media/image1.png, but does not write the bytes unless --extract-media is used, so those Markdown references point at files that do not exist.`, + }; +} + +function buildEmbeddedObjectItem(entries: readonly string[]): DocxUnmappableItem | undefined { + if (entries.length === 0) { + return undefined; + } + + return { + kind: "embedded-object", + count: entries.length, + entries, + summary: `${plural( + entries.length, + "embedded OLE object", + )} under ${DOCX_EMBEDDINGS_PREFIX}. Pandoc's DOCX reader drops embedded objects entirely, so neither the object nor a placeholder appears in the Markdown.`, + }; +} + +/** + * Classifies text boxes by whether Pandoc can reach their content. + * + * The intuitive split, DrawingML versus legacy VML, is the wrong one. Verified + * against Pandoc 3.10: + * + * - bare legacy VML `w:pict`/`v:textbox` is inlined + * - DrawingML inside `mc:AlternateContent` *with* an `mc:Fallback` is inlined, + * read out of the fallback rather than out of the `mc:Choice` + * - DrawingML with no fallback, wrapped or bare, is dropped + * + * Word 2010 and newer always writes the fallback, so the common real-world case + * is preserved. Classifying on the post-strip namespace prefix would report + * exactly that case as a total loss. + */ +function analyzeTextBoxes(xml: string): TextBoxCounts { + let inlined = 0; + let dropped = 0; + let remainder = ""; + let cursor = 0; + + for (const region of findElementRegions(xml, "AlternateContent")) { + remainder += xml.slice(cursor, region.start); + cursor = region.end; + + const hasFallbackTextBox = findElementRegions(region.inner, "Fallback").some((fallback) => + containsElement(fallback.inner, "txbxContent"), + ); + + if (hasFallbackTextBox) { + inlined += 1; + } else if (containsElement(region.inner, "txbxContent")) { + dropped += 1; + } + } + + remainder += xml.slice(cursor); + + // Outside any AlternateContent, a text box is dropped only when its content + // sits directly in a DrawingML `wps:txbx` with no legacy representation. + const remainderTotal = countElements(remainder, "txbxContent"); + const remainderDropped = Math.min(countElements(remainder, "txbx"), remainderTotal); + + return { + inlined: inlined + (remainderTotal - remainderDropped), + dropped: dropped + remainderDropped, + }; +} + +/** + * Removes `mc:Fallback` regions, which restate the preceding `mc:Choice` + * content in a legacy representation rather than adding new content. + */ +function stripFallbackRegions(xml: string): string { + let stripped = ""; + let cursor = 0; + + for (const region of findElementRegions(xml, "Fallback")) { + stripped += xml.slice(cursor, region.start); + cursor = region.end; + } + + return stripped + xml.slice(cursor); +} + +/** + * Finds the outermost balanced regions of an element by local name. + * + * Depth tracking is what makes this correct for a nested `mc:AlternateContent` + * inside an `mc:Fallback`, which is legal and which a non-greedy pattern would + * mis-terminate at the inner closing tag. + */ +function findElementRegions(xml: string, localName: string): readonly ElementRegion[] { + const pattern = new RegExp(`<(/)?(?:[A-Za-z_][\\w.-]*:)?${localName}(?=[\\s/>])([^>]*)>`, "g"); + const regions: ElementRegion[] = []; + let depth = 0; + let start = 0; + let innerStart = 0; + let match = pattern.exec(xml); + + while (match !== null) { + const isClosing = match[1] === "/"; + const isSelfClosing = (match[2] ?? "").endsWith("/"); + + if (!isClosing && !isSelfClosing) { + if (depth === 0) { + start = match.index; + innerStart = match.index + match[0].length; + } + + depth += 1; + } else if (isClosing && depth > 0) { + depth -= 1; + + if (depth === 0) { + regions.push({ + start, + end: match.index + match[0].length, + inner: xml.slice(innerStart, match.index), + }); + } + } + + match = pattern.exec(xml); + } + + return regions; +} + +/** + * Counts elements by local name, ignoring the namespace prefix. + * + * The lookahead is load-bearing: without it `ins` would match the table border + * elements `w:insideH` and `w:insideV`, `del` would match `w:delText`, + * `moveFrom` would match `w:moveFromRangeStart`, `txbx` would match + * `w:txbxContent`, and `comment` would match the `w:comments` root element. + */ +function countElements(xml: string, localName: string): number { + return (xml.match(elementPattern(localName)) ?? []).length; +} + +function containsElement(xml: string, localName: string): boolean { + return elementPattern(localName).test(xml); +} + +function elementPattern(localName: string): RegExp { + return new RegExp(`<(?:[A-Za-z_][\\w.-]*:)?${localName}(?=[\\s/>])`, "g"); +} + +/** Counts `w:rPrChange`, `w:pPrChange`, `w:tblPrChange`, and the rest. */ +function countFormattingRevisions(xml: string): number { + return (xml.match(/<(?:[A-Za-z_][\w.-]*:)?\w*PrChange(?=[\s/>])/g) ?? []).length; +} + +function countMatchingEntries(entries: readonly string[], pattern: RegExp): number { + const matched = entries.filter((entry) => pattern.test(entry)).length; + + return matched === 0 ? entries.length : matched; +} + +function filterEntries(names: readonly string[], prefix: string): readonly string[] { + return names.filter((name) => name.startsWith(prefix)).sort(); +} + +function describeParts(parts: readonly string[]): string { + if (parts.length === 1) { + return parts[0] ?? ""; + } + + return `${parts.slice(0, -1).join(", ")} and ${parts.at(-1)}`; +} + +/** Reads an XML attribute value, accepting either legal quote style. */ +function readAttribute(tag: string, name: string): string | undefined { + const match = new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)')`).exec(tag); + + return match?.[1] ?? match?.[2]; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`); +} + +function isTrackChangesMode(value: string): value is DocxTrackChangesMode { + return (DOCX_TRACK_CHANGES_MODES as readonly string[]).includes(value); +} + +function plural(count: number, singular: string, pluralForm = `${singular}s`): string { + return `${count} ${count === 1 ? singular : pluralForm}`; +} diff --git a/src/errors.ts b/src/errors.ts new file mode 100644 index 0000000..3632fc4 --- /dev/null +++ b/src/errors.ts @@ -0,0 +1,48 @@ +/** + * Shared typed error for every Pandoc-backed conversion path. + * + * This lives in its own module so the pure DOCX archive inspector and the + * Pandoc process layer can both raise the same error type without importing + * each other. + */ + +export type PandocDoctorFailureCode = + | "PANDOC_NOT_FOUND" + | "PANDOC_PROBE_FAILED" + | "PANDOC_VERSION_UNPARSEABLE" + | "PANDOC_UNSUPPORTED_MAJOR"; + +export type PandocErrorCode = + | PandocDoctorFailureCode + | "DOCX_ARCHIVE_INVALID" + | "DOCX_INPUT_INVALID" + | "DOCX_TRACK_CHANGES_INVALID" + | "MARKDOWN_HOOK_FAILED" + | "PANDOC_CONVERSION_FAILED" + | "REFERENCE_DOC_REQUIRED" + | "REFERENCE_DOC_INVALID" + | "SOURCE_DATE_EPOCH_INVALID"; + +export class PandocError extends Error { + readonly code: PandocErrorCode; + readonly guidance: string; + + constructor( + code: PandocErrorCode, + message: string, + guidance: string, + options: { readonly cause?: unknown } = {}, + ) { + const fullMessage = `${message} ${guidance}`; + + if ("cause" in options) { + super(fullMessage, { cause: options.cause }); + } else { + super(fullMessage); + } + + this.name = "PandocError"; + this.code = code; + this.guidance = guidance; + } +} diff --git a/src/index.ts b/src/index.ts index 9d04ad1..43a40d1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,20 @@ +export type { + DocxImportReport, + DocxTrackChangesMode, + DocxTrackedChangeCounts, + DocxUnmappableItem, + DocxUnmappableKind, +} from "./docx.js"; +export { + buildDocxImportReport, + DEFAULT_DOCX_DOCUMENT_PART, + DEFAULT_DOCX_TRACK_CHANGES, + DOCX_MAX_ARCHIVE_ENTRIES, + DOCX_MAX_HEADER_FOOTER_PARTS, + DOCX_MAX_PART_BYTES, + DOCX_MAX_TOTAL_BYTES, + DOCX_TRACK_CHANGES_MODES, +} from "./docx.js"; export type { GoogleDriveAuthClient, GoogleDriveCreateFileParams, @@ -24,7 +41,9 @@ export { validateGoogleDriveDocxSize, } from "./google.js"; export type { + ConvertDocxToMarkdownOptions, ConvertMarkdownToDocxOptions, + DocxToMarkdownResult, MarkdownPostprocessor, MarkdownPreprocessor, MarkdownProcessor, @@ -43,6 +62,7 @@ export type { export { applyMarkdownPostprocessors, applyMarkdownPreprocessors, + convertDocxToMarkdown, convertMarkdownToDocx, DEFAULT_SOURCE_DATE_EPOCH, doctor, diff --git a/src/pandoc.ts b/src/pandoc.ts index e4eeeee..e37160c 100644 --- a/src/pandoc.ts +++ b/src/pandoc.ts @@ -4,24 +4,18 @@ import { tmpdir } from "node:os"; import { basename, extname, join } from "node:path"; import { execa } from "execa"; +import type { DocxImportReport, DocxTrackChangesMode } from "./docx.js"; +import { buildDocxImportReport, DEFAULT_DOCX_TRACK_CHANGES } from "./docx.js"; +import type { PandocDoctorFailureCode } from "./errors.js"; +import { PandocError } from "./errors.js"; + export const SUPPORTED_PANDOC_MAJOR = 3; export const DEFAULT_SOURCE_DATE_EPOCH = "0"; const DEFAULT_PANDOC_BINARY = "pandoc"; -export type PandocDoctorFailureCode = - | "PANDOC_NOT_FOUND" - | "PANDOC_PROBE_FAILED" - | "PANDOC_VERSION_UNPARSEABLE" - | "PANDOC_UNSUPPORTED_MAJOR"; - -export type PandocErrorCode = - | PandocDoctorFailureCode - | "MARKDOWN_HOOK_FAILED" - | "PANDOC_CONVERSION_FAILED" - | "REFERENCE_DOC_REQUIRED" - | "REFERENCE_DOC_INVALID" - | "SOURCE_DATE_EPOCH_INVALID"; +export type { PandocDoctorFailureCode, PandocErrorCode } from "./errors.js"; +export { PandocError } from "./errors.js"; export interface PandocVersion { readonly major: number; @@ -90,7 +84,11 @@ export type MarkdownProcessor = ( export type MarkdownPreprocessor = MarkdownProcessor; /** - * Typed contract for future reverse/textual Markdown pipelines. + * Typed contract for reverse and textual Markdown pipelines. + * + * `convertDocxToMarkdown()` runs these over the Markdown Pandoc produced, which + * is where a consumer restores the Obsidian layer a DOCX cannot carry, such as + * YAML frontmatter and `[[wikilinks]]`, from its own sidecar. * * Markdown-to-DOCX conversion does not run postprocessors because the forward * pipeline returns DOCX bytes, not Markdown text. @@ -105,28 +103,23 @@ export interface ConvertMarkdownToDocxOptions extends PandocDoctorOptions { readonly metadata?: Readonly>; } -export class PandocError extends Error { - readonly code: PandocErrorCode; - readonly guidance: string; - - constructor( - code: PandocErrorCode, - message: string, - guidance: string, - options: { readonly cause?: unknown } = {}, - ) { - const fullMessage = `${message} ${guidance}`; - - if ("cause" in options) { - super(fullMessage, { cause: options.cause }); - } else { - super(fullMessage); - } +export interface ConvertDocxToMarkdownOptions extends PandocDoctorOptions { + /** DOCX bytes, or a filesystem path to a `.docx` file to read. */ + readonly docx: Uint8Array | string; + /** How to resolve Word revision marks. Defaults to `accept`. */ + readonly trackChanges?: DocxTrackChangesMode; + readonly postprocessors?: readonly MarkdownPostprocessor[]; + readonly metadata?: Readonly>; +} - this.name = "PandocError"; - this.code = code; - this.guidance = guidance; - } +export interface DocxToMarkdownResult { + /** GitHub-Flavored Markdown, after every postprocessor has run. */ + readonly markdown: string; + /** + * What the DOCX contained that the Markdown does not. Check + * `report.unmappable` before writing this Markdown back over a source note. + */ + readonly report: DocxImportReport; } export async function doctor(options: PandocDoctorOptions = {}): Promise { @@ -289,7 +282,10 @@ export async function convertMarkdownToDocx( throw new PandocError( "PANDOC_CONVERSION_FAILED", "Pandoc failed while converting Markdown to DOCX.", - conversionGuidance(result.stderr), + conversionGuidance( + "Confirm the Markdown input and reference DOCX are valid, then retry.", + result.stderr, + ), ); } @@ -308,6 +304,141 @@ export async function convertMarkdownToDocx( } } +/** + * Converts a Word document into clean GitHub-Flavored Markdown. + * + * The returned report is informational, never thrown: the caller is expected to + * be a review step with a human in the loop, so it decides what to do about the + * content Pandoc could not carry across. Nothing is silently discarded. + */ +export async function convertDocxToMarkdown( + options: ConvertDocxToMarkdownOptions, +): Promise { + // Snapshot before the first await. The report and the bytes handed to Pandoc + // must describe the same document, so caller mutation cannot be allowed to + // race the conversion. + const docxInput = snapshotDocxInput(options.docx); + const pandoc = await doctor(options); + + if (!pandoc.ok) { + throw pandocFailureToError(pandoc); + } + + const docx = await readDocxInput(docxInput); + // Fails closed on anything that is not a readable Word document, so Pandoc is + // never handed garbage, and inventories the losses before conversion. It also + // validates trackChanges, and the argv below uses the mode it echoes back, so + // the report cannot disagree with the flag Pandoc actually ran with. + const report = buildDocxImportReport(docx, options.trackChanges ?? DEFAULT_DOCX_TRACK_CHANGES); + const runner = options.runner ?? defaultPandocRunner; + const tempDirectory = await mkdtemp(join(tmpdir(), "polydoc-core-")); + + try { + const inputPath = join(tempDirectory, "input.docx"); + const outputPath = join(tempDirectory, "output.md"); + await writeFile(inputPath, docx); + + let result: PandocRunnerResult; + + try { + result = await runner( + pandoc.binary, + [ + "--from", + "docx", + "--to", + "gfm", + "--track-changes", + report.trackChanges, + "--output", + outputPath, + inputPath, + ], + { reject: false }, + ); + } catch (cause) { + if (cause instanceof PandocError) { + throw cause; + } + + throw new PandocError( + "PANDOC_CONVERSION_FAILED", + "Pandoc execution failed while converting DOCX to Markdown.", + "Confirm pandocPath points at a working Pandoc 3.x binary and retry the conversion.", + { cause }, + ); + } + + if (result.exitCode !== 0) { + throw new PandocError( + "PANDOC_CONVERSION_FAILED", + "Pandoc failed while converting DOCX to Markdown.", + conversionGuidance( + "The archive passed DOCX validation, so this is a Pandoc reader failure rather than a malformed file; retry with a Word-generated copy of the document.", + result.stderr, + ), + ); + } + + let markdown: string; + + try { + markdown = await readFile(outputPath, "utf8"); + } catch (cause) { + throw new PandocError( + "PANDOC_CONVERSION_FAILED", + "Pandoc completed but the Markdown output could not be read.", + "Check that the conversion process can write to the system temp directory and that disk space is available.", + { cause }, + ); + } + + return { + markdown: await applyMarkdownPostprocessors( + markdown, + options.postprocessors, + options.metadata, + ), + report, + }; + } finally { + await rm(tempDirectory, { force: true, recursive: true }); + } +} + +/** + * Copies caller-owned DOCX bytes. Anything that is not a `Uint8Array` passes + * through untouched so input validation can report it with a typed error. + */ +function snapshotDocxInput(docx: Uint8Array | string): Uint8Array | string { + return docx instanceof Uint8Array ? docx.slice() : docx; +} + +async function readDocxInput(docx: Uint8Array | string): Promise { + if (typeof docx !== "string") { + return docx; + } + + if (docx.trim() === "") { + throw new PandocError( + "DOCX_INPUT_INVALID", + "A DOCX path is required when docx is passed as a string.", + "Pass the path to a readable .docx file, or pass the DOCX bytes as a Uint8Array.", + ); + } + + try { + return await readFile(docx); + } catch (cause) { + throw new PandocError( + "DOCX_INPUT_INVALID", + `The DOCX file ${basename(docx)} could not be read.`, + "Pass the path to a readable .docx file, or pass the DOCX bytes as a Uint8Array.", + { cause }, + ); + } +} + async function defaultPandocRunner( binary: string, args: readonly string[], @@ -484,11 +615,12 @@ function joinOutput(stdout: string, stderr: string): string { return [stdout, stderr].filter((text) => text.length > 0).join("\n"); } -function conversionGuidance(stderr: string): string { +/** Appends bounded Pandoc stderr to direction-specific guidance. */ +function conversionGuidance(advice: string, stderr: string): string { const trimmed = stderr.trim(); const stderrSuffix = trimmed.length > 0 ? ` Pandoc stderr: ${trimmed.slice(0, 800)}` : ""; - return `Confirm the Markdown input and reference DOCX are valid, then retry.${stderrSuffix}`; + return `${advice}${stderrSuffix}`; } function isNotFoundError(error: unknown): boolean { diff --git a/tests/fixtures/golden/README.md b/tests/fixtures/golden/README.md index 518ed23..b935a91 100644 --- a/tests/fixtures/golden/README.md +++ b/tests/fixtures/golden/README.md @@ -13,5 +13,23 @@ Current paths: IDs, bookmark numeric IDs, relationship numeric IDs, and whitespace-only layout differences while preserving document structure, styles, text, and links. +- `roundtrip/basic-note/expected.md` - the Markdown produced by feeding + `publish/basic-note/input.md` through `convertMarkdownToDocx()` and then + `convertDocxToMarkdown()`. The two fixtures deliberately share one source + note so the round-trip golden stays comparable to the publish golden. + + This fixture is a record of what the round trip actually loses, not an + aspiration. Compared with the source note it has no YAML frontmatter, its + `[[wikilinks]]` survive only as backslash-escaped literal text, its callout + marker is escaped and the callout's internal line break is folded into one + blockquote paragraph, and paragraphs are re-wrapped at Pandoc's default + column width. A consumer restores that layer with its own sidecar through a + `MarkdownPostprocessor`; the library does not invent it. + + Regenerate it only from real Pandoc output, and only alongside a check that + the losses it records are still the losses Pandoc produces. The neighbouring + test asserts the frontmatter and wikilink losses directly so this file cannot + be quietly regenerated into something that hides one. + - `../reference/reference.docx` - the required Pandoc reference document used by integration tests for deterministic DOCX generation. diff --git a/tests/fixtures/golden/roundtrip/basic-note/expected.md b/tests/fixtures/golden/roundtrip/basic-note/expected.md new file mode 100644 index 0000000..c92075a --- /dev/null +++ b/tests/fixtures/golden/roundtrip/basic-note/expected.md @@ -0,0 +1,7 @@ +# Basic TeamWiki Note + +This note links to \[\[TeamWiki\]\] and includes an ordinary Markdown +link to [polydoc](https://github.com/agentic-tooling/polydoc-core). + +> \[!note\] Publish target The conversion core will eventually preserve +> this callout through the Markdown-to-docx publish path. diff --git a/tests/index.test.ts b/tests/index.test.ts index 1d8201b..5557521 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -2,18 +2,29 @@ import { chmod, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/pr import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { strFromU8, unzipSync } from "fflate"; +import { strFromU8, strToU8, unzipSync, zipSync } from "fflate"; import { OAuth2Client } from "google-auth-library"; import { afterEach, describe, expect, it, vi } from "vitest"; import { applyMarkdownPostprocessors, applyMarkdownPreprocessors, + buildDocxImportReport, type ConvertMarkdownToDocxOptions, + convertDocxToMarkdown, convertMarkdownToDocx, createSharePointClientSecretAccessTokenProvider, + DEFAULT_DOCX_TRACK_CHANGES, DEFAULT_SOURCE_DATE_EPOCH, + DOCX_MAX_ARCHIVE_ENTRIES, + DOCX_MAX_HEADER_FOOTER_PARTS, + DOCX_MAX_PART_BYTES, + DOCX_MAX_TOTAL_BYTES, DOCX_MIME_TYPE, + DOCX_TRACK_CHANGES_MODES, + type DocxImportReport, + type DocxUnmappableItem, + type DocxUnmappableKind, defaultGoogleDocName, doctor, encodeSharePointRelativePath, @@ -32,6 +43,7 @@ import { MICROSOFT_GRAPH_DEFAULT_SCOPE, PandocError, type PandocRunner, + type PandocRunnerOptions, SHAREPOINT_REQUIRED_APPLICATION_PERMISSION, SHAREPOINT_SIMPLE_UPLOAD_MAX_BYTES, type SharePointAccessTokenProvider, @@ -54,6 +66,14 @@ const fixtureInputPath = "tests/fixtures/golden/publish/basic-note/input.md"; const fixtureExpectedDocumentXmlPath = "tests/fixtures/golden/publish/basic-note/expected.document.xml"; const fixtureReferenceDocxPath = "tests/fixtures/reference/reference.docx"; +const fixtureRoundTripExpectedPath = "tests/fixtures/golden/roundtrip/basic-note/expected.md"; +/** Smallest valid 1x1 PNG, used as embedded DOCX media. */ +const TINY_PNG = Uint8Array.from( + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", + "base64", + ), +); const realPandocProbe = await doctor(); const canRepresentUnreadableFiles = process.platform !== "win32" && process.getuid?.() !== 0; @@ -542,6 +562,931 @@ describe("markdown to DOCX conversion", () => { }); }); +describe("DOCX import report", () => { + it("reports nothing for a clean document, including Pandoc's always-present empty comments part", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "word/document.xml": documentXml("Plain text."), + // Every Pandoc-generated DOCX carries this part with no comment bodies, + // so its presence alone must never be reported as a loss. + "word/comments.xml": '', + }), + ); + + expect(report).toEqual({ + unmappable: [], + hasUnmappableContent: false, + trackChanges: "accept", + trackedChanges: { insertions: 0, deletions: 0, formattingChanges: 0 }, + documentPart: "word/document.xml", + scannedParts: ["word/document.xml"], + }); + }); + + it("counts comment bodies rather than the comments part or its root element", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "word/comments.xml": [ + '', + 'First.', + 'Second.', + "", + ].join(""), + }), + ); + + expect(findUnmappable(report, "comment")).toMatchObject({ + kind: "comment", + count: 2, + summary: expect.stringContaining("--track-changes=all"), + }); + }); + + it("counts tracked insertions and deletions without matching lookalike elements", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "word/document.xml": documentXml( + [ + "", + // `w:insideH`/`w:insideV` are table borders, `w:delText` is the body + // of a deletion, and `w:moveFromRangeStart` is a range marker, so + // none of them is a revision mark itself. + '', + "", + "", + 'added', + 'also added', + '', + 'removed', + "", + ].join(""), + ), + }), + ); + + expect(report.trackedChanges).toEqual({ + insertions: 2, + deletions: 1, + formattingChanges: 0, + }); + expect(findUnmappable(report, "tracked-change")).toMatchObject({ + kind: "tracked-change", + count: 3, + summary: expect.stringContaining("--track-changes=accept"), + }); + }); + + it("counts tracked moves, which Word writes as neither w:ins nor w:del", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "word/document.xml": documentXml( + [ + "", + 'moved away', + 'moved here', + "", + ].join(""), + ), + }), + ); + + // Verified against Pandoc 3.10: --track-changes=all renders w:moveFrom as + // class="deletion" and w:moveTo as class="insertion", so a reorganized + // document loses its moveFrom text under accept exactly like a deletion. + expect(report.trackedChanges).toMatchObject({ insertions: 1, deletions: 1 }); + expect(findUnmappable(report, "tracked-change")).toBeDefined(); + }); + + it("counts formatting-only revisions, which no track-changes mode preserves", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "word/document.xml": documentXml( + 'styled', + ), + }), + "all", + ); + + expect(report.trackedChanges).toMatchObject({ formattingChanges: 1 }); + expect(findUnmappable(report, "tracked-change")).toMatchObject({ + count: 1, + summary: expect.stringContaining("no representation for a formatting-only revision"), + }); + }); + + it("classifies text boxes by whether a fallback exists, not by namespace", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "word/document.xml": documentXml( + [ + // Word 2010+ writes DrawingML plus a VML mc:Fallback. Verified + // against Pandoc 3.10: the fallback is read, so the text survives. + "", + 'Modern', + "Modern", + "", + // Bare legacy VML is also inlined. + "Legacy", + // DrawingML with no fallback is the only form actually dropped. + "", + 'Orphan', + "", + // Bare DrawingML, no wrapper at all, is dropped too. + "Bare", + ].join(""), + ), + }), + ); + + expect(findUnmappable(report, "text-box")).toMatchObject({ + kind: "text-box", + count: 4, + summary: expect.stringContaining("2 inlined as ordinary body paragraphs"), + }); + expect(findUnmappable(report, "text-box")?.summary).toContain("2 dropped outright"); + }); + + it("counts a text box once when an AlternateContent is nested inside a fallback", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "word/document.xml": documentXml( + [ + "", + 'Outer', + // A non-greedy paired-tag pattern would stop at the inner closing + // tag here, orphan the outer one, and leak un-stripped content. + "Outer", + "", + ].join(""), + ), + }), + ); + + expect(findUnmappable(report, "text-box")).toMatchObject({ count: 1 }); + }); + + it("reports embedded media and OLE objects with their sorted archive part names", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "word/media/image2.png": new Uint8Array([2]), + "word/media/image1.png": new Uint8Array([1]), + "word/embeddings/oleObject1.bin": new Uint8Array([3]), + }), + ); + + expect(findUnmappable(report, "embedded-media")).toMatchObject({ + kind: "embedded-media", + count: 2, + entries: ["word/media/image1.png", "word/media/image2.png"], + summary: expect.stringContaining("--extract-media"), + }); + expect(findUnmappable(report, "embedded-object")).toMatchObject({ + kind: "embedded-object", + count: 1, + entries: ["word/embeddings/oleObject1.bin"], + }); + }); + + it("reports headers and footers that hold text, and ignores empty boilerplate ones", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "word/header1.xml": + 'Running title.', + "word/footer1.xml": + 'Page footer.', + // No w:t anywhere, so this one is Word furniture rather than content. + "word/header2.xml": '', + }), + ); + + expect(findUnmappable(report, "header-footer")).toMatchObject({ + kind: "header-footer", + count: 2, + entries: ["word/footer1.xml", "word/header1.xml"], + summary: expect.stringContaining("ignores headers and footers entirely"), + }); + }); + + it("counts revisions inside footnotes and endnotes, whose text does reach the Markdown", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "word/footnotes.xml": [ + '', + 'cut from a footnote', + "", + ].join(""), + "word/endnotes.xml": [ + '', + 'added to an endnote', + "", + ].join(""), + }), + ); + + // Verified against Pandoc 3.10: a tracked deletion inside a footnote is + // resolved away exactly like one in the body, so scanning only the document + // part would let it vanish with a clean report. + expect(report.scannedParts).toEqual([ + "word/document.xml", + "word/footnotes.xml", + "word/endnotes.xml", + ]); + expect(report.trackedChanges).toMatchObject({ insertions: 1, deletions: 1 }); + }); + + it("reports SmartArt and charts, which live outside the media and embeddings prefixes", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "word/diagrams/data1.xml": '', + "word/diagrams/layout1.xml": '', + "word/charts/chart1.xml": '', + "word/charts/style1.xml": '', + }), + ); + + // One SmartArt graphic spans several parts, so the data part is what makes + // it countable; the same holds for a chart. + expect(findUnmappable(report, "smart-art")).toMatchObject({ + kind: "smart-art", + count: 1, + entries: ["word/diagrams/data1.xml", "word/diagrams/layout1.xml"], + }); + expect(findUnmappable(report, "chart")).toMatchObject({ + kind: "chart", + count: 1, + entries: ["word/charts/chart1.xml", "word/charts/style1.xml"], + }); + }); + + it("reports altChunk imports, whose embedded document Pandoc never reads", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "word/document.xml": documentXml(''), + }), + ); + + expect(findUnmappable(report, "external-content")).toMatchObject({ + kind: "external-content", + count: 2, + summary: expect.stringContaining("altChunk"), + }); + }); + + it("resolves the main document part from the package relationships", () => { + // Verified against Pandoc 3.10: it converts this document fine, so assuming + // word/document.xml would reject a valid Word file as "not a Word document". + const report = buildDocxImportReport( + createDocxArchive({ + "_rels/.rels": [ + '', + '', + "", + ].join(""), + "word/document2.xml": documentXml( + 'added', + ), + "word/document.xml": undefined, + }), + ); + + expect(report.documentPart).toBe("word/document2.xml"); + expect(report.scannedParts).toEqual(["word/document2.xml"]); + expect(report.trackedChanges).toMatchObject({ insertions: 1 }); + }); + + it("refuses to inflate an oversized part instead of exhausting memory", () => { + // A small archive whose main part inflates past the per-part cap: a + // compression ratio near 300:1. Without the cap an earlier build of this + // shape allocated 242 MB of resident memory in 545 ms, scaling linearly + // with the input. Sized just over the cap so the fixture stays cheap. + const bomb = zipSync( + { "word/document.xml": createRepetitiveXmlPart(DOCX_MAX_PART_BYTES + 1024) }, + { level: 1 }, + ); + + expect(bomb.byteLength).toBeLessThan(2 * 1024 * 1024); + expect(() => buildDocxImportReport(bomb)).toThrow( + expect.objectContaining({ + name: "PandocError", + code: "DOCX_ARCHIVE_INVALID", + message: expect.stringContaining("over the"), + }), + ); + }, 30_000); + + it("refuses many individually plausible parts that exceed the total budget", () => { + // The shape neither other limit catches: every part is honestly declared + // and comfortably under the per-part cap, and the entry count is modest, + // but the aggregate blows past the budget. Enumerating header parts is + // what removed the fixed-part-set invariant that used to make the + // per-part cap an aggregate cap, so this is the test that holds the + // budget in place. + // + // The fixture is the least data that can prove that: just over the + // budget, spread across the fewest parts, at the cheapest compression + // level. It still costs ~1.3s because ~128 MB genuinely has to be + // deflated, hence the explicit timeout below rather than silently + // sitting near the default. + const partCount = 10; + const part = createRepetitiveXmlPart(Math.ceil(DOCX_MAX_TOTAL_BYTES / partCount) + 1024); + const entries: Record = { + "word/document.xml": strToU8(documentXml("")), + }; + + for (let index = 1; index <= partCount; index += 1) { + entries[`word/header${index}.xml`] = part; + } + + // Every other guard must be satisfied, or this would prove nothing about + // the budget specifically. + expect(part.byteLength).toBeLessThan(DOCX_MAX_PART_BYTES); + expect(partCount).toBeLessThanOrEqual(DOCX_MAX_HEADER_FOOTER_PARTS); + expect(Object.keys(entries).length).toBeLessThan(DOCX_MAX_ARCHIVE_ENTRIES); + expect(part.byteLength * partCount).toBeGreaterThan(DOCX_MAX_TOTAL_BYTES); + + expect(() => buildDocxImportReport(zipSync(entries, { level: 1 }))).toThrow( + expect.objectContaining({ + name: "PandocError", + code: "DOCX_ARCHIVE_INVALID", + message: expect.stringContaining("in total"), + }), + ); + }, 30_000); + + it("refuses an implausible number of header and footer parts", () => { + const entries: Record = { + "word/document.xml": strToU8(documentXml("")), + }; + + for (let index = 0; index <= DOCX_MAX_HEADER_FOOTER_PARTS; index += 1) { + entries[`word/header${index}.xml`] = strToU8(''); + } + + expect(() => buildDocxImportReport(zipSync(entries))).toThrow( + expect.objectContaining({ + name: "PandocError", + code: "DOCX_ARCHIVE_INVALID", + message: expect.stringContaining("header and footer parts"), + }), + ); + }); + + // Both levels matter. A stored entry is sliced using its COMPRESSED size, so + // an understated uncompressed size is visible as a length mismatch. A deflate + // entry — every real .docx, and anything an attacker would build — is + // inflated into a buffer pre-sized from that same understated number, so the + // inflated length always equals it and a length comparison is tautological. + // Only the CRC catches the deflate case. + it.each([ + ["stored", 0], + ["deflate", 9], + ])( + "refuses a %s part whose entry understates its real size instead of analyzing a fragment", + (_label, level) => { + // The revision sits past byte 5000, so a 512-byte fragment hides it and + // the report would come back clean for a document Pandoc reads in full. + const body = documentXml( + `${"padding ".repeat( + 700, + )}SECRET-INSERTION`, + ); + const archive = zipSync({ "word/document.xml": strToU8(body) }, { level: level as 0 | 9 }); + const view = new DataView(archive.buffer, archive.byteOffset, archive.byteLength); + let patched = false; + + // Rewrite the uncompressed-size field in both the local header and the + // central directory record so the entry claims far less than it holds. + for (let offset = 0; offset + 4 <= archive.byteLength; offset += 1) { + const signature = view.getUint32(offset, true); + + if (signature === 0x0403_4b50) { + view.setUint32(offset + 22, 512, true); + patched = true; + } else if (signature === 0x0201_4b50) { + view.setUint32(offset + 24, 512, true); + patched = true; + } + } + + expect(patched).toBe(true); + expect(() => buildDocxImportReport(archive)).toThrow( + expect.objectContaining({ + name: "PandocError", + code: "DOCX_ARCHIVE_INVALID", + }), + ); + }, + ); + + it("catches the deflate truncation by checksum, the only check that can", () => { + // Pinning the mechanism, not just the rejection: for a deflate entry the + // inflated length always equals the declared size, so if this ever starts + // failing on the size check instead, the CRC has stopped doing the work. + const body = documentXml( + `${"padding ".repeat( + 700, + )}SECRET-INSERTION`, + ); + const archive = zipSync({ "word/document.xml": strToU8(body) }, { level: 9 }); + const view = new DataView(archive.buffer, archive.byteOffset, archive.byteLength); + + for (let offset = 0; offset + 4 <= archive.byteLength; offset += 1) { + const signature = view.getUint32(offset, true); + + if (signature === 0x0403_4b50) { + view.setUint32(offset + 22, 512, true); + } else if (signature === 0x0201_4b50) { + view.setUint32(offset + 24, 512, true); + } + } + + expect(() => buildDocxImportReport(archive)).toThrow( + expect.objectContaining({ + code: "DOCX_ARCHIVE_INVALID", + message: expect.stringContaining("CRC-32"), + }), + ); + }); + + it("accepts an honest archive at every compression level the CRC check sees", () => { + // A checksum check that false-positives on real Word output would be worse + // than the hole it closes, so both storage modes are asserted to pass. + for (const level of [0, 6, 9] as const) { + const archive = zipSync( + { + "word/document.xml": strToU8(documentXml("Plain text.")), + "word/comments.xml": strToU8(''), + "word/footnotes.xml": strToU8(''), + }, + { level }, + ); + + expect(buildDocxImportReport(archive).unmappable).toEqual([]); + } + }); + + it("resolves a main document part named with single-quoted attributes", () => { + const report = buildDocxImportReport( + createDocxArchive({ + "_rels/.rels": [ + "", + "", + "", + ].join(""), + "word/document2.xml": documentXml("Body."), + "word/document.xml": undefined, + }), + ); + + expect(report.documentPart).toBe("word/document2.xml"); + }); + + it("explains that the relationships named no usable target, not that a part is missing", () => { + expect(() => + buildDocxImportReport( + createDocxArchive({ + "_rels/.rels": [ + '', + '', + "", + ].join(""), + "word/document.xml": undefined, + }), + ), + ).toThrow( + expect.objectContaining({ + name: "PandocError", + code: "DOCX_ARCHIVE_INVALID", + message: expect.stringContaining("no officeDocument target present in the archive"), + }), + ); + }); + + it("refuses an archive with an implausible number of entries", () => { + const entries: Record = {}; + + for (let index = 0; index <= DOCX_MAX_ARCHIVE_ENTRIES; index += 1) { + entries[`word/media/image${index}.png`] = new Uint8Array([0]); + } + + expect(() => buildDocxImportReport(zipSync(entries))).toThrow( + expect.objectContaining({ + name: "PandocError", + code: "DOCX_ARCHIVE_INVALID", + message: expect.stringContaining("entries"), + }), + ); + }); + + it("surfaces every unmappable kind at once for a document that has all of them", () => { + const report = buildDocxImportReport(createKitchenSinkDocx()); + + expect(report.hasUnmappableContent).toBe(true); + expect(report.unmappable.map((item) => item.kind)).toEqual([ + "tracked-change", + "comment", + "header-footer", + "text-box", + "external-content", + "smart-art", + "chart", + "embedded-media", + "embedded-object", + ]); + }); + + it("stops reporting comments and revisions as losses when Pandoc keeps them as spans", () => { + const report = buildDocxImportReport(createKitchenSinkDocx(), "all"); + + // Verified against Pandoc 3.10: --track-changes=all emits insertion, + // deletion, comment-start, and comment-end spans instead of dropping them, + // so neither is a loss in that mode. The counts stay on the report, and + // every loss that does not depend on the mode still reports. + expect(report.unmappable.map((item) => item.kind)).toEqual([ + "header-footer", + "text-box", + "external-content", + "smart-art", + "chart", + "embedded-media", + "embedded-object", + ]); + expect(report.trackChanges).toBe("all"); + expect(report.trackedChanges).toMatchObject({ insertions: 1, deletions: 1 }); + }); + + it.each([ + [ + "a manual page break, which is layout rather than content", + documentXml(''), + ], + [ + "direct character formatting with no Markdown equivalent", + documentXml( + 'styled', + ), + ], + ])("does not claim to detect %s", (_label, body) => { + const report = buildDocxImportReport(createDocxArchive({ "word/document.xml": body })); + + // These are real fidelity losses that this report deliberately does not + // cover. The test exists so the gap stays a decision rather than becoming + // an accidental promise that an empty `unmappable` means a lossless import. + expect(report.unmappable).toEqual([]); + }); + + it("reports insertions as discarded when revisions are rejected", () => { + const report = buildDocxImportReport(createKitchenSinkDocx(), "reject"); + + expect(findUnmappable(report, "tracked-change")?.summary).toContain( + "deletions were restored as ordinary text and insertions were discarded", + ); + }); + + it.each([ + ["empty bytes", new Uint8Array()], + ["plain text that is not a ZIP", strToU8("# This is Markdown, not a DOCX")], + ["a legacy .doc OLE compound file header", new Uint8Array([0xd0, 0xcf, 0x11, 0xe0, 0xa1])], + ])("rejects %s as invalid DOCX input", (_label, bytes) => { + expect(() => buildDocxImportReport(bytes)).toThrow( + expect.objectContaining({ + name: "PandocError", + code: "DOCX_INPUT_INVALID", + }), + ); + }); + + it("rejects a ZIP archive that is not a Word document", () => { + expect(() => buildDocxImportReport(zipSync({ mimetype: strToU8("text/plain") }))).toThrow( + expect.objectContaining({ + name: "PandocError", + code: "DOCX_ARCHIVE_INVALID", + message: expect.stringContaining("word/document.xml"), + }), + ); + }); + + it("rejects a truncated ZIP archive", () => { + const truncated = createDocxArchive().slice(0, 40); + + expect(() => buildDocxImportReport(truncated)).toThrow( + expect.objectContaining({ + name: "PandocError", + code: "DOCX_ARCHIVE_INVALID", + }), + ); + }); + + it.each(["", "ACCEPT", "yes", 1, null])("rejects invalid trackChanges value %j", (mode) => { + expect(() => buildDocxImportReport(createDocxArchive(), mode as unknown as "accept")).toThrow( + expect.objectContaining({ + name: "PandocError", + code: "DOCX_TRACK_CHANGES_INVALID", + }), + ); + }); + + it("exposes the supported track-changes modes and the clean-Markdown default", () => { + expect(DOCX_TRACK_CHANGES_MODES).toEqual(["accept", "reject", "all"]); + expect(DEFAULT_DOCX_TRACK_CHANGES).toBe("accept"); + }); +}); + +describe("DOCX to Markdown conversion", () => { + it("fails closed when Pandoc is unsupported before touching the DOCX input", async () => { + const runner: PandocRunner = vi.fn(async () => ({ + stdout: "pandoc 2.19", + stderr: "", + exitCode: 0, + })); + + await expect( + convertDocxToMarkdown({ docx: createDocxArchive(), runner }), + ).rejects.toMatchObject({ + name: "PandocError", + code: "PANDOC_UNSUPPORTED_MAJOR", + }); + expect(runner).toHaveBeenCalledTimes(1); + }); + + it("uses argument arrays with an explicit track-changes value and no writer env", async () => { + const calls: Array<{ + binary: string; + args: readonly string[]; + options: PandocRunnerOptions | undefined; + }> = []; + const docx = createDocxArchive(); + const runner: PandocRunner = vi.fn(async (binary, args, options) => { + calls.push({ binary, args, options }); + + if (args[0] === "--version") { + return { stdout: "pandoc 3.10\nFeatures: +lua", stderr: "", exitCode: 0 }; + } + + const outputPath = args[args.indexOf("--output") + 1]; + const inputPath = args.at(-1); + + if (outputPath === undefined || inputPath === undefined) { + throw new Error("test runner received an incomplete invocation"); + } + + // The bytes handed to Pandoc are the bytes the caller passed in. + expect(new Uint8Array(await readFile(inputPath))).toEqual(docx); + await writeFile(outputPath, "# Converted\n"); + + return { stdout: "", stderr: "", exitCode: 0 }; + }); + + const result = await convertDocxToMarkdown({ docx, runner }); + + expect(result.markdown).toBe("# Converted\n"); + expect(calls).toHaveLength(2); + expect(calls[1]).toEqual({ + binary: "pandoc", + args: [ + "--from", + "docx", + "--to", + "gfm", + "--track-changes", + "accept", + "--output", + expect.stringMatching(/output\.md$/), + expect.stringMatching(/input\.docx$/), + ], + options: { reject: false }, + }); + // SOURCE_DATE_EPOCH only matters to the DOCX writer; Markdown output has no + // embedded timestamps, so the reverse path passes no environment at all. + expect(calls[1]?.options?.env).toBeUndefined(); + }); + + it("passes an explicitly requested track-changes value through to Pandoc", async () => { + const runner = createReverseRunner("# Converted\n"); + + const result = await convertDocxToMarkdown({ + docx: createKitchenSinkDocx(), + trackChanges: "all", + runner, + }); + + expect(runner).toHaveBeenLastCalledWith( + "pandoc", + expect.arrayContaining(["--track-changes", "all"]), + { reject: false }, + ); + expect(result.report.trackChanges).toBe("all"); + }); + + it("rejects an invalid track-changes value before writing any conversion files", async () => { + const runner = createSuccessfulDoctorRunner(); + + await expect( + convertDocxToMarkdown({ + docx: createDocxArchive(), + trackChanges: "maybe" as unknown as "accept", + runner, + }), + ).rejects.toMatchObject({ + name: "PandocError", + code: "DOCX_TRACK_CHANGES_INVALID", + }); + expect(runner).toHaveBeenCalledTimes(1); + }); + + it("runs postprocessors in order with reverse Markdown context and returns the report", async () => { + const seen: string[] = []; + const result = await convertDocxToMarkdown({ + docx: createKitchenSinkDocx(), + metadata: { source: "fixture" }, + postprocessors: [ + (markdown, context) => { + seen.push(`${context.phase}:${context.sourceFormat}:${context.targetFormat}`); + return `${markdown}first\n`; + }, + async (markdown, context) => { + seen.push(`${context.phase}:${String(context.metadata.source)}`); + return `${markdown}second\n`; + }, + ], + runner: createReverseRunner("# Converted\n"), + }); + + expect(result.markdown).toBe("# Converted\nfirst\nsecond\n"); + expect(seen).toEqual(["postprocess:markdown:markdown", "postprocess:fixture"]); + // The report describes the DOCX, so postprocessors cannot mask a loss. + expect(result.report.hasUnmappableContent).toBe(true); + }); + + it("wraps postprocessor failures in typed hook errors", async () => { + await expect( + convertDocxToMarkdown({ + docx: createDocxArchive(), + postprocessors: [ + () => { + throw new Error("sidecar lookup exploded"); + }, + ], + runner: createReverseRunner("# Converted\n"), + }), + ).rejects.toMatchObject({ + name: "PandocError", + code: "MARKDOWN_HOOK_FAILED", + }); + }); + + it("reads a DOCX from a filesystem path", async () => { + const docxPath = join(await createTempDirectory(), "note.docx"); + await writeFile(docxPath, createDocxArchive()); + + const result = await convertDocxToMarkdown({ + docx: docxPath, + runner: createReverseRunner("# From path\n"), + }); + + expect(result.markdown).toBe("# From path\n"); + }); + + it.each([ + ["a missing path", "tests/fixtures/golden/missing-note.docx"], + ["a blank path", " "], + ])("rejects %s with a typed input error", async (_label, docx) => { + const runner = createSuccessfulDoctorRunner(); + + await expect(convertDocxToMarkdown({ docx, runner })).rejects.toMatchObject({ + name: "PandocError", + code: "DOCX_INPUT_INVALID", + }); + expect(runner).toHaveBeenCalledTimes(1); + }); + + it.each([ + ["garbage that is not a ZIP", strToU8("not a docx"), "DOCX_INPUT_INVALID"], + [ + "a ZIP that is not a Word document", + zipSync({ "a.txt": strToU8("hi") }), + "DOCX_ARCHIVE_INVALID", + ], + ])("rejects %s before invoking Pandoc's converter", async (_label, docx, code) => { + const runner = createSuccessfulDoctorRunner(); + + await expect(convertDocxToMarkdown({ docx, runner })).rejects.toMatchObject({ + name: "PandocError", + code, + }); + // Only the version probe ran, so garbage never reached the converter. + expect(runner).toHaveBeenCalledTimes(1); + }); + + it("cleans up temporary files when Pandoc conversion fails", async () => { + let tempOutputPath: string | undefined; + const runner: PandocRunner = vi.fn(async (_binary, args) => { + if (args[0] === "--version") { + return { stdout: "pandoc 3.10", stderr: "", exitCode: 0 }; + } + + tempOutputPath = args[args.indexOf("--output") + 1]; + + return { stdout: "", stderr: "unsupported docx feature", exitCode: 43 }; + }); + + await expect( + convertDocxToMarkdown({ docx: createDocxArchive(), runner }), + ).rejects.toMatchObject({ + name: "PandocError", + code: "PANDOC_CONVERSION_FAILED", + guidance: expect.stringContaining("Pandoc stderr: unsupported docx feature"), + }); + + expect(tempOutputPath).toBeDefined(); + await expect(stat(join(tempOutputPath ?? "", ".."))).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("wraps missing Markdown output after exit 0 and cleans up", async () => { + let tempOutputPath: string | undefined; + const runner: PandocRunner = vi.fn(async (_binary, args) => { + if (args[0] === "--version") { + return { stdout: "pandoc 3.10", stderr: "", exitCode: 0 }; + } + + tempOutputPath = args[args.indexOf("--output") + 1]; + + return { stdout: "", stderr: "", exitCode: 0 }; + }); + + await expect( + convertDocxToMarkdown({ docx: createDocxArchive(), runner }), + ).rejects.toMatchObject({ + name: "PandocError", + code: "PANDOC_CONVERSION_FAILED", + message: expect.stringContaining("Markdown output could not be read"), + }); + + expect(tempOutputPath).toBeDefined(); + await expect(stat(join(tempOutputPath ?? "", ".."))).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("snapshots DOCX bytes so caller mutation cannot desync the report from the output", async () => { + let convertedBytes: Uint8Array | undefined; + const docx = createKitchenSinkDocx(); + const runner: PandocRunner = vi.fn(async (_binary, args) => { + if (args[0] === "--version") { + return { stdout: "pandoc 3.10", stderr: "", exitCode: 0 }; + } + + const outputPath = args[args.indexOf("--output") + 1]; + const inputPath = args.at(-1); + + if (outputPath === undefined || inputPath === undefined) { + throw new Error("test runner received an incomplete invocation"); + } + + convertedBytes = new Uint8Array(await readFile(inputPath)); + await writeFile(outputPath, "# Converted\n"); + + return { stdout: "", stderr: "", exitCode: 0 }; + }); + const original = Uint8Array.from(docx); + + const conversion = convertDocxToMarkdown({ docx, runner }); + docx.fill(0); + const result = await conversion; + + expect(convertedBytes).toEqual(original); + expect(result.report.hasUnmappableContent).toBe(true); + }); + + it("wraps unexpected runner failures and preserves typed ones", async () => { + const typedError = new PandocError( + "PANDOC_CONVERSION_FAILED", + "Injected typed failure.", + "Caller guidance.", + ); + const throwingRunner = (cause: unknown): PandocRunner => + vi.fn(async (_binary, args) => { + if (args[0] === "--version") { + return { stdout: "pandoc 3.10", stderr: "", exitCode: 0 }; + } + + throw cause; + }); + + await expect( + convertDocxToMarkdown({ + docx: createDocxArchive(), + runner: throwingRunner(new Error("process runner exploded")), + }), + ).rejects.toMatchObject({ + name: "PandocError", + code: "PANDOC_CONVERSION_FAILED", + guidance: expect.stringContaining("pandocPath"), + }); + await expect( + convertDocxToMarkdown({ docx: createDocxArchive(), runner: throwingRunner(typedError) }), + ).rejects.toBe(typedError); + }); +}); + describe.skipIf(!realPandocProbe.ok)("Pandoc integration", () => { it("converts the golden Markdown fixture to deterministic DOCX bytes", async () => { const markdown = await readFile(fixtureInputPath, "utf8"); @@ -564,6 +1509,222 @@ describe.skipIf(!realPandocProbe.ok)("Pandoc integration", () => { const documentXml = normalizeDocumentXml(first); expect(`${documentXml}\n`).toBe(expectedDocumentXml); }); + + it("round-trips the golden note back to the golden Markdown fixture", async () => { + const markdown = await readFile(fixtureInputPath, "utf8"); + const expectedMarkdown = await readFile(fixtureRoundTripExpectedPath, "utf8"); + const docx = await convertMarkdownToDocx({ + markdown, + referenceDocxPath: fixtureReferenceDocxPath, + sourceDateEpoch: 1_704_067_200, + preprocessors: [stripYamlFrontmatter], + }); + + const result = await convertDocxToMarkdown({ docx }); + + expect(result.markdown).toBe(expectedMarkdown); + // A Pandoc-written DOCX carries an empty comments part and no media, so a + // clean publish round trip must report no losses at all. + expect(result.report).toEqual({ + unmappable: [], + hasUnmappableContent: false, + trackChanges: "accept", + trackedChanges: { insertions: 0, deletions: 0, formattingChanges: 0 }, + documentPart: "word/document.xml", + scannedParts: ["word/document.xml", "word/footnotes.xml"], + }); + }); + + it("documents the round-trip losses the golden fixture records", async () => { + const source = await readFile(fixtureInputPath, "utf8"); + const roundTripped = await readFile(fixtureRoundTripExpectedPath, "utf8"); + + // These assertions exist so the golden file cannot be quietly regenerated + // into something that hides a loss: the round trip really does drop + // frontmatter and really does destroy wikilink syntax. + expect(source).toContain("title: Basic TeamWiki Note"); + expect(roundTripped).not.toContain("title: Basic TeamWiki Note"); + expect(source).toContain("[[TeamWiki]]"); + expect(roundTripped).not.toContain("[[TeamWiki]]"); + expect(roundTripped).toContain(String.raw`\[\[TeamWiki\]\]`); + }); + + it("restores frontmatter and wikilinks from a consumer-provided sidecar", async () => { + const sourceNote = [ + "---", + "title: Sidecar Note", + "tags:", + " - teamwiki", + "---", + "", + "# Sidecar Note", + "", + "This note links to [[TeamWiki]] and [[Handbook]].", + "", + "A second paragraph with **bold** text.", + "", + "- First item", + "- Second item", + "", + ].join("\n"); + // This note is deliberately built to survive Pandoc's re-wrap: short lines, + // no callout, no paragraph long enough to be reflowed. That is what makes a + // byte-exact assertion possible. The test proves the postprocessor hook + // runs and can restore the Obsidian layer, NOT that byte-exact restoration + // is achievable for an arbitrary note — the golden round-trip fixture next + // to it shows how much a realistic note actually loses. + // + // The sidecar is the consumer's, not the library's: publishing records what + // the DOCX cannot carry, and the reverse import puts it back. + const sidecar: { frontmatter: string; wikilinks: string[] } = { + frontmatter: "", + wikilinks: [], + }; + + const docx = await convertMarkdownToDocx({ + markdown: sourceNote, + referenceDocxPath: fixtureReferenceDocxPath, + sourceDateEpoch: 1_704_067_200, + preprocessors: [ + (markdown) => { + const match = /^---\n[\s\S]*?\n---\n\n?/.exec(markdown); + sidecar.frontmatter = match?.[0] ?? ""; + + return markdown.slice(sidecar.frontmatter.length); + }, + (markdown) => + markdown.replace(/\[\[([^\]]+)\]\]/g, (_match, target: string) => { + sidecar.wikilinks.push(target); + + return target; + }), + ], + }); + + const result = await convertDocxToMarkdown({ + docx, + postprocessors: [ + (markdown) => { + let restored = markdown; + + // replaceAll, not replace: a string first argument to replace() + // rewrites only the first match, so a target that also appears + // earlier as ordinary prose would capture the wrapping and leave the + // real wikilink bare. Deduplicating targets keeps a repeated link + // from being wrapped twice. + // + // A production consumer needs more than this: a bare target string is + // indistinguishable from prose that happens to use the same words, so + // the sidecar should record source offsets rather than just names. + for (const target of new Set(sidecar.wikilinks)) { + restored = restored.replaceAll(target, `[[${target}]]`); + } + + return restored; + }, + (markdown) => `${sidecar.frontmatter}${markdown}`, + ], + }); + + expect(sidecar.wikilinks).toEqual(["TeamWiki", "Handbook"]); + expect(result.markdown).toBe(sourceNote); + expect(result.report.hasUnmappableContent).toBe(false); + }); + + it("surfaces every unmappable construct from a real DOCX", async () => { + const result = await convertDocxToMarkdown({ docx: createKitchenSinkDocx() }); + + expect(result.report.unmappable.map((item) => item.kind)).toEqual([ + "tracked-change", + "comment", + "header-footer", + "text-box", + "external-content", + "smart-art", + "chart", + "embedded-media", + "embedded-object", + ]); + // These assertions pin the behavior each summary claims, so a Pandoc + // upgrade that changes any of it fails here rather than leaving the report + // quietly wrong. + // + // Accepted: the insertion became ordinary text, the deletion is gone. + expect(result.markdown).toContain("INSERTED-TEXT"); + expect(result.markdown).not.toContain("DELETED-TEXT"); + // Dropped outright, each with no placeholder of any kind. + expect(result.markdown).not.toContain("First reviewer comment."); + expect(result.markdown).not.toContain("EMBEDDED-OLE-OBJECT"); + expect(result.markdown).not.toContain("HEADER-TEXT-CONTENT"); + expect(result.markdown).not.toContain("SMARTART-NODE-TEXT"); + expect(result.markdown).not.toContain("CHART-TITLE-TEXT"); + expect(result.markdown).not.toContain("ALTCHUNK-IMPORTED-TEXT"); + // Inlined, losing its text-box framing rather than its text. + expect(result.markdown).toContain("TEXTBOX-CONTENT"); + // Referenced but never written: the image link dangles. + expect(result.markdown).toContain("media/image1.png"); + }); + + it.each([ + [ + "a DrawingML text box with the mc:Fallback Word 2010+ writes", + "" + + 'TEXTBOX-PROBE' + + 'TEXTBOX-PROBE' + + "", + 1, + 0, + true, + ], + [ + "a DrawingML text box with no fallback", + "" + + 'TEXTBOX-PROBE' + + "", + 0, + 1, + false, + ], + [ + "a bare legacy VML text box", + 'TEXTBOX-PROBE', + 1, + 0, + true, + ], + ])( + "classifies %s the way Pandoc actually treats it", + async (_label, body, inlined, dropped, textSurvives) => { + const result = await convertDocxToMarkdown({ + docx: createDocxArchive({ "word/document.xml": documentXml(body) }), + }); + + // The report's classification and Pandoc's real behavior are asserted + // against each other. This case is what caught the classification being + // inverted: the fallback, not the namespace, decides. + expect(findUnmappable(result.report, "text-box")?.summary).toContain( + `${inlined} inlined as ordinary body paragraphs`, + ); + expect(findUnmappable(result.report, "text-box")?.summary).toContain( + `${dropped} dropped outright`, + ); + expect(result.markdown.includes("TEXTBOX-PROBE")).toBe(textSurvives); + }, + ); + + it("reports a header that Pandoc drops, so an empty report never hides one", async () => { + const result = await convertDocxToMarkdown({ + docx: createDocxArchive({ + "word/header1.xml": `HEADER-TEXT-CONTENT`, + }), + }); + + expect(result.markdown).not.toContain("HEADER-TEXT-CONTENT"); + expect(findUnmappable(result.report, "header-footer")).toMatchObject({ + count: 1, + entries: ["word/header1.xml"], + }); + }); }); describe.skipIf(realPandocProbe.ok)("Pandoc integration skip behavior", () => { @@ -1858,6 +3019,161 @@ describe("Google Drive transport", () => { }); }); +const DOCX_NAMESPACES = [ + 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"', + 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"', + 'xmlns:o="urn:schemas-microsoft-com:office:office"', + 'xmlns:v="urn:schemas-microsoft-com:vml"', + 'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"', + 'xmlns:pic="http://schemas.openxmlformats.org/drawingml/2006/picture"', + 'xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"', + 'xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"', + 'xmlns:wps="http://schemas.microsoft.com/office/word/2010/wordprocessingShape"', +].join(" "); + +function documentXml(body: string): string { + return `${body}`; +} + +/** + * Builds a real DOCX archive in memory. The default parts are the minimum a + * Word document needs, so tests can add exactly the one construct they are + * about without hand-rolling a package each time. + */ +function createDocxArchive( + parts: Readonly> = {}, +): Uint8Array { + const defaults: Record = { + "[Content_Types].xml": [ + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + "", + ].join(""), + "_rels/.rels": [ + '', + '', + '', + "", + ].join(""), + "word/_rels/document.xml.rels": [ + '', + '', + '', + '', + '', + '', + '', + '', + '', + "", + ].join(""), + "word/document.xml": documentXml("Plain text."), + }; + const files: Record = {}; + + for (const [name, content] of Object.entries({ ...defaults, ...parts })) { + // An explicit undefined removes a default part, so a test can build an + // archive that deliberately lacks one. + if (content !== undefined) { + files[name] = typeof content === "string" ? strToU8(content) : content; + } + } + + return zipSync(files); +} + +/** + * A well-formed, highly repetitive XML part of at least `minimumBytes`. + * + * The archive-limit tests have to hand the inspector real, honestly declared + * data to be worth anything, so their fixtures are sized from the limit they + * probe rather than from a hardcoded number. That keeps them minimal — building + * one is dominated by deflating the bytes, so 4x more data is 4x the runtime — + * and keeps them correct if a limit is ever retuned. + */ +function createRepetitiveXmlPart(minimumBytes: number): Uint8Array { + const unit = "Header padding text."; + const open = ''; + const close = ""; + const repeats = Math.ceil((minimumBytes - open.length - close.length) / unit.length); + + return strToU8(`${open}${unit.repeat(Math.max(repeats, 1))}${close}`); +} + +/** A DOCX containing one of every construct the import report knows about. */ +function createKitchenSinkDocx(): Uint8Array { + return createDocxArchive({ + "word/document.xml": documentXml( + [ + "", + 'Base paragraph. ', + 'INSERTED-TEXT', + 'DELETED-TEXT', + "", + "", + 'Commented sentence.', + '', + "", + "TEXTBOX-CONTENT", + '', + '', + '', + '', + '', + "Trailing paragraph.", + ].join(""), + ), + "word/comments.xml": [ + ``, + '', + "First reviewer comment.", + "", + ].join(""), + "word/header1.xml": `HEADER-TEXT-CONTENT`, + "word/diagrams/data1.xml": + 'SMARTART-NODE-TEXT', + "word/charts/chart1.xml": + 'CHART-TITLE-TEXT', + "word/afchunk.html": "

ALTCHUNK-IMPORTED-TEXT

", + "word/media/image1.png": TINY_PNG, + "word/embeddings/oleObject1.bin": strToU8("EMBEDDED-OLE-OBJECT"), + }); +} + +function findUnmappable( + report: DocxImportReport, + kind: DocxUnmappableKind, +): DocxUnmappableItem | undefined { + return report.unmappable.find((item) => item.kind === kind); +} + +/** A runner that answers the version probe, then writes fixed Markdown output. */ +function createReverseRunner(markdown: string): PandocRunner { + return vi.fn(async (_binary, args) => { + if (args[0] === "--version") { + return { stdout: "pandoc 3.10", stderr: "", exitCode: 0 }; + } + + const outputPath = args[args.indexOf("--output") + 1]; + + if (outputPath === undefined) { + throw new Error("test runner received no output path"); + } + + await writeFile(outputPath, markdown); + + return { stdout: "", stderr: "", exitCode: 0 }; + }); +} + async function createTempDocx(name: string): Promise { const tempDirectory = await createTempDirectory(); const docxPath = join(tempDirectory, name);