diff --git a/.claude/skills/add-cms-connector/SKILL.md b/.claude/skills/add-cms-connector/SKILL.md index 30ed9cd79..df7fd7da9 100644 --- a/.claude/skills/add-cms-connector/SKILL.md +++ b/.claude/skills/add-cms-connector/SKILL.md @@ -134,6 +134,20 @@ Before inspecting the sample, fetch the CMS's official docs: | url / link | `link` or `url` | | taxonomy / category / tag | `taxonomy` | +⚠️ **Complex field gate — run before generating any code:** +After building the mapping table, scan every row whose sample value is a structured object or array (not a string, number, boolean, or `null`). These are **complex fields**. For each one, **stop and invoke `resolve-complex-field`** before proceeding. Pass: the CMS name, the source field type name, the raw sample value, and the connector file paths (to be created in Steps 2–4). Resume building the connector only after every complex field has been resolved and its converter/schema fix confirmed. + +Fields that ALWAYS require `resolve-complex-field`: +- Rich text / structured text (any RTE format) +- Portable text / DAST / Slate / block content +- Modular blocks / page builder sections +- Nested objects or arrays that map to `group` or `modular_blocks` +- Anything in the mapping table row for `json` unless you are certain the value is already a CS RTE doc + +Fields that do NOT require it (primitive or handled by existing infrastructure): +- `single_line_text`, `multi_line_text`, `number`, `boolean`, `isodate` — no converter needed +- `file` / `reference` — handled by `getAllAssets` + reference resolution in `reference/entry-creation.md` + ### Step 2 — Scaffold Layer A: the parser package `upload-api/migration-/` Create the package from `templates/upload-api-package/`. Copy each template file, replacing ``/``/`` placeholders: - `package.json`, `tsconfig.json`, `config/index.json` diff --git a/.claude/skills/add-cms-connector/reference/touchpoints.md b/.claude/skills/add-cms-connector/reference/touchpoints.md index 8c7d32912..af57dd06e 100644 --- a/.claude/skills/add-cms-connector/reference/touchpoints.md +++ b/.claude/skills/add-cms-connector/reference/touchpoints.md @@ -15,7 +15,7 @@ Line numbers are anchors at time of writing — they drift. Always open the file - `interface/interface.ts` — `Field`, `FieldAdvanced`, `DataConfig`, `CT = Field[]`. Keep `Field` identical to wordpress. - `config/index.json` — `data: "./cmsMigrationData"`, module dir names (`content_types`, `entries`, `assets`, ...). - `libs/extractLocale.ts` — returns `string[]` of locale codes. -- `libs/contentTypes.ts` — `extractContentTypes(affix, filePath, DataConfig)`; reads sample, groups records by type, writes `content_types/*.json`, returns the parsed CTs. +- `libs/contentTypes.ts` — `extractContentTypes(affix, filePath, DataConfig)`; reads sample, groups records by type, writes `content_types/*.json`, returns the parsed CTs. Always call `ensureMandatoryFields(schema)` before writing each CT — it must inject both `title` (mandatory) **and** `url` (non-mandatory text field) unconditionally, regardless of `options.is_page`. Contentstack rejects any CT missing a `url` field when the CT is page-type, and `is_page` defaults to `true` for most models — do not rely on the flag. - `libs/schemaMapper.ts` — switch: source field/widget type → `Field` with chosen `contentstackFieldType`. - `utils/helper.ts` — file I/O helpers. diff --git a/.claude/skills/add-cms-connector/templates/api-service.ts b/.claude/skills/add-cms-connector/templates/api-service.ts index 1feb0934f..d8b5ae1d6 100644 --- a/.claude/skills/add-cms-connector/templates/api-service.ts +++ b/.claude/skills/add-cms-connector/templates/api-service.ts @@ -113,12 +113,25 @@ function transformField( return typeof value === 'string' ? value : String(value ?? ''); case 'html': - case 'json': - // RTE: convert your source rich text into a Contentstack JSON-RTE doc - // ({ type:'doc', uid, attrs:{}, children:[{type:'p',uid,attrs:{},children:[{text}]}] }). - // Don't drop media embedded IN the rich text — emit embedded-asset nodes - // (resolve via assetLookup; shape in reference/entry-creation.md). - return value; + return typeof value === 'string' ? value : ''; + + case 'json': { + // Step 1: already a valid CS JSON-RTE doc — pass through untouched. + // A valid doc has: { type: 'doc', children: [...] } + if (value && typeof value === 'object' && !Array.isArray(value) + && (value as any).type === 'doc' && Array.isArray((value as any).children)) { + return value; + } + // Step 2: ADAPT — call your CMS-specific RTE converter here before returning. + // e.g. for DatoCMS DAST: return convertDastToCSRte(value, entryIdMap, recordToCtUid, locale); + // e.g. for Portable Text: return convertPortableTextToCSRte(value, assetLookup); + // e.g. for Contentful RT: return convertContentfulRteToCSRte(value); + + // Step 3: safe fallback — return an empty doc so the entry is never silently dropped. + // ⚠️ The CS CLI calls jsonRteData.children.forEach() with no null guard; any non-CS-RTE + // value (null, string, DAST object, etc.) crashes and silently drops the WHOLE entry. + return { type: 'doc', uid: newUid(), attrs: {}, children: [{ type: 'p', uid: newUid(), attrs: {}, children: [{ text: '' }] }] }; + } case 'isodate': { if (!value) return null; @@ -227,6 +240,10 @@ async function createEntry( master_locale: string, _project: any, ): Promise { + // ⚠️ Every fs.promises call in this function MUST be awaited. A missing await + // (fire-and-forget) means the function returns before writes complete — the CS CLI + // then reads files that don't exist yet, producing only partial output (e.g. only + // 1 entry visible) with NO error shown. try { const locale = master_locale || 'en-us'; @@ -292,6 +309,9 @@ async function createEntry( const folderPath = path.join(DATA, destinationStackId, ENTRIES_DIR_NAME, folderName, locale); await fs.promises.mkdir(folderPath, { recursive: true }); + // ⚠️ BOTH files are required. CS CLI reads index.json first as a manifest — if it is + // missing, indexerCount = 0 and the entry loop never runs for this CT. The actual entry + // data in .json is never reached and zero entries are created (no error shown). await fs.promises.writeFile(path.join(folderPath, `${locale}.json`), JSON.stringify(entryData, null, 4), 'utf-8'); await fs.promises.writeFile(path.join(folderPath, 'index.json'), JSON.stringify({ '1': `${locale}.json` }, null, 4), 'utf-8'); console.info(`[] ${ct?.contentstackUid}: wrote ${Object.keys(entryData).length} entries`); @@ -323,6 +343,9 @@ async function getAllAssets( ): Promise { const assetsSave = path.join(DATA, destinationStackId, ASSETS_DIR_NAME); await fs.promises.mkdir(path.join(assetsSave, 'files'), { recursive: true }); + // ⚠️ BOTH files are required. CS CLI reads assets.json first as a manifest — if it is + // missing, indexerCount = 0 and the upload loop never runs. The actual asset records in + // index.json are never reached and zero assets are uploaded (no error shown). await fs.promises.writeFile(path.join(assetsSave, ASSETS_FILE_NAME), JSON.stringify({ '1': ASSETS_SCHEMA_FILE }, null, 4)); await fs.promises.writeFile(path.join(assetsSave, ASSETS_FOLDER_FILE_NAME), '{}'); diff --git a/.claude/skills/add-connector-field/SKILL.md b/.claude/skills/add-connector-field/SKILL.md index 1e82a2719..af4d934cc 100644 --- a/.claude/skills/add-connector-field/SKILL.md +++ b/.claude/skills/add-connector-field/SKILL.md @@ -12,6 +12,8 @@ A field flows through two mapping points. To support a new source field type (or > Tone: same meme rules as `add-cms-connector` — real-time, chat-narration only, > one per message max; never in code, tables, or option labels. +⚠️ **Wrong skill for complex fields.** If the field's sample value is a structured object or array — rich text, DAST, portable text, modular blocks, page sections, nested objects — this skill is NOT the right tool. Use `resolve-complex-field` instead. It performs structural analysis, identifies the node format, generates a converter function, and runs a smoke test. This skill only handles simple 1:1 type remappings (e.g. changing a `string` from `single_line_text` to `multi_line_text`, or mapping a previously-dropped `date` field to `isodate`). + ## Inputs you need from the user 1. **Which connector** — `wordpress` | `contentful` | `drupal` | `aem` | `sitecore` | `sanity`. 2. **A sample of the field's value** from a real export (so you can see the source type id and the value shape). diff --git a/.claude/skills/resolve-complex-field/SKILL.md b/.claude/skills/resolve-complex-field/SKILL.md new file mode 100644 index 000000000..a13a81540 --- /dev/null +++ b/.claude/skills/resolve-complex-field/SKILL.md @@ -0,0 +1,318 @@ +--- +name: resolve-complex-field +description: Analyzes any CMS field structure — known or completely unknown — and generates a Contentstack-compatible converter from scratch. Vocabulary-independent, source-agnostic, zero silent drops. Use when add-cms-connector hits a complex field or when an existing connector produces blank or wrong output for a specific field. +--- + +# resolve-complex-field + +> **Design principle — target-oracle, not source-templates.** +> The only fixed knowledge is the target. The five CS hard rules are the invariant contract — everything about the source is discovered at runtime from the sample value. Known-format signatures exist only as an optional shortcut registry, never as the design center. If every signature check fails, the generic path still produces structurally valid output. + +This skill exists because type names lie. A field named `rich_text` may contain modular blocks. A field named `structured_text` may use DAST, a proprietary tree, or something never seen before. A field named `json` may crash the CS CLI if passed non-RTE content. + +Correctness never depends on recognizing the source. The skill analyses the **actual sample value shape** and produces a converter that satisfies the CS hard rules — not a template with `// ADAPT` gaps. + +> Read `reference/cs-rte-spec.md` — the target oracle. Every output node must satisfy the CS hard rules. +> Read `reference/node-inference.md` — structural roles + two-stage name refinement. +> Read `reference/marks-patterns.md` — the three marks patterns and per-node detection. +> Read `reference/signatures.md` — optional shortcut registry for known formats (DAST, Contentful, etc.). +> Read `reference/modular-blocks.md` — the two block embedding patterns. + +## When this skill is invoked + +**Auto — from `add-cms-connector` Step 1b:** +Any field whose sample value is a structured object or array (not a primitive) triggers this skill before connector code is generated. + +**Manual — standalone:** +When an existing connector produces blank, null, or wrong output for a specific field. Pass the raw sample value — no prior knowledge of the source format needed. + +## Inputs + +1. **Source CMS name** — e.g. `datocms`, `sanity`, `strapi` +2. **Source field type name** — e.g. `structured_text`, `rich_text`, `blocks` (context only — never the mapping decision) +3. **Sample field value** — the actual raw JSON value from a real export record (this drives everything) +4. **Path to the connector's `schemaMapper.ts`** — Layer A, the parser package +5. **Path to the connector's service file** — Layer C, e.g. `api/src/services/datocms.service.ts` + +If any input is missing, ask for it. The sample field value is mandatory — do not invent or guess the structure. + +## CS hard rules — the invariant contract + +Source-independent. Every generated converter is validated against these. Each one, if violated, crashes the CS CLI or silently drops the entry. + +**Rule 1 — Every non-leaf node has `uid` — hex, no hyphens.** +```ts +const uid = () => + (Math.random().toString(16).slice(2) + Math.random().toString(16).slice(2)) + .padEnd(32, '0').slice(0, 32); +``` +Two random segments concatenated — full 32-hex entropy. Never pass a UUID or `undefined`. + +**Rule 2 — `attrs: {}` always present, always an object.** +Even when a node has no attributes. Only `a` (`{url, target}`), `code` block (`{'code-type': lang}`), and reference nodes populate it — but the key must exist on every node. + +**Rule 3 — `children` never null or empty on block nodes.** +Guard at every level, not just the root. Minimum safe value: `[{text:''}]`. +```ts +const safe = (arr: any[]) => + arr.filter(Boolean).length ? arr.filter(Boolean) : [{ text: '' }]; +``` +Call `safe()` on **every** children array assignment. + +**Rule 4 — `li` children must be block nodes, never raw text.** +Any text-leaf inside a list item gets wrapped in `p` before output, regardless of source shape. Apply the `li` guard **before** the `safe()` guard. + +**Rule 5 — CS type `json` means JSON RTE only — never arbitrary data.** +Non-RTE structured objects crash on `.children.forEach()`. Map them to `multi_line_text` + `JSON.stringify` instead. + +## Workflow + +### Step 0 — Check signature registry first + +Before running the full Analysis Protocol, check `reference/signatures.md` for a fingerprint match on the sample value. A match means a pre-verified template is available — skip directly to A5 using that template. + +No match → proceed to Step 1 (full generic path). **Adding a new CMS never requires touching the signature registry.** + +### Step 1 — Classify the field (structural only) + +Walk the sample value. Names are hints — structure decides: + +``` +sample value +│ +├─ Recursive tree? objects containing arrays of objects, ≥2 levels, +│ sharing a repeated key (children / content / nodes / …) +│ └─ YES → RTE-like document → Analysis Protocol (Step 2) +│ +├─ Flat array of objects, each with a discriminator key? +│ └─ YES → Modular blocks → Step 3A +│ +├─ Single object, fixed stable key set, no recursion? +│ └─ YES → Group → Step 3B +│ +└─ Anything else (irregular / opaque / mixed) + └─ Custom JSON → Step 3C (Rule 5) +``` + +**Tree detection — find the children-key:** +Walk the sample. Find the key whose value is most often an array of objects **and** which appears at multiple depths. If nesting depth ≥ 2, it's a tree. Record that key as the **children-key** — it may not be called `children`. The converter is parameterized on whatever key it is. + +### Step 2 — Analysis Protocol (for tree-shaped documents) + +**Run before writing any code.** Every step is defined structurally — works on vocabulary the skill has never seen. + +#### A1 — Find the discriminator (coverage-first) + +Walk every object in the sample tree. Collect all short-string-valued keys as candidates (`type`, `nodeType`, `_type`, `kind`, `tag` seeded first, but any qualifying key competes). + +Selection rule: +1. **Primary: coverage** — present on > 90% of nodes +2. **Tiebreak: distinct values** — must be ≥ 2 +3. **No key passes** → classify per-node by shape alone (string-valued = leaf, has children-key = block) + +The converter's main `switch` branches on this field. + +> v4 note: "most distinct values wins" (v3) misroutes when a coarse `type` coexists with a richer orthogonal key. Coverage-first is more reliable. + +#### A2 — Build the node inventory + +For every distinct discriminator value across the entire sample tree, record its structural fingerprint: +- Has children-key array? → block node +- Has a text-bearing property (`text`, `value`, `content` as string)? → leaf node +- Has formatting hints? (boolean props / marks array / wrapper shape) → which marks pattern — see `reference/marks-patterns.md` +- Has a reference-bearing property? (uuid-ish string, `_ref`, `itemId`, `item`) → reference/embed candidate +- Has a media hint? (URL string, mime type, dimensions object) → embedded asset candidate +- Remaining attribute payload keys → attrs + +**Coverage guarantee:** every inventory value gets an explicit `case` handler. No `// ADAPT` gaps — the case list is derived from the sample, not templated. + +#### A3 — Infer CS type — structure first, names second + +**Stage 1 — structural role decides:** + +| Fingerprint | → Role | +|---|---| +| Text-bearing, no children | Text leaf | +| Children of only text/inline nodes | Paragraph-like → `p` | +| Children of only blocks, no payload | Wrapper → pass-through or `p`-wrap | +| Repeated homogeneous item-like children | List → `ol`/`ul` + `li` | +| Grid pattern — uniform row width | `table` → `tr` → `td`/`th` | +| Reference-bearing property | CS reference node (block/inline by position) | +| Media-hinted | Embedded asset node | +| No children, no text, no payload | Void → `hr` + `children:[{text:''}]` | + +**Stage 2 — name refinement, narrows only:** +After structural role is decided, check if the node's discriminator value matches a known semantic name. If it matches, upgrade the role. If it doesn't match, keep the structural role — a node called `absatz` or `blok_tekstowy` still lands as `p` instead of falling through. + +Name → CS type upgrades: +- `heading` + `level` attr → `h1`–`h6` +- `quote` / `blockquote` / `block_quote` → `blockquote` +- `pre` / `fence` / `code_block` / `codeBlock` → `code` block +- `hyperlink` / `link` / `anchor` → `a` +- `bullet_list` / `unordered_list` / `ul` → `ul` +- `ordered_list` / `numbered_list` / `ol` → `ol` +- `list_item` / `listItem` / `li` → `li` +- `table_row` / `tr` → `tr` +- `table_cell` / `td` → `td` +- `table_header` / `th` → `th` +- `horizontal_rule` / `thematic_break` → `hr` + +Log every inference as `source type → structural role → CS type` so misroutes are auditable. + +#### A4 — Marks detection + normalization (per-node) + +Sources can **mix** patterns — detection is per-node, not per-document. See `reference/marks-patterns.md` for full detail. All three patterns normalize to the same CS text leaf shape. + +**Pattern 1 — marks array on span:** +``` +{ type: 'span', marks: ['strong', 'em'], value: 'text' } +→ { text: 'text', bold: true, italic: true } +``` + +**Pattern 2 — boolean props on leaf (map name variants: isBold / b / strong → bold):** +``` +{ text: 'text', bold: true } → pass through +{ text: 'text', isBold: true } → { text: 'text', bold: true } +``` + +**Pattern 3 — wrapper nodes (detected structurally — inline-only children, no payload):** +``` +{ type: 'strong', children: [{ type: 'em', children: [{ text: 'hi' }] }] } +→ unwrap recursively, collect marks into a Set, apply to innermost leaf +⚠ never block-render a wrapper node +``` + +Inline code is always a mark (`code: true` on leaf). Only block-level code nodes with their own discriminator value and block children become CS `code` blocks with `attrs: {'code-type': lang}`. + +#### A5 — Generate the converter with all guards baked in + +Use `templates/rte-converter.ts` as the parameterized pattern. The template is NOT pre-filled — it is a structural skeleton adapted to the specific inventory from A2. + +**Every case in the generated converter must:** +- Call `uid()` on every non-leaf node created +- Set `attrs: {}` (populated only for `a`, `code` block, reference nodes) +- Wrap children with `safe()` on every assignment +- Apply `li` guard (wrap text-leaf children in `p`) before `safe()` +- Read `node[CHILDREN_KEY]`, never hardcoded `children` + +**Fallback ladder — nothing dropped silently:** + +| Condition | Action | +|---|---| +| Unknown type + has children | Convert children, wrap in `p`, warn | +| Unknown type + has text prop | Extract as text leaf, warn | +| Matches wrapper fingerprint | Unwrap as marks (Pattern 3), warn | +| None of the above | **Stringify escape hatch**: `{ type:'p', uid:uid(), attrs:{}, children:[{ text: JSON.stringify(node).slice(0,200) }] }` + warn | + +> v4: The stringify escape hatch replaces `return null` from v3. Content is always preserved visibly — zero silent content loss guaranteed. A null-dropped node looks like "it migrated" when it didn't. A stringified fallback is ugly but honest. + +**Outputs:** +- Node inventory comment at the top of the converter (source type → structural role → CS type, one line per type) +- `convertToCSRte(value, entryIdMap, recordToCtUid, locale)` function +- Updated `case 'json'` in the service file + +### Step 3A — Modular blocks (flat array with type discriminator) + +The sample is an array where each element carries a type discriminator with ≥ 2 distinct values. + +**Schema fix (Layer A — `schemaMapper.ts`):** +Emit `modular_blocks` parent row + one `modular_blocks_child` row per distinct type + field rows per block. See `reference/modular-blocks.md` for the exact row format. + +If any block's fields are themselves complex (nested RTE, nested array with discriminator), **recurse into this skill** for those fields before writing the parent converter. + +Standalone CTs — only when a block type is reused across ≥ 2 fields. Flag in the report. Inline schema otherwise. + +**Entry value (Layer C — service file):** +```json +[ + { "": { "fieldA": "value" } }, + { "": { "fieldC": "value" } } +] +``` + +No converter function needed — value is a structured object, not an RTE tree. + +### Step 3B — Group (single object, fixed key set) + +Update `schemaMapper.ts` to emit `group` with children derived from the sample's keys. Use `advanced.multiple: true` for arrays of objects. + +If any child value is itself complex, **recurse into this skill** for that child field. + +No converter needed — existing `case 'group'` in the service handles it. See `add-cms-connector/reference/entry-creation.md` § Nested groups for the dotted-child contract. + +### Step 3C — Custom JSON (non-RTE, no tree structure) + +Map to `multi_line_text` in `schemaMapper.ts`. Serialize with `JSON.stringify(value)` in the service file. + +**Never use CS type `json` for this.** Rule 5. + +### Step 4 — Smoke test + +Generate and run a throwaway test script. Delete it after all passes. + +```ts +import { convertToCSRte } from './path/to/converter'; + +// Check 1 — shape +const result = convertToCSRte(sample); +console.assert(result.type === 'doc', 'root type'); +console.assert(Array.isArray(result.children) && result.children.length > 0, 'children'); + +// Check 2 — inventory coverage +// Every type from the A2 inventory must appear in the output, or be +// explicitly logged as merged (wrappers). Add one assert per inventory type. + +// Check 3 — structural walk (the oracle check) +function walk(node: any, path: string) { + if (!node || typeof node !== 'object') return; + if (node.text !== undefined) return; // leaf — ok + console.assert( + typeof node.uid === 'string' && node.uid.length === 32 && !node.uid.includes('-'), + `uid missing/invalid at ${path}` + ); + console.assert(node.attrs !== undefined, `attrs missing at ${path}`); + console.assert( + Array.isArray(node.children) && node.children.filter(Boolean).length > 0, + `children empty at ${path}` + ); + if (node.type === 'li') { + node.children.forEach((c: any, i: number) => + console.assert(c?.type && c.text === undefined, `li child ${i} at ${path} is not a block`) + ); + } + node.children.forEach((c: any, i: number) => walk(c, `${path}.children[${i}]`)); +} +walk(result, 'root'); + +// Check 4 — adversarial inputs (must return valid doc or preserved fallback, never throw) +[null, undefined, {}, [], { type: 'root', children: [{ type: '__UNKNOWN_XYZ__', text: 'hi' }] }] + .forEach(input => { + const r = convertToCSRte(input); + console.assert(r?.type === 'doc', `adversarial input produced invalid root: ${JSON.stringify(input)}`); + }); + +// Check 5 — typecheck +// npx tsc --noEmit → no new errors in edited files +``` + +Report results honestly — if any assert fails, fix the converter before calling the skill done. + +## Report format — per invocation + +After completing, output: +1. **File + line** changed in `schemaMapper.ts` +2. **Converter location** (file path) +3. **Full inference table** — `source type → structural role → CS type` (one row per inventory type) +4. **Marks pattern(s) detected** — Pattern 1 / 2 / 3 or mixed, per node +5. **Fallback-ladder hits** — any nodes that triggered the stringify escape hatch (with their type names) +6. **Blocks needing standalone CTs** — if any block type appears in ≥ 2 fields +7. **Structural-walk result** — pass / fail with assertion detail + +## What gets written to disk + +| Category | Layer A change | Layer C change | +|---|---|---| +| RTE (any format) | `schemaMapper.ts` — `case` → `json` | New `convert*ToCSRte()` function; `case 'json'` update in service | +| Modular blocks | `schemaMapper.ts` — parent + child + field rows | Entry transform handles blocks array | +| Group | `schemaMapper.ts` — `group` + dotted children | Existing `case 'group'` handles it | +| Custom JSON | `schemaMapper.ts` — `multi_line_text` | `JSON.stringify(value)` in service | diff --git a/.claude/skills/resolve-complex-field/reference/cs-rte-spec.md b/.claude/skills/resolve-complex-field/reference/cs-rte-spec.md new file mode 100644 index 000000000..cf9fcf238 --- /dev/null +++ b/.claude/skills/resolve-complex-field/reference/cs-rte-spec.md @@ -0,0 +1,201 @@ +# CS JSON RTE Node Specification — Target Oracle + +This is the shape oracle. Every converter generated by `resolve-complex-field` must produce exactly these structures. Any deviation — missing `uid`, missing `attrs`, empty `children` — causes the CS CLI to crash or silently drop the entry. + +## Guard utilities — copy verbatim into every converter + +```ts +// Rule 1 — full 32-hex entropy, two random segments, no hyphens possible +const uid = () => + (Math.random().toString(16).slice(2) + Math.random().toString(16).slice(2)) + .padEnd(32, '0').slice(0, 32); + +// Rule 3 — empty-children guard, call on every children array assignment +const safe = (arr: any[]) => + arr.filter(Boolean).length ? arr.filter(Boolean) : [{ text: '' }]; +``` + +--- + +## Root document + +```json +{ + "type": "doc", + "uid": "<32-hex-no-hyphens>", + "attrs": {}, + "children": [ ] +} +``` + +--- + +## Block nodes + +All block nodes share the same envelope: `type`, `uid`, `attrs: {}`, `children`. +`children` must **never** be null or empty — minimum `[{text:''}]`. + +### Paragraph +```json +{ "type": "p", "uid": "...", "attrs": {}, "children": [ ] } +``` + +### Headings +```json +{ "type": "h1", "uid": "...", "attrs": {}, "children": [ ] } +{ "type": "h2", "uid": "...", "attrs": {}, "children": [ ] } +{ "type": "h3", "uid": "...", "attrs": {}, "children": [ ] } +{ "type": "h4", "uid": "...", "attrs": {}, "children": [ ] } +{ "type": "h5", "uid": "...", "attrs": {}, "children": [ ] } +{ "type": "h6", "uid": "...", "attrs": {}, "children": [ ] } +``` + +### Unordered / ordered list +```json +{ "type": "ul", "uid": "...", "attrs": {}, "children": [
  • ] } +{ "type": "ol", "uid": "...", "attrs": {}, "children": [
  • ] } +``` + +### List item +```json +{ "type": "li", "uid": "...", "attrs": {}, "children": [

    ] } +``` + +⚠️ `li.children` must be block nodes only. A raw text leaf directly inside `li` crashes the CLI: +```ts +// WRONG — crashes +{ type: 'li', children: [{ text: 'hello' }] } + +// CORRECT — wrap in p +{ type: 'li', uid: uid(), attrs: {}, children: [{ type: 'p', uid: uid(), attrs: {}, children: [{ text: 'hello' }] }] } +``` + +### Blockquote +```json +{ "type": "blockquote", "uid": "...", "attrs": {}, "children": [

    ] } +``` + +### Code block +```json +{ "type": "code", "uid": "...", "attrs": { "code-type": "javascript" }, "children": [ ] } +``` +`attrs['code-type']` = language identifier. Use `""` when unknown. + +### Thematic break +```json +{ "type": "hr", "uid": "...", "attrs": {}, "children": [{ "text": "" }] } +``` +`children` required even for a void element — use `[{text:''}]`. + +### Table (v4 — new) +```json +{ "type": "table", "uid": "...", "attrs": {}, "children": [ ] } +{ "type": "tr", "uid": "...", "attrs": {}, "children": [ ] } +{ "type": "td", "uid": "...", "attrs": {}, "children": [

    ] } +{ "type": "th", "uid": "...", "attrs": {}, "children": [

    ] } +``` + +--- + +## Inline nodes + +### Link +```json +{ "type": "a", "uid": "...", "attrs": { "url": "https://...", "target": "_blank" }, "children": [ ] } +``` +`attrs.target` = `"_blank"` for external, `""` for internal. + +### Inline code (mark — not a block node) +```json +{ "text": "someCode()", "code": true } +``` +Inline code is always a text leaf mark. Only when source has a block-level discriminator with block children does it become a CS `code` block node. + +--- + +## Text leaves + +No `type`, `uid`, `attrs`, or `children` — only `text` plus optional formatting booleans. + +```json +{ "text": "Hello world" } +{ "text": "Bold", "bold": true } +{ "text": "Italic", "italic": true } +{ "text": "Underline", "underline": true } +{ "text": "Struck", "strikethrough": true } +{ "text": "code", "code": true } +{ "text": "super", "superscript": true } +{ "text": "combo", "bold": true, "italic": true } +{ "text": "" } +``` + +Empty text `{text:''}` is valid — use as minimum child for nodes that need children but have none. + +--- + +## Reference nodes (embedded entries) + +### Embedded entry — block display +```json +{ + "uid": "...", + "type": "reference", + "attrs": { + "display-type": "block", + "entry-uid": "", + "content-type-uid": "", + "locale": "en-us", + "type": "entry", + "class-name": "embedded-entry-block" + }, + "children": [{ "text": "" }] +} +``` + +### Embedded entry — inline display +```json +{ + "uid": "...", + "type": "reference", + "attrs": { + "display-type": "inline", + "entry-uid": "", + "content-type-uid": "", + "locale": "en-us", + "type": "entry", + "class-name": "embedded-entry-inline", + "inline": true + }, + "children": [{ "text": "" }] +} +``` + +### Embedded asset +```json +{ + "uid": "...", + "type": "reference", + "attrs": { + "display-type": "display", + "asset-uid": "", + "content-type-uid": "sys_assets", + "asset-link": "/assets/", + "asset-name": "", + "asset-type": "", + "type": "asset", + "class-name": "embedded-asset", + "inline": false + }, + "children": [{ "text": "" }] +} +``` + +All reference nodes: `children: [{text:''}]` always. + +--- + +## What CS type `json` means + +CS field type `json` expects a JSON RTE document — exactly the structure above. Passing an arbitrary object or array causes the CLI to call `.children.forEach()` on it and crash. + +For non-RTE structured data (color objects, SEO objects, config), use `multi_line_text` + `JSON.stringify(value)`. Never use `json` for arbitrary data. (Rule 5) diff --git a/.claude/skills/resolve-complex-field/reference/marks-patterns.md b/.claude/skills/resolve-complex-field/reference/marks-patterns.md new file mode 100644 index 000000000..ca6a20d53 --- /dev/null +++ b/.claude/skills/resolve-complex-field/reference/marks-patterns.md @@ -0,0 +1,201 @@ +# Marks Patterns — Detection and Normalization + +Marks (bold, italic, etc.) appear in three distinct structural patterns across different CMSes. Detection is **per-node**, not per-document — a single source document can mix all three patterns. All three normalize to the same CS text leaf shape. + +--- + +## Pattern 1 — Marks array on a span/leaf node + +The leaf node carries a `marks` array of string names alongside its text value. + +**Source shape:** +```json +{ "type": "span", "marks": ["strong", "emphasis"], "value": "Hello" } +{ "type": "text", "marks": ["bold", "underline"], "text": "World" } +``` + +**Detection signal:** node has both a text-bearing property (`text` or `value`) AND a `marks` array property. + +**Normalization:** map each mark string to its CS boolean prop: + +| Source mark string | CS boolean | +|---|---| +| `strong` / `bold` | `bold: true` | +| `emphasis` / `em` / `italic` | `italic: true` | +| `underline` | `underline: true` | +| `strikethrough` / `strike` / `del` | `strikethrough: true` | +| `code` | `code: true` | +| `highlight` / `mark` | `superscript: true` | + +**Output:** +```json +{ "text": "Hello", "bold": true, "italic": true } +``` + +**Code:** +```ts +const MARK_MAP: Record = { + strong: 'bold', bold: 'bold', + emphasis: 'italic', em: 'italic', italic: 'italic', + underline: 'underline', + strikethrough: 'strikethrough', strike: 'strikethrough', del: 'strikethrough', + code: 'code', + highlight: 'superscript', mark: 'superscript', +}; + +function convertSpan(node: any): any { + const leaf: any = { text: String(node.value ?? node.text ?? '') }; + for (const m of node.marks ?? []) { + const prop = MARK_MAP[m]; + if (prop) leaf[prop] = true; + } + return leaf; +} +``` + +--- + +## Pattern 2 — Boolean props already on the leaf + +The leaf node already has formatting as boolean properties. May use canonical names (`bold`) or variant names (`isBold`, `b`, `strong`). + +**Source shape:** +```json +{ "text": "Hello", "bold": true, "italic": true } +{ "text": "World", "isBold": true, "isItalic": true } +{ "text": "Foo", "b": true, "i": true } +``` + +**Detection signal:** node has a text-bearing property AND one or more formatting-named boolean properties. No `marks` array. + +**Normalization:** map variant prop names to CS canonical names. Canonical names pass through unchanged. + +| Source prop | CS prop | +|---|---| +| `bold` / `isBold` / `b` / `strong` | `bold` | +| `italic` / `isItalic` / `i` / `em` | `italic` | +| `underline` / `isUnderline` / `u` | `underline` | +| `strikethrough` / `isStrikethrough` / `strike` / `del` / `s` | `strikethrough` | +| `code` / `isCode` / `inlineCode` | `code` | +| `highlight` / `isHighlight` / `superscript` | `superscript` | + +**Code:** +```ts +const PROP_MAP: Record = { + bold: 'bold', isBold: 'bold', b: 'bold', strong: 'bold', + italic: 'italic', isItalic: 'italic', i: 'italic', em: 'italic', + underline: 'underline', isUnderline: 'underline', u: 'underline', + strikethrough: 'strikethrough', isStrikethrough: 'strikethrough', + strike: 'strikethrough', del: 'strikethrough', s: 'strikethrough', + code: 'code', isCode: 'code', inlineCode: 'code', + highlight: 'superscript', isHighlight: 'superscript', superscript: 'superscript', +}; + +function convertLeaf(node: any): any { + const leaf: any = { text: String(node.text ?? node.value ?? '') }; + for (const [k, v] of Object.entries(node)) { + if (k === 'text' || k === 'value') continue; + const prop = PROP_MAP[k]; + if (prop && v === true) leaf[prop] = true; + } + return leaf; +} +``` + +--- + +## Pattern 3 — Wrapper nodes + +Formatting is expressed as structural wrappers — each mark is a node that wraps its children. These nest arbitrarily. + +**Source shape:** +```json +{ + "type": "strong", + "children": [{ + "type": "em", + "children": [{ "text": "Hello" }] + }] +} +``` + +**Detection signal (structural):** node has a children-key AND only contains inline/text nodes (no block-level siblings). The node's type matches a known mark name (or matches the MARK_MAP). These are never block nodes — do not render them as `p` or any block type. + +⚠️ **Never block-render a wrapper.** A wrapper inside a paragraph is an inline mark, not a separate block. + +**Normalization:** walk down recursively, collecting mark types into a Set, apply the full set to the innermost text leaf. + +**Code:** +```ts +const WRAPPER_MARKS: Record = { + strong: 'bold', b: 'bold', + em: 'italic', i: 'italic', emphasis: 'italic', + underline: 'underline', u: 'underline', + strikethrough: 'strikethrough', s: 'strikethrough', del: 'strikethrough', + code: 'code', inlineCode: 'code', + highlight: 'superscript', mark: 'superscript', +}; + +function isWrapperNode(node: any): boolean { + return WRAPPER_MARKS[node.type] !== undefined; +} + +function unwrapMarks(node: any, marks: Set = new Set()): any | any[] { + if (WRAPPER_MARKS[node.type]) marks.add(WRAPPER_MARKS[node.type]); + + // Leaf — has text or value, no children (or children is empty) + const text = node.text ?? node.value; + if (text !== undefined) { + const leaf: any = { text: String(text) }; + for (const m of marks) leaf[m] = true; + return leaf; + } + + // Single child — recurse + const children = node[CHILDREN_KEY] ?? node.children ?? []; + if (children.length === 1) return unwrapMarks(children[0], marks); + + // Multiple children — each inherits the collected marks + return children.map((c: any) => unwrapMarks(c, new Set(marks))); +} +``` + +When `unwrapMarks` returns an array (multiple children), the caller must flatten it into the parent's children array. + +--- + +## Mixed patterns in one document + +Some sources use Pattern 1 for most nodes but Pattern 3 for specific inline styles. Detection must happen per-node: + +```ts +function convertNode(node: any, ctx: Ctx): any | any[] | null { + // Check Pattern 3 first — structural wrapper check + if (isWrapperNode(node)) return unwrapMarks(node); + + switch (node[DISCRIMINATOR]) { + case 'span': + case 'text': + // Check Pattern 1 + if (Array.isArray(node.marks)) return convertSpan(node); + // Check Pattern 2 + return convertLeaf(node); + + // ... other cases + } +} +``` + +--- + +## Inline code — always a mark, never a block + +A node with `type: 'code'` that wraps a single short text node and appears **inside** a paragraph is always a mark — Pattern 3: +```json +{ "type": "code", "children": [{ "text": "x.foo()" }] } +→ { "text": "x.foo()", "code": true } +``` + +Only when a code node appears at the block level (direct child of root/paragraph-container) AND has a language attribute AND its children are text lines → CS `code` block node with `attrs: {'code-type': lang}`. + +Position in the tree determines which it is. diff --git a/.claude/skills/resolve-complex-field/reference/modular-blocks.md b/.claude/skills/resolve-complex-field/reference/modular-blocks.md new file mode 100644 index 000000000..fa4222aa1 --- /dev/null +++ b/.claude/skills/resolve-complex-field/reference/modular-blocks.md @@ -0,0 +1,184 @@ +# Modular Blocks — Two Distinct Cases + +These two cases look similar from a naming perspective but are architecturally different in Contentstack. Misidentifying them causes silent drops or schema build failures. + +## Case 1 — Modular blocks field (inline schema, NOT standalone CT) + +**What it is:** A top-level field on a content type whose value is an array of objects, each representing a "block" of a specific type. Each block type has its own schema embedded inline in the parent content type — they are NOT standalone content types in Contentstack. + +**Detection signals:** +- Source field type: `modular_blocks`, `rich_text` (DatoCMS), `pageBuilder`, `sections` +- Sample value: array of objects with a per-element type discriminator +- ≥ 2 distinct type values across the array elements +- In DatoCMS: the field has `validators.rich_text_blocks.item_types` set (linking to block models with `modular_block: true`) + +**DatoCMS example:** +```json +"page_sections": [ + { "id": "abc", "item_type": { "id": "banner" }, "title": "Hero Banner", "cta_label": "Learn More" }, + { "id": "def", "item_type": { "id": "text_block" }, "body": { "schema": "dast", ... } } +] +``` + +**CS schema rows (3 levels):** + +``` +Parent row: + otherCmsField: "page_sections" + otherCmsType: "rich_text" ← the SOURCE type name + contentstackFieldType: "modular_blocks" + contentstackFieldUid: "page_sections" + isDeleted: false ← MANDATORY, CT builder filters strictly + +Block row (one per distinct block type): + otherCmsField: "banner" ← RAW source type string (the join key at entry time) + otherCmsType: "modular_blocks_child" + contentstackFieldType: "modular_blocks_child" + contentstackFieldUid: "page_sections.banner" + isDeleted: false + +Field rows inside the block (one per block field): + otherCmsField: "title" + otherCmsType: "String" + contentstackFieldType: "single_line_text" + contentstackFieldUid: "page_sections.banner.title" + isDeleted: false +``` + +Key rules: +- Parent row: `modular_blocks`, NO `advanced.multiple` — CT builder hardcodes `multiple: true` +- Block rows: `contentstackFieldType: 'modular_blocks_child'` AND `otherCmsType: 'modular_blocks_child'` +- Block row `otherCmsField` = the RAW source discriminator value — this is the join key at entry creation time +- ALL rows: `isDeleted: false` — the CT builder path for blocks filters on `isDeleted === false` **strictly** (not truthy — explicitly `=== false`) +- Block uid = `.` (one level of dotting) +- Field uid = `..` (two levels of dotting) + +**Entry value shape:** +```json +"page_sections": [ + { "banner": { "title": "Hero Banner", "cta_label": "Learn More" } }, + { "text_block": { "body": { "type": "doc", "uid": "...", "attrs": {}, "children": [...] } } } +] +``` + +- Array of single-key objects: `{ "": { } }` +- Block type uid is the `contentstackFieldUid` last segment of the block row (e.g. `banner` from `page_sections.banner`) +- Source order is preserved — important for UI rendering +- Elements with no matching block row are **skipped with a log** (never synthesize a catch-all block) + +**Entry transform:** +```ts +const blockRows = ct.fieldMapping.filter(f => + f.contentstackFieldType === 'modular_blocks_child' && + f.contentstackFieldUid.startsWith(`${parentField.contentstackFieldUid}.`) +); + +const result = []; +for (const el of sourceArray) { + const sourceType = el.item_type?.id ?? el._type ?? el.__typename; + const blockRow = blockRows.find(r => + r.otherCmsField === sourceType || r.backupFieldUid === sourceType + ); + if (!blockRow) { + console.warn(`[modular_blocks] no block row for type "${sourceType}", skipping`); + continue; + } + const blockUid = getLastUid(blockRow.contentstackFieldUid); // strips parent prefix + const childRows = ct.fieldMapping.filter(f => + f.contentstackFieldUid.startsWith(`${blockRow.contentstackFieldUid}.`) + ); + const blockObj: Record = {}; + for (const childRow of childRows) { + const childUid = getLastUid(childRow.contentstackFieldUid); + blockObj[childUid] = transformField(childRow, el[childRow.otherCmsField], ...); + } + result.push({ [blockUid]: blockObj }); +} +return result; +``` + +--- + +## Case 2 — Blocks embedded inside structured text (standalone CTs, RTE reference nodes) + +**What it is:** An RTE field (DAST, Slate, etc.) whose nodes include references to external records by ID. The referenced records are of types that ARE standalone content types in CS. They appear inline inside the RTE text as reference nodes. + +**Detection signals:** +- Source field type: `structured_text` (DatoCMS), `richText` with linked entries (Contentful) +- Sample RTE tree contains nodes like: `{type:'block', item:''}` or `{type:'inlineItem', item:''}` (DAST), or `nodeType: 'embedded-entry-block'` (Contentful) +- The referenced records are regular content type instances, NOT block models + +**DatoCMS DAST example:** +```json +{ + "schema": "dast", + "document": { + "type": "root", + "children": [ + { "type": "paragraph", "children": [{ "type": "span", "value": "See also:" }] }, + { "type": "block", "item": "12345678" } + ] + } +} +``` + +**CS output inside the RTE doc:** +```json +{ + "type": "reference", + "uid": "...", + "attrs": { + "display-type": "block", + "entry-uid": "", + "content-type-uid": "", + "locale": "en-us", + "type": "entry", + "class-name": "embedded-entry-block" + }, + "children": [{ "text": "" }] +} +``` + +**Lookup index required:** +Build a map before entry creation: +```ts +// Built during getAllAssets / before createEntry loop +const sourceIdToEntryUid: Record = {}; // sourceRecordId → cs entry uid +const sourceIdToCtUid: Record = {}; // sourceRecordId → cs CT uid + +// Populated as each source record is processed +sourceIdToEntryUid[record.id] = toEntryUid(record.id); +sourceIdToCtUid[record.id] = getCtUidForType(record.item_type.id); +``` + +Pass both maps into the RTE converter: +```ts +convertStructuredTextToCSRte(value, sourceIdToEntryUid, sourceIdToCtUid, masterLocale) +``` + +**These block types DO need standalone CTs** — they're regular entries referenced from inside the RTE, not inline schemas. + +--- + +## How to tell Cases 1 and 2 apart quickly + +| Check | Case 1 | Case 2 | +|---|---|---| +| Source field type | `rich_text`, `modular_blocks`, non-text | `structured_text`, `richText` | +| DatoCMS model flag | block models have `modular_block: true` | referenced models are regular CTs | +| Value is | top-level array | a nested tree with id-references inside | +| CS schema | blocks inline in parent CT | referenced records are their own CTs | +| CS value | array of `{blockUid: {...}}` | RTE doc with reference nodes inside | +| Converter needed? | No — just schema rows + entry transform | Yes — full RTE converter with reference resolution | + +--- + +## Common mistakes + +1. **Mapping a DatoCMS `rich_text` field to CS `json`** — the value is not an RTE tree; it's an array of block objects. The CLI crashes trying to call `.children.forEach()`. + +2. **Using `modular_blocks` without `isDeleted: false`** — the CT builder skips these rows silently. Everything appears to work until you check the generated schema and find the blocks are empty. + +3. **Inverting the cases** — creating standalone CTs for Case 1 block models and trying to reference them from entries. The block models are meant to be schema-inline only; they have no meaningful standalone existence. + +4. **Preserving source array order accidentally broken** — when building entry values for Case 1, use `.map()` over the source array in order. Don't group by type and then rebuild — this reorders the blocks and breaks the author's intended interleaving. diff --git a/.claude/skills/resolve-complex-field/reference/node-inference.md b/.claude/skills/resolve-complex-field/reference/node-inference.md new file mode 100644 index 000000000..d1dd88931 --- /dev/null +++ b/.claude/skills/resolve-complex-field/reference/node-inference.md @@ -0,0 +1,170 @@ +# Node Inference — Two-Stage Structural Classification + +Use this reference during Analysis Protocol steps A3. Stage 1 decides the role from structure alone. Stage 2 refines using the name only when it matches — unmatched names always keep their structural role. + +This means a node called `absatz` (German for paragraph), `blok_tekstowy` (Polish for text block), or anything proprietary still lands correctly as `p` instead of falling through to the unknown fallback. + +Every inference must be logged as: `source type → structural role → CS type` + +--- + +## Stage 1 — Structural role table + +Examine the node's fingerprint (what keys it has and what type their values are): + +| Fingerprint | Structural role | CS type | +|---|---|---| +| Has text-bearing prop (`text`, `value`, `content` as string), no children | Text leaf | — (no type/uid/attrs) | +| Has children-key, children are only text leaves or inline nodes | Paragraph-like | `p` | +| Has children-key, children are only block nodes, no other payload | Wrapper | pass-through (recurse children, discard wrapper) | +| Has children-key, children are homogeneous item-like objects in sequence | List | `ul` or `ol` + `li` children | +| Has children-key, children have uniform width (each row has same key count) | Table | `table` → `tr` → `td`/`th` | +| Has a reference-bearing prop (uuid-ish string, `_ref`, `itemId`, `item`) | Reference | CS reference node | +| Has a media hint (URL string ending in image ext, mime type key, `width`/`height`) | Embedded asset | CS embedded-asset reference node | +| No children, no text, no payload (empty or only non-string keys) | Void | `hr` with `children:[{text:''}]` | +| Has children of mixed block + inline content | Mixed block | `p` (safest containing block) | + +**Ambiguous children check:** +- A block containing only inline/text nodes → `p` +- A block containing only block nodes → pass-through wrapper (recurse, discard the wrapper itself) +- A block containing mixed → `p` (wrap everything) + +--- + +## Stage 2 — Name refinement table + +Only runs after structural role is assigned. Only upgrades a role, never downgrades. If the name is not in this table, the structural role from Stage 1 stands unchanged. + +| Discriminator value (case-insensitive, partial match ok) | Structural role required | CS type | +|---|---|---| +| `heading` + `level` attribute (1–6) | paragraph-like or wrapper | `h1`–`h6` | +| `h1`…`h6` literally | paragraph-like or wrapper | `h1`–`h6` | +| `quote` / `blockquote` / `block_quote` / `pullquote` | paragraph-like | `blockquote` | +| `pre` / `fence` / `code_block` / `codeBlock` / `fenced_code` | block with text or inline children | `code` block — set `attrs: {'code-type': lang ?? ''}` | +| `hyperlink` / `link` / `anchor` / `a` | inline with text children | `a` — set `attrs: {url, target}` | +| `bullet_list` / `unordered_list` / `ul` / `bulleted_list` | list | `ul` | +| `ordered_list` / `numbered_list` / `ol` | list | `ol` | +| `list_item` / `listItem` / `li` / `item` | block with mixed/text | `li` (apply `li` guard) | +| `table` | grid structure | `table` | +| `table_row` / `tr` | row | `tr` | +| `table_cell` / `td` / `table_data` | cell | `td` | +| `table_header` / `th` | cell | `th` | +| `horizontal_rule` / `thematic_break` / `divider` / `hr` / `break` | void | `hr` | +| `image` / `asset` / `media` / `figure` | media-hinted | embedded-asset reference node | +| `block` / `embed` / `component` + reference-bearing prop | reference-bearing | CS reference node (block display) | +| `inline_item` / `inlineItem` / `inline_embed` + reference-bearing prop | reference-bearing | CS reference node (inline display) | + +--- + +## Node inventory comment format + +At the top of every generated converter, record the A2 inventory result as a comment: + +```ts +/* + * Node inventory — + * Discriminator field: `` + * Children-key: `` + * + * Block nodes: + * → → + * ... + * + * Leaf nodes: + * → text leaf + * + * Reference nodes: + * → reference node (block/inline) + * + * Marks: — → , ... + * + * Fallback hits: + */ +``` + +--- + +## Pass-through wrapper handling + +A wrapper node — one that only contains blocks and has no payload of its own — should be **discarded**, not rendered. Its children are spliced directly into the parent's children array. + +```ts +// Wrapper detected — no type emitted, recurse children only +case 'section': +case 'article': +case 'container': { + const children = convertChildren(node[CHILDREN_KEY], ctx); + // Return array, not a single node — splice into parent + return children; +} +``` + +When a converter case returns an array instead of a single node, the parent must flatten it: +```ts +const children = safe( + (node[CHILDREN_KEY] ?? []) + .flatMap((c: any) => { + const r = convertNode(c, ctx); + return Array.isArray(r) ? r : [r]; + }) + .filter(Boolean) +); +``` + +--- + +## Reference node shapes + +**Block embed (position: direct child of root/block):** +```ts +{ + uid: uid(), type: 'reference', + attrs: { + 'display-type': 'block', + 'entry-uid': entryIdMap[sourceId], + 'content-type-uid': recordToCtUid[sourceId], + locale: ctx.locale, + type: 'entry', + 'class-name': 'embedded-entry-block' + }, + children: [{ text: '' }] +} +``` + +**Inline embed (position: inside a paragraph or inline context):** +```ts +{ + uid: uid(), type: 'reference', + attrs: { + 'display-type': 'inline', + 'entry-uid': entryIdMap[sourceId], + 'content-type-uid': recordToCtUid[sourceId], + locale: ctx.locale, + type: 'entry', + 'class-name': 'embedded-entry-inline', + inline: true + }, + children: [{ text: '' }] +} +``` + +**Embedded asset:** +```ts +{ + uid: uid(), type: 'reference', + attrs: { + 'display-type': 'display', + 'asset-uid': assetIdMap[sourceId], + 'content-type-uid': 'sys_assets', + 'asset-link': assetRecord.urlPath, + 'asset-name': assetRecord.title, + 'asset-type': assetRecord.content_type, + type: 'asset', + 'class-name': 'embedded-asset', + inline: false + }, + children: [{ text: '' }] +} +``` + +All reference nodes: `children: [{text:''}]` always — never empty children. diff --git a/.claude/skills/resolve-complex-field/reference/signatures.md b/.claude/skills/resolve-complex-field/reference/signatures.md new file mode 100644 index 000000000..e2aafbdff --- /dev/null +++ b/.claude/skills/resolve-complex-field/reference/signatures.md @@ -0,0 +1,155 @@ +# Signature Registry — Optional Shortcut + +This file is an **optional shortcut**, not the design center. The Analysis Protocol in SKILL.md works without this file for any format. Signatures exist only to skip redundant analysis for formats that are already fully understood. + +A signature match means: skip A1–A4, go directly to A5 using the pre-verified node mapping below. The generated converter still must pass all five CS hard rules and the full smoke test. + +**Adding a new CMS never requires touching this file.** If no signature matches, the generic path runs and produces valid output. + +--- + +## Known signatures + +### DAST (DatoCMS structured_text) + +**Fingerprint:** top-level has `schema: 'dast'` OR root node has `type: 'root'` with a `children` array. + +```json +{ "schema": "dast", "document": { "type": "root", "children": [...] } } +``` + +**Discriminator field:** `type` +**Children-key:** `children` +**Marks pattern:** Pattern 1 — `marks` array on `span` nodes + +**Node mapping (pre-verified):** + +| Source `type` | CS type | Notes | +|---|---|---| +| `root` | `doc` | Root wrapper — not emitted as a child | +| `paragraph` | `p` | | +| `heading` | `h1`–`h6` | `level` attr (1–6) | +| `list` | `ul` / `ol` | `style: 'bulleted'` → `ul`, `style: 'numbered'` → `ol` | +| `listItem` | `li` | Apply `li` guard | +| `blockquote` | `blockquote` | | +| `code` | `code` block | `language` attr → `attrs['code-type']` | +| `thematicBreak` | `hr` | | +| `link` | `a` | `url` attr or `meta` array `{id:'url', value:'...'}` | +| `span` | text leaf | Pattern 1 marks — see marks table below | +| `block` | reference node (block) | `item` field → entry lookup | +| `inlineItem` | reference node (inline) | `item` field → entry lookup | + +**DAST marks (Pattern 1 string → CS boolean):** +`strong`→`bold`, `emphasis`→`italic`, `underline`→`underline`, `strikethrough`→`strikethrough`, `code`→`code`, `highlight`→`superscript` + +**Value wrapper:** DAST field value is either `{ schema:'dast', document:{type:'root',...} }` or the root node directly. Unwrap: `const root = value.document ?? value`. + +--- + +### Contentful Rich Text + +**Fingerprint:** top-level has `nodeType: 'document'` with a `content` array. + +```json +{ "nodeType": "document", "content": [...], "data": {} } +``` + +**Discriminator field:** `nodeType` +**Children-key:** `content` +**Marks pattern:** Pattern 1 — `marks` array of objects `{type:'bold'}` on `text` nodes + +**Node mapping (pre-verified):** + +| Source `nodeType` | CS type | Notes | +|---|---|---| +| `document` | `doc` | Root wrapper | +| `paragraph` | `p` | | +| `heading-1`…`heading-6` | `h1`–`h6` | | +| `unordered-list` | `ul` | | +| `ordered-list` | `ol` | | +| `list-item` | `li` | Apply `li` guard | +| `blockquote` | `blockquote` | | +| `hr` | `hr` | | +| `hyperlink` | `a` | `data.uri` → `attrs.url` | +| `text` | text leaf | `marks: [{type:'bold'}]` → Pattern 1 | +| `embedded-entry-block` | reference node (block) | `data.target.sys.id` → entry lookup | +| `embedded-entry-inline` | reference node (inline) | `data.target.sys.id` → entry lookup | +| `embedded-asset-block` | embedded asset node | `data.target.sys.id` → asset lookup | + +**Contentful marks (Pattern 1 object → CS boolean):** +`bold`→`bold`, `italic`→`italic`, `underline`→`underline`, `code`→`code`, `superscript`→`superscript`, `subscript` → (no CS analog — drop) + +--- + +### ProseMirror / Tiptap + +**Fingerprint:** top-level has `type: 'doc'` with a `content` array. No `schema` or `nodeType` key. + +```json +{ "type": "doc", "content": [...] } +``` + +**Discriminator field:** `type` +**Children-key:** `content` +**Marks pattern:** Pattern 3 — wrapper nodes (ProseMirror default) OR Pattern 1 on `text` nodes + +**Note:** ProseMirror/Tiptap schemas are highly customizable — the node types below are the defaults. Custom node types not in this table fall through to the generic Analysis Protocol. Check the sample inventory (A2) and extend as needed. + +| Source `type` | CS type | Notes | +|---|---|---| +| `doc` | `doc` | Root | +| `paragraph` | `p` | | +| `heading` | `h1`–`h6` | `attrs.level` (1–6) | +| `bulletList` | `ul` | | +| `orderedList` | `ol` | | +| `listItem` | `li` | Apply `li` guard | +| `blockquote` | `blockquote` | | +| `codeBlock` | `code` block | `attrs.language` | +| `horizontalRule` | `hr` | | +| `text` | text leaf | Marks via `marks` array of objects | +| `image` | embedded asset | `attrs.src` | + +--- + +### Sanity Portable Text + +**Fingerprint:** top-level is an **array** (not an object). Elements have `_type` discriminator. Block elements have `_type: 'block'` with a `children` array of spans. + +```json +[ + { "_type": "block", "style": "normal", "children": [...] }, + { "_type": "image", "asset": { "_ref": "image-..." } } +] +``` + +**Discriminator field:** `_type` +**Children-key:** `children` +**Marks pattern:** Pattern 1 — `marks` array on span nodes (`markDefs` for links/references) + +**Note:** Portable Text wraps the array in a `doc`. The converter must create the root `doc` node manually: +```ts +return { type: 'doc', uid: uid(), attrs: {}, children: safe(portableTextArray.map(n => convertNode(n, ctx))) }; +``` + +| Source `_type` | CS type | Notes | +|---|---|---| +| `block` (style: `normal`) | `p` | | +| `block` (style: `h1`…`h6`) | `h1`–`h6` | `style` attr drives level | +| `block` (style: `blockquote`) | `blockquote` | | +| `span` | text leaf | Pattern 1 marks + `markDefs` for links | +| `image` | embedded asset | `asset._ref` → asset lookup | +| Custom `_type` | Analysis Protocol | Run A1–A4 on this block type specifically | + +--- + +## How to add a new signature + +If you encounter a format repeatedly across multiple connectors: + +1. Identify the fingerprint (one or two distinctive keys/values at the top level) +2. Run the Analysis Protocol once, fully, on a real sample +3. Document the node mapping table here +4. Mark any non-standard or custom node types as requiring A2 inventory check +5. Note the marks pattern and children-key + +Do not add speculative signatures — only add after the Analysis Protocol has been run and the mapping has been smoke-tested against a real sample. diff --git a/.claude/skills/resolve-complex-field/templates/rte-converter.ts b/.claude/skills/resolve-complex-field/templates/rte-converter.ts new file mode 100644 index 000000000..9586403ef --- /dev/null +++ b/.claude/skills/resolve-complex-field/templates/rte-converter.ts @@ -0,0 +1,438 @@ +/** + * RTE converter — parameterized skeleton (v4) + * + * This is a PATTERN, not a pre-filled source file. When resolve-complex-field + * runs A5, it generates a converter by: + * 1. Replacing and with the real names + * 2. Setting CHILDREN_KEY and DISCRIMINATOR to the values found in A1 + * 3. Filling one `case` per inventory type from A2 + * 4. Removing cases that don't apply to this specific source + * + * The node inventory comment below must be filled in for every generated converter. + * + * Node inventory — + * Discriminator field: `` + * Children-key: `` + * + * Block nodes: + * → → + * + * Leaf nodes: + * → text leaf + * + * Reference nodes: + * → reference node (block/inline) + * + * Marks: — → + * + * Fallback hits: + */ + +// ── Discovered at runtime by Analysis Protocol ──────────────────────────────── +// Set these to the values found in A1 for the specific source format. + +const CHILDREN_KEY = 'children'; // the key that holds child node arrays (may not be 'children') +const DISCRIMINATOR = 'type'; // the key that identifies node type + +// ── Guards (Rule 1, 3, 4 — copy verbatim, never modify) ────────────────────── + +const uid = () => + (Math.random().toString(16).slice(2) + Math.random().toString(16).slice(2)) + .padEnd(32, '0').slice(0, 32); + +const safe = (arr: any[]): any[] => + arr.filter(Boolean).length ? arr.filter(Boolean) : [{ text: '' }]; + +// ── Marks helpers ───────────────────────────────────────────────────────────── + +// Pattern 1 — marks array of strings +const MARK_MAP: Record = { + strong: 'bold', bold: 'bold', + emphasis: 'italic', em: 'italic', italic: 'italic', + underline: 'underline', + strikethrough: 'strikethrough', strike: 'strikethrough', del: 'strikethrough', + code: 'code', + highlight: 'superscript', mark: 'superscript', +}; + +// Pattern 2 — boolean prop name variants +const PROP_MAP: Record = { + bold: 'bold', isBold: 'bold', b: 'bold', strong: 'bold', + italic: 'italic', isItalic: 'italic', i: 'italic', em: 'italic', + underline: 'underline', isUnderline: 'underline', u: 'underline', + strikethrough: 'strikethrough', isStrikethrough: 'strikethrough', + strike: 'strikethrough', del: 'strikethrough', s: 'strikethrough', + code: 'code', isCode: 'code', inlineCode: 'code', + highlight: 'superscript', isHighlight: 'superscript', superscript: 'superscript', +}; + +// Pattern 3 — structural wrapper nodes +const WRAPPER_MARKS: Record = { + strong: 'bold', b: 'bold', + em: 'italic', i: 'italic', emphasis: 'italic', + underline: 'underline', u: 'underline', + strikethrough: 'strikethrough', s: 'strikethrough', del: 'strikethrough', + code: 'code', inlineCode: 'code', + highlight: 'superscript', mark: 'superscript', +}; + +function isWrapperNode(node: any): boolean { + return typeof node[DISCRIMINATOR] === 'string' && + WRAPPER_MARKS[node[DISCRIMINATOR]] !== undefined && + Array.isArray(node[CHILDREN_KEY]); +} + +function unwrapMarks(node: any, marks: Set = new Set()): any | any[] { + const mark = WRAPPER_MARKS[node[DISCRIMINATOR]]; + if (mark) marks.add(mark); + + const text = node.text ?? node.value; + if (text !== undefined) { + const leaf: any = { text: String(text) }; + for (const m of marks) leaf[m] = true; + return leaf; + } + + const children = node[CHILDREN_KEY] ?? []; + if (children.length === 1) return unwrapMarks(children[0], marks); + return children.map((c: any) => unwrapMarks(c, new Set(marks))); +} + +function applyPattern1Marks(node: any): any { + const leaf: any = { text: String(node.value ?? node.text ?? '') }; + for (const m of node.marks ?? []) { + const prop = MARK_MAP[typeof m === 'string' ? m : m?.type ?? '']; + if (prop) leaf[prop] = true; + } + return leaf; +} + +function applyPattern2Marks(node: any): any { + const leaf: any = { text: String(node.text ?? node.value ?? '') }; + for (const [k, v] of Object.entries(node)) { + if (k === 'text' || k === 'value' || k === DISCRIMINATOR) continue; + const prop = PROP_MAP[k]; + if (prop && v === true) leaf[prop] = true; + } + return leaf; +} + +// ── Context ─────────────────────────────────────────────────────────────────── + +interface Ctx { + entryIdMap: Record; // sourceRecordId → cs entry uid + recordToCtUid: Record; // sourceRecordId → cs CT uid + assetIdMap: Record; // sourceAssetId → cs asset record + locale: string; +} + +// ── Children conversion (handles wrapper arrays + safe guard) ───────────────── + +function convertChildren(nodes: any[] | undefined, ctx: Ctx): any[] { + if (!Array.isArray(nodes)) return []; + return nodes + .flatMap(n => { + const r = convertNode(n, ctx); + return Array.isArray(r) ? r : [r]; + }) + .filter(Boolean); +} + +// ── Main dispatch ───────────────────────────────────────────────────────────── + +function convertNode(node: any, ctx: Ctx): any | any[] | null { + if (!node || typeof node !== 'object') return null; + + // Pattern 3 wrapper check — before switch (structural, not name-based) + if (isWrapperNode(node)) { + return unwrapMarks(node); + } + + switch (node[DISCRIMINATOR]) { + + // ── Root wrapper — should not appear as a child ───────────────────────── + case 'root': + case 'document': + return null; + + // ── Block nodes ───────────────────────────────────────────────────────── + + case 'paragraph': + case 'p': + return { + type: 'p', uid: uid(), attrs: {}, + children: safe(convertChildren(node[CHILDREN_KEY], ctx)), + }; + + case 'heading': { + const level = Math.min(Math.max(node.level ?? node.attrs?.level ?? 1, 1), 6); + return { + type: `h${level}`, uid: uid(), attrs: {}, + children: safe(convertChildren(node[CHILDREN_KEY], ctx)), + }; + } + + case 'h1': case 'h2': case 'h3': + case 'h4': case 'h5': case 'h6': + return { + type: node[DISCRIMINATOR], uid: uid(), attrs: {}, + children: safe(convertChildren(node[CHILDREN_KEY], ctx)), + }; + + case 'list': + case 'bullet_list': + case 'ordered_list': { + const isOrdered = + node.style === 'numbered' || + node[DISCRIMINATOR] === 'ordered_list' || + node.listType === 'ordered'; + return { + type: isOrdered ? 'ol' : 'ul', uid: uid(), attrs: {}, + children: safe(convertChildren(node[CHILDREN_KEY], ctx)), + }; + } + + case 'listItem': + case 'list_item': + case 'li': { + const raw = convertChildren(node[CHILDREN_KEY], ctx); + // Rule 4 — li children must be block nodes, wrap text leaves in p + const children = raw.map((c: any) => + c.text !== undefined || !c.type + ? { type: 'p', uid: uid(), attrs: {}, children: [c] } + : c + ); + return { type: 'li', uid: uid(), attrs: {}, children: safe(children) }; + } + + case 'blockquote': + case 'block_quote': + case 'quote': + return { + type: 'blockquote', uid: uid(), attrs: {}, + children: safe(convertChildren(node[CHILDREN_KEY], ctx)), + }; + + case 'code_block': + case 'codeBlock': + case 'pre': + case 'fence': + return { + type: 'code', uid: uid(), + attrs: { 'code-type': node.language ?? node.attrs?.language ?? '' }, + children: safe(convertChildren(node[CHILDREN_KEY], ctx)), + }; + + case 'thematicBreak': + case 'horizontal_rule': + case 'hr': + case 'divider': + return { type: 'hr', uid: uid(), attrs: {}, children: [{ text: '' }] }; + + // ── Table (v4) ─────────────────────────────────────────────────────────── + + case 'table': + return { + type: 'table', uid: uid(), attrs: {}, + children: safe(convertChildren(node[CHILDREN_KEY], ctx)), + }; + + case 'table_row': + case 'tr': + return { + type: 'tr', uid: uid(), attrs: {}, + children: safe(convertChildren(node[CHILDREN_KEY], ctx)), + }; + + case 'table_cell': + case 'td': + return { + type: 'td', uid: uid(), attrs: {}, + children: safe(convertChildren(node[CHILDREN_KEY], ctx)), + }; + + case 'table_header': + case 'th': + return { + type: 'th', uid: uid(), attrs: {}, + children: safe(convertChildren(node[CHILDREN_KEY], ctx)), + }; + + // ── Inline nodes ───────────────────────────────────────────────────────── + + case 'link': + case 'hyperlink': + case 'anchor': { + const url = node.url ?? node.href ?? node.attrs?.url ?? node.data?.uri ?? ''; + return { + type: 'a', uid: uid(), + attrs: { url, target: url.startsWith('http') ? '_blank' : '' }, + children: safe(convertChildren(node[CHILDREN_KEY], ctx)), + }; + } + + // ── Leaf nodes ──────────────────────────────────────────────────────────── + + case 'span': + case 'text': { + // Detect which marks pattern this leaf uses + if (Array.isArray(node.marks)) return applyPattern1Marks(node); + return applyPattern2Marks(node); + } + + // ── Reference nodes ─────────────────────────────────────────────────────── + + case 'block': { + const sourceId = node.item ?? node.id ?? node.entryId; + if (!sourceId) { + console.warn('[rte-converter] block node missing id — stringify fallback'); + return { type: 'p', uid: uid(), attrs: {}, children: [{ text: JSON.stringify(node).slice(0, 200) }] }; + } + const entryUid = ctx.entryIdMap[sourceId]; + const ctUid = ctx.recordToCtUid[sourceId]; + if (!entryUid || !ctUid) { + console.warn(`[rte-converter] block "${sourceId}" not in lookup — stringify fallback`); + return { type: 'p', uid: uid(), attrs: {}, children: [{ text: JSON.stringify(node).slice(0, 200) }] }; + } + return { + uid: uid(), type: 'reference', + attrs: { + 'display-type': 'block', 'entry-uid': entryUid, + 'content-type-uid': ctUid, locale: ctx.locale, + type: 'entry', 'class-name': 'embedded-entry-block', + }, + children: [{ text: '' }], + }; + } + + case 'inlineItem': + case 'inline_item': { + const sourceId = node.item ?? node.id ?? node.entryId; + if (!sourceId) { + console.warn('[rte-converter] inlineItem node missing id — stringify fallback'); + return { type: 'p', uid: uid(), attrs: {}, children: [{ text: JSON.stringify(node).slice(0, 200) }] }; + } + const entryUid = ctx.entryIdMap[sourceId]; + const ctUid = ctx.recordToCtUid[sourceId]; + if (!entryUid || !ctUid) { + console.warn(`[rte-converter] inlineItem "${sourceId}" not in lookup — stringify fallback`); + return { type: 'p', uid: uid(), attrs: {}, children: [{ text: JSON.stringify(node).slice(0, 200) }] }; + } + return { + uid: uid(), type: 'reference', + attrs: { + 'display-type': 'inline', 'entry-uid': entryUid, + 'content-type-uid': ctUid, locale: ctx.locale, + type: 'entry', 'class-name': 'embedded-entry-inline', inline: true, + }, + children: [{ text: '' }], + }; + } + + case 'image': + case 'asset': + case 'media': { + const sourceId = node.item ?? node.id ?? node.asset?._ref ?? node.assetId; + const rec = sourceId ? ctx.assetIdMap[sourceId] : null; + if (!rec) { + console.warn(`[rte-converter] asset "${sourceId}" not in lookup — stringify fallback`); + return { type: 'p', uid: uid(), attrs: {}, children: [{ text: JSON.stringify(node).slice(0, 200) }] }; + } + return { + uid: uid(), type: 'reference', + attrs: { + 'display-type': 'display', 'asset-uid': rec.uid, + 'content-type-uid': 'sys_assets', 'asset-link': rec.urlPath, + 'asset-name': rec.title, 'asset-type': rec.content_type, + type: 'asset', 'class-name': 'embedded-asset', inline: false, + }, + children: [{ text: '' }], + }; + } + + // ── Fallback ladder (v4 — nothing dropped silently) ─────────────────────── + + default: { + const children = node[CHILDREN_KEY]; + const text = node.text ?? node.value; + + // Rung 1 — has children → convert children, wrap in p + if (Array.isArray(children) && children.length > 0) { + console.warn(`[rte-converter] unknown type "${node[DISCRIMINATOR]}" with children → p`); + return { + type: 'p', uid: uid(), attrs: {}, + children: safe(convertChildren(children, ctx)), + }; + } + + // Rung 2 — has text prop → extract as leaf + if (text !== undefined) { + console.warn(`[rte-converter] unknown type "${node[DISCRIMINATOR]}" with text → leaf`); + return { text: String(text) }; + } + + // Rung 3 — matches wrapper fingerprint → handled above (isWrapperNode check) + // If we reach here, it wasn't caught by isWrapperNode — safe to stringify. + + // Rung 4 — stringify escape hatch: content preserved, never null-dropped + console.warn(`[rte-converter] unknown type "${node[DISCRIMINATOR]}" — stringify fallback`); + return { + type: 'p', uid: uid(), attrs: {}, + children: [{ text: JSON.stringify(node).slice(0, 200) }], + }; + } + } +} + +// ── Public entry point ──────────────────────────────────────────────────────── + +/** + * Converts a source RTE value to a Contentstack JSON RTE doc. + * + * Replace and with the real names when generating. + * Set CHILDREN_KEY and DISCRIMINATOR at the top of the file. + * + * @param value Raw field value from the source export record + * @param entryIdMap sourceRecordId → cs entry uid + * @param recordToCtUid sourceRecordId → cs CT uid + * @param assetIdMap sourceAssetId → cs asset record object + * @param locale Master locale string (e.g. 'en-us') + */ +export function convertToCSRte( + value: unknown, + entryIdMap: Record = {}, + recordToCtUid: Record = {}, + assetIdMap: Record = {}, + locale = 'en-us', +): Record { + const emptyDoc = () => ({ + type: 'doc', uid: uid(), attrs: {}, + children: [{ type: 'p', uid: uid(), attrs: {}, children: [{ text: '' }] }], + }); + + // Adversarial input guard — null, undefined, {}, [] all return empty doc + if (!value || typeof value !== 'object') return emptyDoc(); + if (Array.isArray(value) && value.length === 0) return emptyDoc(); + + const ctx: Ctx = { entryIdMap, recordToCtUid, assetIdMap, locale }; + + // Handle wrapper formats: { schema:'dast', document:{...} } or { type:'doc', content:[...] } + // Adapt this unwrap for the specific source format found in Step 0 + const raw = value as any; + const root = raw.document ?? raw; + + // For array-root formats (Portable Text), wrap in doc manually: + // if (Array.isArray(value)) { + // return { type: 'doc', uid: uid(), attrs: {}, children: safe(convertChildren(value as any[], ctx)) }; + // } + + if (!root[CHILDREN_KEY] && !Array.isArray(root)) return emptyDoc(); + + const rootChildren = Array.isArray(root) + ? root + : root[CHILDREN_KEY]; + + const children = safe(convertChildren(rootChildren, ctx)); + + return { type: 'doc', uid: uid(), attrs: {}, children }; +}