diff --git a/.changeset/named-import-mappings.md b/.changeset/named-import-mappings.md new file mode 100644 index 0000000000..ec3dd08f27 --- /dev/null +++ b/.changeset/named-import-mappings.md @@ -0,0 +1,6 @@ +--- +'@objectstack/rest': minor +'@objectstack/spec': minor +--- + +feat(rest,spec): named import mappings (#2611) — `POST /data/:object/import` accepts `mappingName`, resolving a registered `defineMapping` artifact (stack `mappings:`) and applying its fieldMapping pipeline (rename + constant/map/split/join; lookup delegates to the built-in reference resolution) as a strict projection before coercion. The artifact's `mode`/`upsertKey` serve as writeMode/matchFields defaults; explicit request values win. Errors are loud and specific: `MAPPING_NOT_FOUND`, `MAPPING_TARGET_MISMATCH`, `MAPPING_FORMAT_MISMATCH`, `CONFLICTING_MAPPING` (mutually exclusive with the inline rename), and `UNSUPPORTED_TRANSFORM` for `javascript` (no server-side sandbox — never silently skipped). `defineStack` cross-reference validation now rejects mappings targeting undefined objects and `javascript` transforms at build time. diff --git a/content/docs/references/api/metadata.mdx b/content/docs/references/api/metadata.mdx index 6f4f80785e..1642554ad9 100644 --- a/content/docs/references/api/metadata.mdx +++ b/content/docs/references/api/metadata.mdx @@ -314,7 +314,7 @@ Metadata query with filtering, sorting, and pagination | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **types** | `Enum<'object' \| 'field' \| 'validation' \| 'hook' \| 'seed' \| 'view' \| 'page' \| 'dashboard' \| 'app' \| 'action' \| 'report' \| 'dataset' \| 'flow' \| 'job' \| 'datasource' \| 'external_catalog' \| 'translation' \| 'email_template' \| 'doc' \| 'book' \| 'permission' \| 'profile' \| 'role' \| 'agent' \| 'tool' \| 'skill'>[]` | optional | Filter by metadata types | +| **types** | `Enum<'object' \| 'field' \| 'validation' \| 'hook' \| 'seed' \| 'mapping' \| 'view' \| 'page' \| 'dashboard' \| 'app' \| 'action' \| 'report' \| 'dataset' \| 'flow' \| 'job' \| 'datasource' \| 'external_catalog' \| 'translation' \| 'email_template' \| 'doc' \| 'book' \| 'permission' \| 'profile' \| 'role' \| 'agent' \| 'tool' \| 'skill'>[]` | optional | Filter by metadata types | | **namespaces** | `string[]` | optional | Filter by namespaces | | **packageId** | `string` | optional | Filter by owning package | | **search** | `string` | optional | Full-text search query | @@ -349,7 +349,7 @@ Metadata query with filtering, sorting, and pagination | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **type** | `Enum<'object' \| 'field' \| 'validation' \| 'hook' \| 'seed' \| 'view' \| 'page' \| 'dashboard' \| 'app' \| 'action' \| 'report' \| 'dataset' \| 'flow' \| 'job' \| 'datasource' \| 'external_catalog' \| 'translation' \| 'email_template' \| 'doc' \| 'book' \| 'permission' \| 'profile' \| 'role' \| 'agent' \| 'tool' \| 'skill'>` | ✅ | Metadata type | +| **type** | `Enum<'object' \| 'field' \| 'validation' \| 'hook' \| 'seed' \| 'mapping' \| 'view' \| 'page' \| 'dashboard' \| 'app' \| 'action' \| 'report' \| 'dataset' \| 'flow' \| 'job' \| 'datasource' \| 'external_catalog' \| 'translation' \| 'email_template' \| 'doc' \| 'book' \| 'permission' \| 'profile' \| 'role' \| 'agent' \| 'tool' \| 'skill'>` | ✅ | Metadata type | | **name** | `string` | ✅ | Item name (snake_case) | | **data** | `Record` | ✅ | Metadata payload | | **namespace** | `string` | optional | Optional namespace | diff --git a/content/docs/references/kernel/metadata-plugin.mdx b/content/docs/references/kernel/metadata-plugin.mdx index 3620d322fb..f73f024796 100644 --- a/content/docs/references/kernel/metadata-plugin.mdx +++ b/content/docs/references/kernel/metadata-plugin.mdx @@ -130,7 +130,7 @@ const result = MetadataBulkRegisterRequest.parse(data); | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | **event** | `Enum<'metadata.registered' \| 'metadata.updated' \| 'metadata.unregistered' \| 'metadata.validated' \| 'metadata.deployed' \| 'metadata.overlay.applied' \| 'metadata.overlay.removed' \| 'metadata.imported' \| 'metadata.exported'>` | ✅ | Event type | -| **metadataType** | `Enum<'object' \| 'field' \| 'validation' \| 'hook' \| 'seed' \| 'view' \| 'page' \| 'dashboard' \| 'app' \| 'action' \| 'report' \| 'dataset' \| 'flow' \| 'job' \| 'datasource' \| 'external_catalog' \| 'translation' \| 'email_template' \| 'doc' \| 'book' \| 'permission' \| 'profile' \| 'role' \| 'agent' \| 'tool' \| 'skill'>` | ✅ | Metadata type | +| **metadataType** | `Enum<'object' \| 'field' \| 'validation' \| 'hook' \| 'seed' \| 'mapping' \| 'view' \| 'page' \| 'dashboard' \| 'app' \| 'action' \| 'report' \| 'dataset' \| 'flow' \| 'job' \| 'datasource' \| 'external_catalog' \| 'translation' \| 'email_template' \| 'doc' \| 'book' \| 'permission' \| 'profile' \| 'role' \| 'agent' \| 'tool' \| 'skill'>` | ✅ | Metadata type | | **name** | `string` | ✅ | Metadata item name | | **namespace** | `string` | optional | Namespace | | **packageId** | `string` | optional | Owning package ID | @@ -147,7 +147,7 @@ const result = MetadataBulkRegisterRequest.parse(data); | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **types** | `Enum<'object' \| 'field' \| 'validation' \| 'hook' \| 'seed' \| 'view' \| 'page' \| 'dashboard' \| 'app' \| 'action' \| 'report' \| 'dataset' \| 'flow' \| 'job' \| 'datasource' \| 'external_catalog' \| 'translation' \| 'email_template' \| 'doc' \| 'book' \| 'permission' \| 'profile' \| 'role' \| 'agent' \| 'tool' \| 'skill'>[]` | optional | Filter by metadata types | +| **types** | `Enum<'object' \| 'field' \| 'validation' \| 'hook' \| 'seed' \| 'mapping' \| 'view' \| 'page' \| 'dashboard' \| 'app' \| 'action' \| 'report' \| 'dataset' \| 'flow' \| 'job' \| 'datasource' \| 'external_catalog' \| 'translation' \| 'email_template' \| 'doc' \| 'book' \| 'permission' \| 'profile' \| 'role' \| 'agent' \| 'tool' \| 'skill'>[]` | optional | Filter by metadata types | | **namespaces** | `string[]` | optional | Filter by namespaces | | **packageId** | `string` | optional | Filter by owning package | | **search** | `string` | optional | Full-text search query | @@ -185,6 +185,7 @@ const result = MetadataBulkRegisterRequest.parse(data); * `validation` * `hook` * `seed` +* `mapping` * `view` * `page` * `dashboard` diff --git a/examples/app-showcase/objectstack.config.ts b/examples/app-showcase/objectstack.config.ts index 34c8033750..27c901909c 100644 --- a/examples/app-showcase/objectstack.config.ts +++ b/examples/app-showcase/objectstack.config.ts @@ -41,6 +41,7 @@ import { allPortals } from './src/ui/portals/index.js'; import { ShowcaseSeedData } from './src/data/seed/index.js'; import { allCubes } from './src/data/analytics/showcase.cube.js'; import { allObjectExtensions } from './src/data/extensions/account.extension.js'; +import { allMappings } from './src/data/mappings/index.js'; // Ambient `process` for the env-var overrides below — the showcase tsconfig // doesn't pull in `@types/node`, but the CLI provides the real `process` at @@ -164,6 +165,9 @@ export default defineStack({ // (`/api/v1/analytics/*`) — no `requires` token needed; the CLI always // loads it and registers these cubes (see src/data/analytics/). analyticsCubes: allCubes, + // Named import mappings (#2611) — resolved by the import endpoint via + // `mappingName` (see src/data/mappings/). + mappings: allMappings, // UI apps: [ShowcaseApp], diff --git a/examples/app-showcase/src/coverage.ts b/examples/app-showcase/src/coverage.ts index ef4a259bb3..c3325309fa 100644 --- a/examples/app-showcase/src/coverage.ts +++ b/examples/app-showcase/src/coverage.ts @@ -74,6 +74,12 @@ export const KIND_COVERAGE: Record = { }, hook: { status: 'demonstrated', files: ['src/data/hooks/index.ts'] }, seed: { status: 'demonstrated', files: ['src/data/seed/index.ts'] }, + mapping: { + status: 'demonstrated', + files: ['src/data/mappings/index.ts'], + notes: + 'Named import mapping resolved via mappingName at POST /data/:object/import (#2611); promoted to a registry kind per the ADR-0088 admission test.', + }, // ── ui ── view: { status: 'demonstrated', files: ['src/ui/views/task.view.ts', 'src/ui/views/project.view.ts'] }, @@ -165,12 +171,6 @@ export const STACK_COLLECTION_COVERAGE: Record = { notes: 'Declarative ApiEndpoint metadata (object_operation + flow targets), executed by the runtime dispatcher (handleApiEndpoint). Complements the code-mounted endpoint in src/system/server/ (router kind stays waived: code-only).', }, - mappings: { - status: 'waived', - reason: - 'defineMapping artifacts are registered but never consumed — the REST import path only accepts an inline per-request mapping, so a stack-level mapping is inert.', - issue: 'https://github.com/objectstack-ai/framework/issues/2611', - }, connectors: { status: 'waived', reason: diff --git a/examples/app-showcase/src/data/mappings/index.ts b/examples/app-showcase/src/data/mappings/index.ts new file mode 100644 index 0000000000..2d409a9758 --- /dev/null +++ b/examples/app-showcase/src/data/mappings/index.ts @@ -0,0 +1,41 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +import { defineMapping } from '@objectstack/spec'; + +/** + * Named import mapping (#2611) — the reusable, governed counterpart of the + * import wizard's one-off inline rename. This one models the classic + * enterprise case: a marketing tool exports leads weekly with ITS column + * names and ITS channel codes; the mapping is defined once, ships with the + * package, and every import (human or scheduled job) references it by name: + * + * POST /api/v1/data/showcase_inquiry/import + * { "format": "csv", "csv": "...", "mappingName": "showcase_inquiry_feed" } + * + * mode/upsertKey make re-importing the same file idempotent (dedupe on + * email) — no writeMode/matchFields needed on the request. The `map` + * transform translates the source system's channel codes; the import + * pipeline's own coercion + reference resolution still run afterwards. + */ +export const InquiryFeedMapping = defineMapping({ + name: 'showcase_inquiry_feed', + label: 'Inquiry feed (marketing CSV)', + sourceFormat: 'csv', + targetObject: 'showcase_inquiry', + fieldMapping: [ + { source: 'Full Name', target: 'name' }, + { source: 'E-mail', target: 'email' }, + { source: 'Company', target: 'company' }, + { source: 'Message', target: 'message' }, + { + source: 'Channel', + target: 'source', + transform: 'map', + params: { valueMap: { Webform: 'website', 'Partner Referral': 'referral' } }, + }, + ], + mode: 'upsert', + upsertKey: ['email'], +}); + +export const allMappings = [InquiryFeedMapping]; diff --git a/examples/app-showcase/src/docs/showcase_tour_data.md b/examples/app-showcase/src/docs/showcase_tour_data.md index 4512d461a9..3073ec359f 100644 --- a/examples/app-showcase/src/docs/showcase_tour_data.md +++ b/examples/app-showcase/src/docs/showcase_tour_data.md @@ -45,11 +45,17 @@ object: showcase_project name: project_status_flow ``` -## Hooks & seed data +## Hooks, seed data & import mappings - `src/data/hooks/` — data-layer lifecycle hooks (before/after CRUD). - `src/data/seed/` — the seed dataset that makes every view render something real on first boot. +- `src/data/mappings/` — a **named import mapping**: the marketing tool's + CSV columns (`Full Name`, `Channel`, …) map onto `showcase_inquiry` once, + as governed metadata; every import then just passes + `mappingName: "showcase_inquiry_feed"` to + `POST /api/v1/data/showcase_inquiry/import` — re-importing the same file + is idempotent (artifact-declared upsert on email). ## Extensions & analytics diff --git a/examples/app-showcase/test/gap-fill.test.ts b/examples/app-showcase/test/gap-fill.test.ts index 967609dd56..708cfd8545 100644 --- a/examples/app-showcase/test/gap-fill.test.ts +++ b/examples/app-showcase/test/gap-fill.test.ts @@ -33,6 +33,33 @@ describe('showcase gap fill — analytics cube', () => { }); }); +describe('showcase gap fill — named import mapping (#2611)', () => { + it('is wired into the stack definition', () => { + const mappings = (stack as { mappings?: Array<{ name: string }> }).mappings ?? []; + expect(mappings.map((m) => m.name)).toContain('showcase_inquiry_feed'); + }); + + it('targets an existing object and its upsertKey fields exist on that object', () => { + const mappings = (stack as { mappings?: Array<{ targetObject: string; upsertKey?: string[]; fieldMapping: Array<{ target: string | string[] }> }> }).mappings ?? []; + const objects = ((stack as { objects?: Array<{ name: string; fields?: Record }> }).objects ?? []); + for (const m of mappings) { + const target = objects.find((o) => o.name === m.targetObject); + expect(target, `mapping targets missing object '${m.targetObject}'`).toBeDefined(); + const fieldNames = new Set(Object.keys(target?.fields ?? {})); + for (const key of m.upsertKey ?? []) { + expect(fieldNames.has(key), `upsertKey '${key}' not a field of '${m.targetObject}'`).toBe(true); + } + // Every mapped target lands on a real field — a typo'd target would + // silently import into nowhere. + for (const entry of m.fieldMapping) { + for (const t of Array.isArray(entry.target) ? entry.target : [entry.target]) { + expect(fieldNames.has(t), `mapped target '${t}' not a field of '${m.targetObject}'`).toBe(true); + } + } + } + }); +}); + describe('showcase gap fill — declarative api endpoints', () => { it('is wired into the stack definition', () => { const apis = (stack as { apis?: Array<{ name: string }> }).apis ?? []; diff --git a/packages/metadata-core/src/types.ts b/packages/metadata-core/src/types.ts index ff32d7c5cf..09f646c6d7 100644 --- a/packages/metadata-core/src/types.ts +++ b/packages/metadata-core/src/types.ts @@ -21,6 +21,7 @@ export const MetadataTypeSchema = z.enum([ 'field', 'validation', 'hook', + 'mapping', 'view', 'page', 'dashboard', diff --git a/packages/rest/src/import-integration.test.ts b/packages/rest/src/import-integration.test.ts index 4a7e7ebb46..54426b3c7d 100644 --- a/packages/rest/src/import-integration.test.ts +++ b/packages/rest/src/import-integration.test.ts @@ -309,3 +309,110 @@ describe('import route — real engine + protocol integration', () => { expect(String(res._json.error)).toMatch(/xlsx/i); }); }); + +// --------------------------------------------------------------------------- +// Named mapping artifacts (#2611) — `mappingName` resolves a registered +// `mapping` item and applies its fieldMapping pipeline before coercion. +// --------------------------------------------------------------------------- +describe('import route — named mapping artifact (#2611)', () => { + let route: any; + let engine: any; + + const TASK_CSV_MAPPING = { + name: 'task_feed_import', + label: 'Task feed import', + sourceFormat: 'csv', + targetObject: 'task', + fieldMapping: [ + { source: 'ID', target: 'id', transform: 'none' }, + { source: 'Task Title', target: 'title', transform: 'none' }, + // Source system codes → select LABELS; the built-in metaMap coercion + // then turns the label (高/低) into the storage code (high/low) — + // the artifact transform and the coercion pipeline COMPOSE. + { source: 'Prio', target: 'priority', transform: 'map', params: { valueMap: { P1: '高', P3: '低' } } }, + { source: 'Assignee', target: 'owner', transform: 'lookup' }, + { source: 'ignored_by_projection', target: 'score', transform: 'constant', params: { value: 5 } }, + ], + mode: 'upsert', + upsertKey: ['id'], + }; + + beforeEach(async () => { + ({ route, engine } = await boot()); + engine.registry.registerItem('mapping', TASK_CSV_MAPPING as any, 'name'); + engine.registry.registerItem( + 'mapping', + { ...TASK_CSV_MAPPING, name: 'user_only_mapping', targetObject: 'user' } as any, + 'name', + ); + engine.registry.registerItem( + 'mapping', + { + name: 'task_js_mapping', targetObject: 'task', sourceFormat: 'csv', + fieldMapping: [{ source: 'x', target: 'title', transform: 'javascript' }], + } as any, + 'name', + ); + engine.registry.registerItem( + 'mapping', + { ...TASK_CSV_MAPPING, name: 'task_json_mapping', sourceFormat: 'json' } as any, + 'name', + ); + }); + + it('applies rename + map + constant + lookup, strict projection, artifact upsert defaults', async () => { + const csv = [ + 'ID,Task Title,Prio,Assignee,Junk Column', + 't1,迁移旧数据,P1,张三,DROP-ME', + 't2,巡检,P3,李四,DROP-ME-TOO', + ].join('\n'); + // No writeMode/matchFields in the request — the artifact's + // mode:'upsert' + upsertKey:['id'] apply as defaults. + const res = await call(route, { format: 'csv', csv, mappingName: 'task_feed_import' }); + expect(res._json).toMatchObject({ total: 2, ok: 2 }); + + const one = await engine.findOne('task', { where: { id: 't1' } }); + // map: P1→高, then coercion 高→high; lookup: 张三→u1 via metaMap; + // constant: score=5; strict projection: Junk Column never lands. + expect(one).toMatchObject({ title: '迁移旧数据', priority: 'high', owner: 'u1', score: 5 }); + expect(one['Junk Column']).toBeUndefined(); + + // Re-import the same file → artifact upsert semantics update, not dupe. + const res2 = await call(route, { format: 'csv', csv, mappingName: 'task_feed_import' }); + expect(res2._json.ok).toBe(2); + const all = await engine.find('task', { where: {} }); + expect(all.filter((r: any) => r.id === 't1')).toHaveLength(1); + }); + + it('404s on an unknown mappingName', async () => { + const res = await call(route, { format: 'csv', csv: 'ID\nx', mappingName: 'nope' }); + expect(res._status).toBe(404); + expect(res._json.code).toBe('MAPPING_NOT_FOUND'); + }); + + it('400s when the mapping targets a different object', async () => { + const res = await call(route, { format: 'csv', csv: 'ID\nx', mappingName: 'user_only_mapping' }); + expect(res._status).toBe(400); + expect(res._json.code).toBe('MAPPING_TARGET_MISMATCH'); + }); + + it('400s when mappingName and an inline mapping are both provided', async () => { + const res = await call(route, { + format: 'csv', csv: 'ID\nx', mappingName: 'task_feed_import', mapping: { ID: 'id' }, + }); + expect(res._status).toBe(400); + expect(res._json.code).toBe('CONFLICTING_MAPPING'); + }); + + it('400s on a javascript transform instead of silently skipping it', async () => { + const res = await call(route, { format: 'csv', csv: 'x\n1', mappingName: 'task_js_mapping' }); + expect(res._status).toBe(400); + expect(res._json.code).toBe('UNSUPPORTED_TRANSFORM'); + }); + + it('400s when the payload format contradicts the artifact sourceFormat', async () => { + const res = await call(route, { format: 'csv', csv: 'ID\nx', mappingName: 'task_json_mapping' }); + expect(res._status).toBe(400); + expect(res._json.code).toBe('MAPPING_FORMAT_MISMATCH'); + }); +}); diff --git a/packages/rest/src/import-mapping.test.ts b/packages/rest/src/import-mapping.test.ts new file mode 100644 index 0000000000..fb98cde3e2 --- /dev/null +++ b/packages/rest/src/import-mapping.test.ts @@ -0,0 +1,66 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect } from 'vitest'; +import { applyMappingToRows, type MappingArtifactLike } from './import-mapping'; + +const artifact = (fieldMapping: MappingArtifactLike['fieldMapping']): MappingArtifactLike => ({ + name: 'm', targetObject: 'o', fieldMapping, +}); + +describe('applyMappingToRows — transform semantics', () => { + it('none renames; output is a strict projection (unmapped columns drop)', () => { + const r = applyMappingToRows( + [{ A: '1', Junk: 'x' }], + artifact([{ source: 'A', target: 'a', transform: 'none' }]), + ); + expect(r).toEqual({ ok: true, rows: [{ a: '1' }] }); + }); + + it('constant writes params.value regardless of the source cell', () => { + const r = applyMappingToRows( + [{ A: 'whatever' }], + artifact([{ source: 'A', target: 'tier', transform: 'constant', params: { value: 'gold' } }]), + ); + expect(r).toEqual({ ok: true, rows: [{ tier: 'gold' }] }); + }); + + it('map translates known values and passes unknown ones through', () => { + const r = applyMappingToRows( + [{ S: 'Open' }, { S: 'Weird' }], + artifact([{ source: 'S', target: 's', transform: 'map', params: { valueMap: { Open: 'draft' } } }]), + ); + expect(r).toEqual({ ok: true, rows: [{ s: 'draft' }, { s: 'Weird' }] }); + }); + + it('split fans one column into positional targets (missing parts → undefined)', () => { + const r = applyMappingToRows( + [{ Name: 'John Doe' }, { Name: 'Cher' }], + artifact([{ source: 'Name', target: ['first', 'last'], transform: 'split', params: { separator: ' ' } }]), + ); + expect(r).toEqual({ ok: true, rows: [{ first: 'John', last: 'Doe' }, { first: 'Cher', last: undefined }] }); + }); + + it('join concatenates source columns, skipping empties', () => { + const r = applyMappingToRows( + [{ City: 'Berlin', Street: 'Unter den Linden' }, { City: 'Rome', Street: '' }], + artifact([{ source: ['City', 'Street'], target: 'address', transform: 'join', params: { separator: ', ' } }]), + ); + expect(r).toEqual({ ok: true, rows: [{ address: 'Berlin, Unter den Linden' }, { address: 'Rome' }] }); + }); + + it('lookup copies the raw value through (metaMap resolves it downstream)', () => { + const r = applyMappingToRows( + [{ Owner: '张三' }], + artifact([{ source: 'Owner', target: 'owner', transform: 'lookup' }]), + ); + expect(r).toEqual({ ok: true, rows: [{ owner: '张三' }] }); + }); + + it('rejects an unknown transform loudly', () => { + const r = applyMappingToRows( + [{ A: '1' }], + artifact([{ source: 'A', target: 'a', transform: 'zip' as never }]), + ); + expect(r).toMatchObject({ ok: false, status: 400, code: 'UNSUPPORTED_TRANSFORM' }); + }); +}); diff --git a/packages/rest/src/import-mapping.ts b/packages/rest/src/import-mapping.ts new file mode 100644 index 0000000000..ce20cad4a0 --- /dev/null +++ b/packages/rest/src/import-mapping.ts @@ -0,0 +1,167 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Named import mappings (#2611) — resolve a registered `mapping` artifact + * (`defineMapping`, stack `mappings:`) by name and apply its `fieldMapping` + * pipeline to parsed rows. + * + * Seam with the inline request `mapping`: + * • inline = a plain `{ sourceColumn: targetField }` RENAME for one-off, + * wizard-driven imports — unmapped columns pass through untouched. + * • artifact = a reusable, governed ETL projection for recurring / + * programmatic imports — the output row contains ONLY the mapped + * targets (a strict projection; source files from external systems + * routinely carry junk columns that must not leak into the write path). + * + * Transform support (Prime Directive #10 — implement or reject loudly): + * none/constant/map/split/join — applied here. + * lookup — the value is copied through; the import pipeline's built-in + * reference resolution (metaMap) turns lookup names into record ids, + * so a dedicated re-implementation here would be a second dialect. + * javascript — REJECTED (400). No server-side sandbox is wired into the + * import path yet; silently skipping a declared transform would corrupt + * data. Tracked on framework#2611. + */ + +export interface MappingArtifactLike { + name: string; + targetObject: string; + sourceFormat?: 'csv' | 'json' | 'xml' | 'sql'; + fieldMapping: Array<{ + source: string | string[]; + target: string | string[]; + transform?: 'none' | 'constant' | 'lookup' | 'split' | 'join' | 'javascript' | 'map'; + params?: { + value?: unknown; + valueMap?: Record; + separator?: string; + } & Record; + }>; + mode?: 'insert' | 'update' | 'upsert'; + upsertKey?: string[]; +} + +export type MappingFailure = { ok: false; status: number; code: string; error: string }; +export type ResolveMappingResult = { ok: true; artifact: MappingArtifactLike } | MappingFailure; + +/** Unwrap `{ item }` meta envelopes returned by protocol getMetaItem. */ +function unwrapEnvelope(r: unknown): unknown { + if (r && typeof r === 'object' && 'item' in (r as Record)) { + return (r as Record).item; + } + return r; +} + +/** + * Resolve a named mapping artifact and check it against the request: + * target object must match the URL object, and the artifact's declared + * sourceFormat (when set) must match the payload format actually sent. + */ +export async function resolveNamedMapping( + p: { getMetaItem?: (req: { type: string; name: string }) => Promise }, + opts: { mappingName: string; objectName: string; detectedFormat: 'csv' | 'json' | 'xlsx' }, +): Promise { + const { mappingName, objectName, detectedFormat } = opts; + if (typeof p?.getMetaItem !== 'function') { + return { ok: false, status: 500, code: 'INTERNAL', error: 'Metadata protocol unavailable; cannot resolve mappingName' }; + } + let artifact: MappingArtifactLike | undefined; + try { + artifact = unwrapEnvelope(await p.getMetaItem({ type: 'mapping', name: mappingName })) as MappingArtifactLike | undefined; + } catch { /* treated as not found below */ } + if (!artifact || typeof artifact !== 'object' || !Array.isArray(artifact.fieldMapping)) { + return { ok: false, status: 404, code: 'MAPPING_NOT_FOUND', error: `No mapping artifact named "${mappingName}" is registered` }; + } + if (artifact.targetObject !== objectName) { + return { + ok: false, status: 400, code: 'MAPPING_TARGET_MISMATCH', + error: `Mapping "${mappingName}" targets object "${artifact.targetObject}", not "${objectName}"`, + }; + } + const declared = artifact.sourceFormat; + if (declared === 'xml' || declared === 'sql') { + return { + ok: false, status: 400, code: 'MAPPING_FORMAT_UNSUPPORTED', + error: `Mapping "${mappingName}" declares sourceFormat "${declared}", which the import endpoint does not accept (csv/json/xlsx)`, + }; + } + // xlsx rows are tabular like csv; a csv-declared mapping applies to both. + const compatible = declared === undefined + || (declared === 'json' && detectedFormat === 'json') + || (declared === 'csv' && (detectedFormat === 'csv' || detectedFormat === 'xlsx')); + if (!compatible) { + return { + ok: false, status: 400, code: 'MAPPING_FORMAT_MISMATCH', + error: `Mapping "${mappingName}" declares sourceFormat "${declared}" but the payload is "${detectedFormat}"`, + }; + } + for (const entry of artifact.fieldMapping) { + if (entry?.transform === 'javascript') { + return { + ok: false, status: 400, code: 'UNSUPPORTED_TRANSFORM', + error: `Mapping "${mappingName}" uses transform "javascript", which the import path does not execute (no server-side sandbox; see framework#2611)`, + }; + } + } + return { ok: true, artifact }; +} + +const first = (v: string | string[]): string => (Array.isArray(v) ? v[0] : v); + +/** + * Apply the artifact's fieldMapping pipeline to raw parsed rows (headers as + * in the source file). Returns NEW rows containing only mapped targets. + */ +export function applyMappingToRows( + rows: Array>, + artifact: MappingArtifactLike, +): { ok: true; rows: Array> } | MappingFailure { + const out: Array> = []; + for (const row of rows) { + const mapped: Record = {}; + for (const entry of artifact.fieldMapping) { + const transform = entry.transform ?? 'none'; + const sep = entry.params?.separator ?? ' '; + switch (transform) { + case 'none': + case 'lookup': { // lookup values resolve downstream via metaMap + mapped[first(entry.target)] = row[first(entry.source)]; + break; + } + case 'constant': { + mapped[first(entry.target)] = entry.params?.value; + break; + } + case 'map': { + const raw = row[first(entry.source)]; + const valueMap = entry.params?.valueMap ?? {}; + mapped[first(entry.target)] = + typeof raw === 'string' && raw in valueMap ? valueMap[raw] : raw; + break; + } + case 'split': { + const raw = row[first(entry.source)]; + const targets = Array.isArray(entry.target) ? entry.target : [entry.target]; + const parts = typeof raw === 'string' ? raw.split(sep) : []; + targets.forEach((t, i) => { mapped[t] = parts[i]?.trim(); }); + break; + } + case 'join': { + const sources = Array.isArray(entry.source) ? entry.source : [entry.source]; + mapped[first(entry.target)] = sources + .map((s) => row[s]) + .filter((v) => v !== undefined && v !== null && v !== '') + .join(sep); + break; + } + default: + return { + ok: false, status: 400, code: 'UNSUPPORTED_TRANSFORM', + error: `Mapping "${artifact.name}" uses unknown transform "${transform}"`, + }; + } + } + out.push(mapped); + } + return { ok: true, rows: out }; +} diff --git a/packages/rest/src/rest-server.ts b/packages/rest/src/rest-server.ts index 14c8c5e469..c7ca3f6ba5 100644 --- a/packages/rest/src/rest-server.ts +++ b/packages/rest/src/rest-server.ts @@ -13,6 +13,7 @@ import { type ExportFieldMeta, } from './export-format.js'; import { runImport } from './import-runner.js'; +import { resolveNamedMapping, applyMappingToRows, type MappingArtifactLike } from './import-mapping.js'; // Node-safe logger — avoids importing 'console' which is absent from ES2020 lib typings. const logError = (...args: unknown[]) => (globalThis as any).console?.error(...args); @@ -486,9 +487,9 @@ async function prepareImportRequest( const { p, objectName, environmentId, maxRows } = opts; const dryRun = body?.dryRun === true; - const writeMode: 'insert' | 'update' | 'upsert' = + let writeMode: 'insert' | 'update' | 'upsert' = body?.writeMode === 'update' || body?.writeMode === 'upsert' ? body.writeMode : 'insert'; - const matchFields: string[] = Array.isArray(body?.matchFields) + let matchFields: string[] = Array.isArray(body?.matchFields) ? body.matchFields.filter((f: any) => typeof f === 'string' && f.length > 0) : []; const runAutomations = body?.runAutomations === true; @@ -499,6 +500,43 @@ async function prepareImportRequest( const createMissingOptions = body?.createMissingOptions === true; const skipBlankMatchKey = body?.skipBlankMatchKey === true; + // ── Named mapping artifact (#2611) ──────────────────────────────── + // `mappingName` references a registered `mapping` artifact + // (defineMapping / stack `mappings:`) — the reusable, governed form for + // recurring & programmatic imports. Mutually exclusive with the inline + // `mapping` rename: one mapping source of truth per request. + const mappingName: string | undefined = + typeof body?.mappingName === 'string' && body.mappingName.length > 0 ? body.mappingName : undefined; + const hasInlineMapping = + (Array.isArray(body?.mapping) && body.mapping.length > 0) || + (!Array.isArray(body?.mapping) && body?.mapping && typeof body.mapping === 'object' && Object.keys(body.mapping).length > 0); + if (mappingName && hasInlineMapping) { + return { ok: false, status: 400, code: 'CONFLICTING_MAPPING', error: 'Provide either mappingName or an inline mapping, not both' }; + } + let mappingArtifact: MappingArtifactLike | undefined; + if (mappingName) { + const detectedFormat: 'csv' | 'json' | 'xlsx' | undefined = + (body?.format === 'json' && Array.isArray(body?.rows)) || Array.isArray(body) ? 'json' + : typeof body?.csv === 'string' ? 'csv' + : typeof body?.xlsxBase64 === 'string' ? 'xlsx' + : undefined; + if (!detectedFormat) { + return { ok: false, status: 400, code: 'INVALID_REQUEST', error: 'Provide format:"csv" with csv text, format:"json" with rows[], or format:"xlsx" with xlsxBase64' }; + } + const resolved = await resolveNamedMapping(p, { mappingName, objectName, detectedFormat }); + if (!resolved.ok) return resolved; + mappingArtifact = resolved.artifact; + // Artifact-declared write semantics apply as DEFAULTS: an explicit + // request writeMode/matchFields wins; absent ones fall back to the + // artifact's mode/upsertKey. + if (body?.writeMode === undefined && (mappingArtifact.mode === 'update' || mappingArtifact.mode === 'upsert')) { + writeMode = mappingArtifact.mode; + } + if (matchFields.length === 0 && Array.isArray(mappingArtifact.upsertKey)) { + matchFields = mappingArtifact.upsertKey.filter((f) => typeof f === 'string' && f.length > 0); + } + } + if (writeMode !== 'insert' && matchFields.length === 0) { return { ok: false, status: 400, code: 'INVALID_REQUEST', error: `writeMode "${writeMode}" requires a non-empty matchFields[]` }; } @@ -550,6 +588,15 @@ async function prepareImportRequest( return { ok: false, status: 413, code: 'PAYLOAD_TOO_LARGE', error: `Import limit is ${maxRows} rows per request (got ${rows.length}).` }; } + // Apply the named mapping's fieldMapping pipeline (rename + transforms; + // strict projection — only mapped targets reach the write path). Inline + // `mapping` was empty in this branch, so rows still carry raw headers. + if (mappingArtifact) { + const applied = applyMappingToRows(rows, mappingArtifact); + if (!applied.ok) return applied; + rows = applied.rows; + } + // Resolve the object's field metadata so cells coerce to storage values // (booleans, numbers, dates→ISO, select label→code) and lookup names resolve // to record ids. Best-effort: a failed lookup leaves `metaMap` empty and diff --git a/packages/spec/src/kernel/metadata-create-seeds.ts b/packages/spec/src/kernel/metadata-create-seeds.ts index 0fa34e8983..c6cef2410e 100644 --- a/packages/spec/src/kernel/metadata-create-seeds.ts +++ b/packages/spec/src/kernel/metadata-create-seeds.ts @@ -90,6 +90,13 @@ const BUILTIN_METADATA_CREATE_SEEDS: Partial> = { object: PLACEHOLDER_OBJECT, events: [], }, + mapping: { + name: 'new_mapping', + label: 'New Import Mapping', + targetObject: PLACEHOLDER_OBJECT, + // At least one rename so the wizard's save produces a usable artifact. + fieldMapping: [{ source: 'Column A', target: 'field_a' }], + }, dataset: { name: 'new_dataset', label: 'New Dataset', diff --git a/packages/spec/src/kernel/metadata-plugin.test.ts b/packages/spec/src/kernel/metadata-plugin.test.ts index 5e3e6633b1..59ab12b206 100644 --- a/packages/spec/src/kernel/metadata-plugin.test.ts +++ b/packages/spec/src/kernel/metadata-plugin.test.ts @@ -18,7 +18,7 @@ describe('MetadataPluginProtocol', () => { describe('MetadataTypeSchema', () => { it('should accept all built-in metadata types', () => { const types = [ - 'object', 'field', 'validation', 'hook', 'seed', + 'object', 'field', 'validation', 'hook', 'seed', 'mapping', 'view', 'page', 'dashboard', 'app', 'action', 'report', 'flow', // ADR-0020: `workflow` retired as a metadata type // ADR-0088: `trigger`/`router`/`function`/`service` retired as kinds diff --git a/packages/spec/src/kernel/metadata-plugin.zod.ts b/packages/spec/src/kernel/metadata-plugin.zod.ts index 35d24433b7..e83e222133 100644 --- a/packages/spec/src/kernel/metadata-plugin.zod.ts +++ b/packages/spec/src/kernel/metadata-plugin.zod.ts @@ -79,6 +79,7 @@ export const MetadataTypeSchema = lazySchema(() => z.enum([ 'validation', // Validation rules (ValidationSchema) 'hook', // Data hooks (HookSchema) 'seed', // Seed/fixture data — runtime-draftable; publishing applies it (SeedSchema) + 'mapping', // Import/export field mappings (MappingSchema) — consumed by POST /data/:object/import via mappingName (#2611); promoted to a kind per the ADR-0088 admission test once the consumer landed // UI Protocol 'view', // List/form views (ViewSchema) @@ -608,6 +609,12 @@ export const DEFAULT_METADATA_TYPE_REGISTRY: MetadataTypeRegistryEntry[] = [ // last in the data domain so every referenced object/field already exists. // NOTE: distinct from the (analytics-bound) `dataset` name — see ADR. { type: 'seed', label: 'Seed Data', description: 'Fixture / initialization data applied on publish', filePatterns: ['**/*.seed.ts', '**/*.seed.yml', '**/*.seed.json'], supportsOverlay: false, allowOrgOverride: false, allowRuntimeCreate: true, supportsVersioning: true, executionPinned: false, loadOrder: 95, domain: 'data' }, + // `mapping` (#2611, admitted per ADR-0088): reusable import/export field + // mappings, consumed by the import endpoint via `mappingName`. + // `allowRuntimeCreate: true` so the import wizard can SAVE a hand-built + // mapping as a named artifact; packaged mappings stay locked + // (`allowOrgOverride: false`) like every artifact-backed item. + { type: 'mapping', label: 'Import Mapping', description: 'Reusable import/export field mapping (rename + transforms), referenced by name at import', filePatterns: ['**/*.mapping.ts', '**/*.mapping.yml', '**/*.mapping.json'], supportsOverlay: false, allowOrgOverride: false, allowRuntimeCreate: true, supportsVersioning: true, executionPinned: false, loadOrder: 96, domain: 'data' }, // UI Protocol // `view/page/dashboard/action/report`: UI artifacts benefit from version diff --git a/packages/spec/src/kernel/metadata-type-schemas.ts b/packages/spec/src/kernel/metadata-type-schemas.ts index 6cd1c47ff6..754532c03d 100644 --- a/packages/spec/src/kernel/metadata-type-schemas.ts +++ b/packages/spec/src/kernel/metadata-type-schemas.ts @@ -32,6 +32,7 @@ import { HookSchema } from '../data/hook.zod'; import { ValidationRuleSchema } from '../data/validation.zod'; import { DatasourceSchema } from '../data/datasource.zod'; import { SeedSchema } from '../data/seed.zod'; +import { MappingSchema } from '../data/mapping.zod'; import { ViewSchema } from '../ui/view.zod'; import { PageSchema } from '../ui/page.zod'; @@ -72,6 +73,7 @@ const BUILTIN_METADATA_TYPE_SCHEMAS: Partial> = hook: HookSchema, validation: ValidationRuleSchema, seed: SeedSchema, // fixture/init data; runtime-draftable, applied on publish + mapping: MappingSchema as unknown as z.ZodType, // #2611: reusable import mapping; runtime-creatable so the wizard can save one // UI Protocol view: ViewSchema, diff --git a/packages/spec/src/stack.test.ts b/packages/spec/src/stack.test.ts index 18e4aed419..27988477d0 100644 --- a/packages/spec/src/stack.test.ts +++ b/packages/spec/src/stack.test.ts @@ -895,6 +895,60 @@ describe('defineStack - Seed Data Cross-Reference Validation', () => { }); }); +describe('defineStack - Mapping Cross-Reference Validation (#2611)', () => { + const baseManifest = { + id: 'com.example.test', + name: 'test-project', + version: '1.0.0', + type: 'app' as const, + }; + const account = { name: 'account', fields: { name: { type: 'text' } } }; + + it('should detect a mapping targeting an undefined object', () => { + const config = { + manifest: baseManifest, + objects: [account], + mappings: [{ + name: 'lead_import', + targetObject: 'ghost_object', + fieldMapping: [{ source: 'Name', target: 'name' }], + }], + }; + expect(() => defineStack(config)).toThrow('ghost_object'); + expect(() => defineStack(config)).toThrow('cross-reference validation failed'); + }); + + it('should reject a javascript transform at build time (no server-side sandbox)', () => { + const config = { + manifest: baseManifest, + objects: [account], + mappings: [{ + name: 'lead_import', + targetObject: 'account', + fieldMapping: [{ source: 'Name', target: 'name', transform: 'javascript' as const }], + }], + }; + expect(() => defineStack(config)).toThrow('javascript'); + expect(() => defineStack(config)).toThrow('cross-reference validation failed'); + }); + + it('should pass a well-formed mapping (rename + map transform)', () => { + const config = { + manifest: baseManifest, + objects: [account], + mappings: [{ + name: 'lead_import', + targetObject: 'account', + fieldMapping: [ + { source: 'Company', target: 'name' }, + { source: 'Stage', target: 'name', transform: 'map' as const, params: { valueMap: { Hot: 'active' } } }, + ], + }], + }; + expect(() => defineStack(config)).not.toThrow(); + }); +}); + describe('defineStack - Navigation Cross-Reference Validation', () => { const baseManifest = { id: 'com.example.test', diff --git a/packages/spec/src/stack.zod.ts b/packages/spec/src/stack.zod.ts index 615ee01e92..5f5fa6493c 100644 --- a/packages/spec/src/stack.zod.ts +++ b/packages/spec/src/stack.zod.ts @@ -612,6 +612,30 @@ function validateCrossReferences(config: ObjectStackDefinition): string[] { } } + // Validate mapping → object references + executable-transform gate (#2611). + // A mapping whose targetObject doesn't exist can never be applied by the + // import endpoint (it 400s on target mismatch), and a `javascript` + // transform has no server-side sandbox — both must fail at build time, + // not at first use (Prime Directive #12: reject at the producer). + if (config.mappings) { + for (const m of config.mappings) { + if (m.targetObject && !objectNames.has(m.targetObject)) { + errors.push( + `Mapping '${m.name}' targets object '${m.targetObject}' which is not defined in objects.`, + ); + } + for (const entry of m.fieldMapping ?? []) { + if (entry.transform === 'javascript') { + errors.push( + `Mapping '${m.name}' uses transform 'javascript', which the import path does not execute ` + + `(no server-side sandbox — see framework#2611). Use none/constant/map/split/join/lookup, ` + + `or model the logic as a flow.`, + ); + } + } + } + } + // Validate permission-set / profile object grants → object references. // A grant keyed by an object that isn't declared (e.g. a short `lead` instead // of the namespaced `crm_lead`) silently applies to NOTHING: the