diff --git a/apps/console/vite.config.ts b/apps/console/vite.config.ts index 6916ef387..5ac046ded 100644 --- a/apps/console/vite.config.ts +++ b/apps/console/vite.config.ts @@ -101,6 +101,23 @@ const workspaceAliases: Record = { '@object-ui/plugin-designer': path.resolve(__dirname, '../../packages/plugin-designer/src'), }; +// Opt-in override of the installed `@objectstack/client`. The published client +// (11.2.0) predates the async import-job API (`data.createImportJob` et al.), +// so to exercise the full background-import + undo flow through the real +// console before that client ships, point OBJECTSTACK_CLIENT_DIST at a locally +// built client (its dist entry or package dir). Inert when unset — production +// and CI builds use the installed client unchanged. +const clientDistOverride = process.env.OBJECTSTACK_CLIENT_DIST; +// Extra dirs the dev server may read the override from — it lives outside the +// workspace root, so Vite's default `server.fs.allow` would 403 it (blank page). +const clientFsAllow: string[] = []; +if (clientDistOverride) { + const resolved = path.resolve(clientDistOverride); + workspaceAliases['@objectstack/client'] = resolved; + // Allow the containing package (…/dist/index.mjs → …/) so Vite can serve it. + clientFsAllow.push(path.dirname(resolved), path.resolve(path.dirname(resolved), '..')); +} + // https://vitejs.dev/config/ export default defineConfig({ base: basePath, @@ -234,6 +251,9 @@ export default defineConfig({ }, server: { port: 5180, + // Widen the fs allow-list only when an out-of-tree client override is set + // (see OBJECTSTACK_CLIENT_DIST above); otherwise keep Vite's defaults. + ...(clientFsAllow.length ? { fs: { allow: [path.resolve(__dirname, '../..'), ...clientFsAllow] } } : {}), proxy: { '/api': { target: process.env.DEV_PROXY_TARGET || 'http://localhost:3000', changeOrigin: true }, }, diff --git a/e2e/import-console/import-console-undo.spec.ts b/e2e/import-console/import-console-undo.spec.ts new file mode 100644 index 000000000..9d85742ff --- /dev/null +++ b/e2e/import-console/import-console-undo.spec.ts @@ -0,0 +1,114 @@ +import { test, expect } from '@playwright/test'; + +/** + * The real-product Import Wizard flow, driven exactly the way a user reaches it: + * + * log in → open an object's list view (ObjectView) → click the toolbar + * "Import" button → upload a small CSV → opt into a BACKGROUND import → run it + * → open History → Undo → confirm the created rows are gone. + * + * This is the end-to-end guard for the "background import" gap fix: the wizard + * only routed to an undoable async job for files over the async threshold (5000 + * rows), but the server only captures undo state at/under it — so an undoable + * job was unreachable through the UI. The `import-opt-background` toggle closes + * that gap; here we prove a 3-row import made through the *real console* is + * actually undoable, asserting record counts at the backend on both sides. + * + * Setup + skip behaviour: see playwright.import-console.config.ts. Gated on + * IMPORT_CONSOLE_LIVE=1 (the flow needs an import-job-capable client wired into + * the console) and additionally skips when the backend exposes no import-job + * route — so an unconfigured run reports skipped, not failed. + */ +const API = process.env.LIVE_API_URL || 'http://localhost:3000'; +const APP_NAME = process.env.LIVE_IMPORT_APP || 'crm_app'; +const OBJECT = process.env.LIVE_IMPORT_OBJECT || 'crm_lead'; + +test.describe('Import Wizard — real console: background import + undo', () => { + test('a small background import made through the console is undoable', async ({ page }) => { + test.skip( + process.env.IMPORT_CONSOLE_LIVE !== '1', + 'set IMPORT_CONSOLE_LIVE=1 (and wire an import-job-capable client into the console) to run this real-console flow', + ); + + // The whole flow depends on the backend having the async import-job routes. + let jobsSupported = false; + try { + const r = await page.request.get(`${API}/api/v1/data/import/jobs`); + jobsSupported = r.status() !== 404; + } catch { + jobsSupported = false; + } + test.skip(!jobsSupported, `backend at ${API} has no import-job route (/api/v1/data/import/jobs)`); + + // Count rows straight from the backend (cookie carried by the auth context). + const countRecords = async (): Promise => { + const res = await page.request.get(`${API}/api/v1/data/${OBJECT}`); + const body = await res.json(); + return (body.records ?? []).length; + }; + + // 1) Land on the REAL object list view and find the REAL toolbar button. + await page.goto(`/apps/${APP_NAME}/${OBJECT}`); + const importBtn = page.getByTestId('object-view-import-button'); + await expect(importBtn).toBeVisible({ timeout: 20_000 }); + + const baseline = await countRecords(); + + // 2) Open the wizard and upload a 3-row CSV (well under the async threshold). + await importBtn.click(); + const stamp = Date.now(); + const csv = [ + 'name,email,status', + `RC One,rc.one.${stamp}@example.test,new`, + `RC Two,rc.two.${stamp}@example.test,new`, + `RC Three,rc.three.${stamp}@example.test,new`, + '', + ].join('\n'); + await page.locator('input[type=file]').setInputFiles({ + name: 'rc-import.csv', + mimeType: 'text/csv', + buffer: Buffer.from(csv, 'utf8'), + }); + + // 3) Mapping → Preview (exact-name headers auto-map name/email/status). + await page.getByTestId('import-next-btn').click(); + + // 4) The background-import toggle must be offered for a sub-threshold file — + // this is the gap fix — and we opt in. + const backgroundOpt = page.getByTestId('import-opt-background'); + await expect(backgroundOpt).toBeVisible(); + await backgroundOpt.getByRole('checkbox').click(); + + // 5) Run it — the toggle routes it through the async job path. + const jobCreate = page.waitForResponse( + (r) => /\/import\/jobs$/.test(r.url()) && r.request().method() === 'POST' && r.status() === 201, + ); + await page.getByTestId('import-run-btn').click(); + await jobCreate; + + // Rows land at the backend. + await expect.poll(countRecords, { timeout: 20_000 }).toBe(baseline + 3); + + // Identify the fresh, undoable job created by this run. + const jobsBody = await (await page.request.get(`${API}/api/v1/data/import/jobs`)).json(); + const jobs: Array<{ jobId: string; undoable: boolean; revertedAt: string | null; createdAt: string }> = + jobsBody.jobs ?? jobsBody.records ?? jobsBody; + const mine = jobs + .filter((j) => j.undoable && !j.revertedAt) + .sort((a, b) => (a.createdAt < b.createdAt ? 1 : -1))[0]; + expect(mine, 'a fresh undoable job should exist').toBeTruthy(); + + // 6) Undo through the History UI. Reload for a clean wizard rather than + // dismissing the result screen (whose Close button re-renders as the job + // settles, making the click flaky). + page.on('dialog', (d) => d.accept()); // accept the confirm() prompt + await page.reload(); + await page.getByTestId('object-view-import-button').click(); + await page.getByTestId('import-history-toggle').click(); + await page.getByTestId(`import-history-undo-${mine.jobId}`).click(); + + // 7) The created rows are deleted and the job row flips to reverted. + await expect.poll(countRecords, { timeout: 20_000 }).toBe(baseline); + await expect(page.getByTestId(`import-history-reverted-${mine.jobId}`)).toBeVisible(); + }); +}); diff --git a/e2e/import-harness/import-undo.spec.ts b/e2e/import-harness/import-undo.spec.ts new file mode 100644 index 000000000..963072f5b --- /dev/null +++ b/e2e/import-harness/import-undo.spec.ts @@ -0,0 +1,103 @@ +import { test, expect } from '@playwright/test'; + +/** + * Full-stack Import Wizard flow against a real backend: + * upload a small CSV → opt into a BACKGROUND import → run it → open History → + * Undo → assert the created rows are gone. + * + * This is the regression guard for the "background import" gap: the wizard used + * to run a job only for files over the async threshold (5000 rows), but the + * server only captures undo state for files at or under it — so an undoable job + * was unreachable through the UI. The `import-opt-background` toggle closes that + * gap; this test proves a 3-row import made through the UI is actually undoable. + * + * Prereqs and skip behaviour: see playwright.import-harness.config.ts. The test + * skips (does not fail) when the harness origin isn't serving `/live.html`, so + * it's CI-safe. + */ +const HARNESS_PATH = '/live.html'; +const OBJECT = process.env.IMPORT_HARNESS_OBJECT || 'crm_lead'; + +test.describe('Import Wizard — background import + undo (live UI + backend)', () => { + test('a small background import creates an undoable job; Undo deletes the rows', async ({ page, baseURL }) => { + // Reachability guard: skip cleanly when the machine-specific harness is down. + let reachable = false; + try { + const res = await page.request.get(`${baseURL}${HARNESS_PATH}`); + reachable = res.ok(); + } catch { + reachable = false; + } + test.skip(!reachable, `import harness not reachable at ${baseURL}${HARNESS_PATH}`); + + // Backend record count via the same proxied origin the harness uses. + const countRecords = async (): Promise => { + const res = await page.request.get(`/api/v1/data/${OBJECT}`); + const body = await res.json(); + return (body.records ?? []).length; + }; + + await page.goto(HARNESS_PATH); + await expect(page.getByText('connected & authenticated')).toBeVisible({ timeout: 15_000 }); + + const baseline = await countRecords(); + + // 1) Open the wizard and upload a 3-row CSV (well under the async threshold). + await page.getByRole('button', { name: 'Open import' }).click(); + const stamp = Date.now(); + const csv = [ + 'first_name,last_name,email', + `Bg,One,bg.one.${stamp}@example.test`, + `Bg,Two,bg.two.${stamp}@example.test`, + `Bg,Three,bg.three.${stamp}@example.test`, + '', + ].join('\n'); + await page.locator('input[type=file]').setInputFiles({ + name: 'bg-import.csv', + mimeType: 'text/csv', + buffer: Buffer.from(csv, 'utf8'), + }); + + // 2) Mapping → Preview. + await page.getByRole('button', { name: /^Next/ }).click(); + + // 3) The background-import toggle must be offered for a sub-threshold file + // when the data source supports jobs — this is the gap fix. + const backgroundOpt = page.getByTestId('import-opt-background'); + await expect(backgroundOpt).toBeVisible(); + await backgroundOpt.getByRole('checkbox').click(); + + // 4) Run it — routes through the async job path because of the toggle. + const jobCreate = page.waitForResponse( + (r) => /\/import\/jobs$/.test(r.url()) && r.request().method() === 'POST' && r.status() === 201, + ); + await page.getByRole('button', { name: /Import\s+3\s+Rows/i }).click(); + await jobCreate; + + // Rows land at the backend. + await expect.poll(countRecords, { timeout: 20_000 }).toBe(baseline + 3); + + // Identify the fresh, undoable job created by this run. + const jobsBody = await (await page.request.get('/api/v1/data/import/jobs')).json(); + const jobs: Array<{ jobId: string; undoable: boolean; revertedAt: string | null; createdAt: string }> = + jobsBody.jobs ?? jobsBody.records ?? jobsBody; + const mine = jobs + .filter((j) => j.undoable && !j.revertedAt) + .sort((a, b) => (a.createdAt < b.createdAt ? 1 : -1))[0]; + expect(mine, 'a fresh undoable job should exist').toBeTruthy(); + + // 5) Undo through the History UI. Reload for a clean wizard rather than + // dismissing the result screen (whose Close button re-renders as the job + // settles, making the click flaky). + page.on('dialog', (d) => d.accept()); // accept the confirm() prompt + await page.reload(); + await expect(page.getByText('connected & authenticated')).toBeVisible({ timeout: 15_000 }); + await page.getByRole('button', { name: 'Open import' }).click(); + await page.getByTestId('import-history-toggle').click(); + await page.getByTestId(`import-history-undo-${mine.jobId}`).click(); + + // 6) The created rows are deleted and the row flips to reverted. + await expect.poll(countRecords, { timeout: 20_000 }).toBe(baseline); + await expect(page.getByTestId(`import-history-reverted-${mine.jobId}`)).toBeVisible(); + }); +}); diff --git a/package.json b/package.json index fe02d9740..20e3f726f 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,9 @@ "changeset:publish": "changeset publish", "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui", - "test:e2e:live": "playwright test --config=playwright.live.config.ts" + "test:e2e:live": "playwright test --config=playwright.live.config.ts", + "test:e2e:import-harness": "playwright test --config=playwright.import-harness.config.ts", + "test:e2e:import-console": "playwright test --config=playwright.import-console.config.ts" }, "devDependencies": { "@changesets/cli": "^2.31.0", diff --git a/packages/app-shell/src/views/ObjectView.tsx b/packages/app-shell/src/views/ObjectView.tsx index efa3ad0ca..628995bc0 100644 --- a/packages/app-shell/src/views/ObjectView.tsx +++ b/packages/app-shell/src/views/ObjectView.tsx @@ -1616,12 +1616,25 @@ function ObjectViewInner({ dataSource, objects, onEdit, externalRefreshKey }: an onOpenChange={setShowImport} objectName={objectDef.name} objectLabel={objectLabel(objectDef)} - fields={Object.entries(objectDef.fields || {}).map(([name, def]: [string, any]) => ({ - name, - label: def?.label || name, - type: def?.type || 'text', - required: !!def?.required, - }))} + fields={Object.entries(objectDef.fields || {}) + // Only writable fields are importable targets. Computed + // types (formula/summary/autonumber) and fields flagged + // readonly / write:false are server-rejected, so we omit + // them from the mapping step rather than let a user map to + // a column the import will silently drop. + .filter(([, def]: [string, any]) => + !['formula', 'summary', 'autonumber'].includes(def?.type) && + !def?.readonly && + def?.permissions?.write !== false, + ) + .map(([name, def]: [string, any]) => ({ + name, + label: def?.label || name, + type: def?.type || 'text', + required: !!def?.required, + // Enum options seed the downloadable template's example row. + ...(def?.options ? { options: def.options } : {}), + }))} dataSource={dataSource} onComplete={(result) => { setRefreshKey(k => k + 1); diff --git a/packages/data-objectstack/src/importJob.test.ts b/packages/data-objectstack/src/importJob.test.ts new file mode 100644 index 000000000..cd28bccba --- /dev/null +++ b/packages/data-objectstack/src/importJob.test.ts @@ -0,0 +1,71 @@ +/** + * ObjectUI + * Copyright (c) 2024-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { ObjectStackAdapter } from './index'; + +/** + * Adapter over a mock client `data` namespace. The async import-job methods are + * thin pass-throughs to the client SDK, so we assert delegation + argument + * shaping, plus graceful degradation when the client lacks the job API. + */ +function makeDS(stub: Record) { + const ds: any = new ObjectStackAdapter({ + baseUrl: 'http://test.local', + fetch: vi.fn(async () => new Response(JSON.stringify({ success: true, data: { capabilities: {}, routes: {} } }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + })), + }); + ds.connected = true; + ds.connectionState = 'connected'; + ds.client = { data: stub }; + return ds; +} + +describe('ObjectStackAdapter async import jobs', () => { + it('createImportJob delegates to client.data.createImportJob', async () => { + const createImportJob = vi.fn().mockResolvedValue({ jobId: 'imp_1', object: 'task', status: 'pending', total: 2 }); + const ds = makeDS({ createImportJob }); + const req = { format: 'json' as const, rows: [{ a: 1 }, { a: 2 }], writeMode: 'upsert' as const, matchFields: ['a'] }; + const res = await ds.createImportJob('task', req); + expect(createImportJob).toHaveBeenCalledTimes(1); + expect(createImportJob.mock.calls[0]).toEqual(['task', req]); + expect(res).toMatchObject({ jobId: 'imp_1', status: 'pending', total: 2 }); + }); + + it('getImportJobProgress / getImportJobResults / cancelImportJob delegate by jobId', async () => { + const getImportJobProgress = vi.fn().mockResolvedValue({ jobId: 'imp_1', status: 'running', percentComplete: 50 }); + const getImportJobResults = vi.fn().mockResolvedValue({ jobId: 'imp_1', status: 'succeeded', results: [], resultsTruncated: false }); + const cancelImportJob = vi.fn().mockResolvedValue({ success: true }); + const ds = makeDS({ createImportJob: vi.fn(), getImportJobProgress, getImportJobResults, cancelImportJob }); + + expect((await ds.getImportJobProgress('imp_1')).percentComplete).toBe(50); + expect(getImportJobProgress).toHaveBeenCalledWith('imp_1'); + + expect((await ds.getImportJobResults('imp_1')).resultsTruncated).toBe(false); + expect(getImportJobResults).toHaveBeenCalledWith('imp_1'); + + await ds.cancelImportJob('imp_1'); + expect(cancelImportJob).toHaveBeenCalledWith('imp_1'); + }); + + it('listImportJobs forwards filters and returns the jobs array', async () => { + const listImportJobs = vi.fn().mockResolvedValue([{ jobId: 'imp_1', object: 'task', status: 'succeeded' }]); + const ds = makeDS({ createImportJob: vi.fn(), listImportJobs }); + const jobs = await ds.listImportJobs({ object: 'task', status: 'succeeded', limit: 10 }); + expect(listImportJobs).toHaveBeenCalledWith({ object: 'task', status: 'succeeded', limit: 10 }); + expect(jobs).toHaveLength(1); + }); + + it('throws UNSUPPORTED_OPERATION when the client lacks the job API', async () => { + const ds = makeDS({ import: vi.fn() }); // sync import only, no createImportJob + await expect(ds.createImportJob('task', { format: 'json', rows: [] })).rejects.toMatchObject({ code: 'UNSUPPORTED_OPERATION' }); + await expect(ds.getImportJobProgress('imp_1')).rejects.toMatchObject({ code: 'UNSUPPORTED_OPERATION' }); + }); +}); diff --git a/packages/data-objectstack/src/index.ts b/packages/data-objectstack/src/index.ts index d2d32a161..179d9423f 100644 --- a/packages/data-objectstack/src/index.ts +++ b/packages/data-objectstack/src/index.ts @@ -7,7 +7,21 @@ */ import { ObjectStackClient, type QueryOptions as ObjectStackQueryOptions } from '@objectstack/client'; -import type { DataSource, QueryParams, QueryResult, FileUploadResult, ExportDownloadRequest } from '@object-ui/types'; +import type { + DataSource, + QueryParams, + QueryResult, + FileUploadResult, + ExportDownloadRequest, + ImportRequestOptions, + ImportRecordsResult, + CreateImportJobResult, + ImportJobProgressInfo, + ImportJobResultsInfo, + ImportJobSummaryInfo, + ImportJobUndoResult, + ListImportJobsOptions, +} from '@object-ui/types'; import { convertFiltersToAST } from '@object-ui/core'; import { MetadataCache } from './cache/MetadataCache'; import { @@ -1047,6 +1061,183 @@ export class ObjectStackAdapter implements DataSource { } } + /** + * Bulk-import raw spreadsheet rows in a single server round-trip via + * `POST /api/v1/data/:object/import`. The server performs all value coercion + * (booleans, numbers, dates→ISO, select label→code, lookup name→id) from the + * object's field metadata, so this method forwards the request verbatim and + * returns the aggregate + per-row result untouched. + * + * Requires `@objectstack/client` with `data.import` (server `/import` route). + * Callers should feature-detect (`typeof dataSource.importRecords`) and fall + * back to a per-row `create` loop when unavailable. + */ + async importRecords( + resource: string, + request: ImportRequestOptions, + ): Promise { + await this.connect(); + const importFn = (this.client.data as { import?: unknown }).import; + if (typeof importFn !== 'function') { + throw new ObjectStackError( + 'The connected @objectstack/client does not support data.import(). ' + + 'Upgrade the client, or import via a per-row create fallback.', + 'UNSUPPORTED_OPERATION', + 400, + ); + } + try { + const result = await (importFn as ( + object: string, + req: ImportRequestOptions, + ) => Promise).call(this.client.data, resource, request); + return result; + } catch (err) { + throw normaliseClientError(err); + } + } + + /** + * Feature-detect the async import-job API on the connected client. Older + * clients/servers lack these routes; callers fall back to {@link importRecords}. + */ + private importJobApi(): { + createImportJob: (object: string, req: ImportRequestOptions) => Promise; + getImportJobProgress: (jobId: string) => Promise; + getImportJobResults: (jobId: string) => Promise; + listImportJobs: (query: ListImportJobsOptions) => Promise; + cancelImportJob: (jobId: string) => Promise<{ success: boolean }>; + undoImportJob: (jobId: string) => Promise; + } | undefined { + const d = this.client.data as Record; + if (typeof d.createImportJob !== 'function') return undefined; + return d as any; + } + + /** + * Start an asynchronous import job — the large-file counterpart to + * {@link importRecords}. Posts the whole payload once; the server processes + * rows in the background. Requires an `@objectstack/client` new enough to + * expose `data.createImportJob` (server `/import/jobs` route). Callers should + * feature-detect (`typeof dataSource.createImportJob`) and fall back to the + * synchronous path when unavailable. + */ + async createImportJob( + resource: string, + request: ImportRequestOptions, + ): Promise { + await this.connect(); + const api = this.importJobApi(); + if (!api) { + throw new ObjectStackError( + 'The connected @objectstack/client does not support async import jobs (data.createImportJob). ' + + 'Upgrade the client, or use the synchronous importRecords() path.', + 'UNSUPPORTED_OPERATION', + 400, + ); + } + try { + return await api.createImportJob.call(this.client.data, resource, request); + } catch (err) { + throw normaliseClientError(err); + } + } + + /** Poll an import job's progress. Requires {@link createImportJob} support. */ + async getImportJobProgress(jobId: string): Promise { + await this.connect(); + const api = this.importJobApi(); + if (!api) { + throw new ObjectStackError( + 'The connected @objectstack/client does not support async import jobs.', + 'UNSUPPORTED_OPERATION', + 400, + ); + } + try { + return await api.getImportJobProgress.call(this.client.data, jobId); + } catch (err) { + throw normaliseClientError(err); + } + } + + /** Fetch an import job's capped per-row results. */ + async getImportJobResults(jobId: string): Promise { + await this.connect(); + const api = this.importJobApi(); + if (!api) { + throw new ObjectStackError( + 'The connected @objectstack/client does not support async import jobs.', + 'UNSUPPORTED_OPERATION', + 400, + ); + } + try { + return await api.getImportJobResults.call(this.client.data, jobId); + } catch (err) { + throw normaliseClientError(err); + } + } + + /** List recent import jobs (history), newest first. */ + async listImportJobs(options: ListImportJobsOptions = {}): Promise { + await this.connect(); + const api = this.importJobApi(); + if (!api) { + throw new ObjectStackError( + 'The connected @objectstack/client does not support async import jobs.', + 'UNSUPPORTED_OPERATION', + 400, + ); + } + try { + return await api.listImportJobs.call(this.client.data, options); + } catch (err) { + throw normaliseClientError(err); + } + } + + /** Cancel a pending/running import job (cooperative). */ + async cancelImportJob(jobId: string): Promise { + await this.connect(); + const api = this.importJobApi(); + if (!api) { + throw new ObjectStackError( + 'The connected @objectstack/client does not support async import jobs.', + 'UNSUPPORTED_OPERATION', + 400, + ); + } + try { + await api.cancelImportJob.call(this.client.data, jobId); + } catch (err) { + throw normaliseClientError(err); + } + } + + /** + * Logically roll back a finished import job — delete the records it created + * and restore the records it updated to their pre-import values. Requires an + * `@objectstack/client` new enough to expose `data.undoImportJob`, and a job + * the server captured an undo log for (see {@link ImportJobProgressInfo.undoable}). + */ + async undoImportJob(jobId: string): Promise { + await this.connect(); + const api = this.importJobApi(); + if (!api || typeof (api as { undoImportJob?: unknown }).undoImportJob !== 'function') { + throw new ObjectStackError( + 'The connected @objectstack/client does not support undoing import jobs (data.undoImportJob).', + 'UNSUPPORTED_OPERATION', + 400, + ); + } + try { + return await api.undoImportJob.call(this.client.data, jobId); + } catch (err) { + throw normaliseClientError(err); + } + } + /** * Normalize the result from data.find() or data.query() into a consistent QueryResult. */ diff --git a/packages/i18n/src/locales/zh.ts b/packages/i18n/src/locales/zh.ts index 2ff4d4db9..444f66cb9 100644 --- a/packages/i18n/src/locales/zh.ts +++ b/packages/i18n/src/locales/zh.ts @@ -231,6 +231,79 @@ const zh = { requiredMark: '*', required: '必填', invalidType: '{{type}} 格式无效', + // 自动匹配（Airtable 风格） + autoMatched: '自动匹配', + autoMatchedSummary: '已自动匹配 {{count}} 列 — 请在下方检查并调整。', + confidence: { + high: '高置信度', + medium: '中置信度', + low: '低置信度', + }, + // 下载模板 + downloadTemplate: '下载模板', + downloadTemplateHint: '获取带正确列的 CSV（必填字段标 *）。', + // 校验（服务端 dryRun 预检） + validate: '校验数据', + validateHint: '导入前在服务端逐行校验。', + validating: '校验中…', + validatePassed: '全部 {{ok}} 行均有效。', + validateFailed: '{{ok}} 条有效，{{errors}} 条有错误。', + errorCount: '{{count}} 个错误', + errorRowPrefix: '第 {{row}} 行：', + // 导入选项 / 写入模式 + options: '导入选项', + writeMode: '当某行匹配到已有记录时', + writeModeOpt: { + insert: '始终新建', + update: '更新已有（无匹配则跳过）', + upsert: '匹配则更新，否则新建', + }, + matchFields: '匹配字段', + matchFieldsPlaceholder: '选择匹配字段…', + matchFieldsHint: '按这些字段将行匹配到已有记录。', + needMatchFields: '请至少选择一个匹配字段。', + optRunAutomations: '运行自动化与触发器', + optCreateOptions: '保留未知选项值', + optSkipBlankKey: '跳过匹配值为空的行', + optBackground: '后台导入', + optBackgroundHint: '(作为可撤销任务运行)', + // 结果 + createdCount: '新建 {{count}} 条', + updatedCount: '更新 {{count}} 条', + resultsTruncated: '显示前 {{count}} 行结果（共 {{total}} 行）。', + downloadFailed: '下载失败行', + cancelImport: '取消导入', + importCancelled: '导入已取消', + // 大文件 / 异步导入 + largeSampleNotice: '正在预览前 {{shown}}/{{total}} 行。', + asyncLargeHint: '文件较大，将在后台导入。', + asyncQueued: '已排队 — 正在准备导入…', + asyncProcessing: '正在导入第 {{processed}}/{{total}} 行… {{progress}}%', + // 导入历史 + history: '历史', + historyBack: '返回导入', + historyDescription: '此对象的近期导入记录。', + historyHint: '后台导入任务，最新在前。', + historyRefresh: '刷新', + historyLoading: '加载中…', + historyEmpty: '暂无导入记录。', + historyUnsupported: '此数据源不支持导入历史。', + historyColStatus: '状态', + historyColRows: '行数', + historyColResult: '结果', + historyColTime: '时间', + jobStatus: { + pending: '等待中', + running: '进行中', + succeeded: '成功', + failed: '失败', + cancelled: '已取消', + }, + // 撤销 / 逻辑回滚 + undoImport: '撤销导入', + undoing: '撤销中…', + reverted: '已撤销', + undoConfirm: '撤销此次导入？它新建的记录将被删除，它更新的记录将恢复为之前的值。', }, }, calendar: { diff --git a/packages/plugin-grid/src/ImportWizard.tsx b/packages/plugin-grid/src/ImportWizard.tsx index 71b8d6eb6..abdd98c8b 100644 --- a/packages/plugin-grid/src/ImportWizard.tsx +++ b/packages/plugin-grid/src/ImportWizard.tsx @@ -5,16 +5,28 @@ import React, { useState, useCallback, useMemo, useEffect } from 'react'; import { - cn, Button, Badge, Progress, Input, + cn, Button, Badge, Progress, Input, Checkbox, Label, Dialog, DialogContent, DialogHeader, DialogFooter, DialogTitle, DialogDescription, Select, SelectContent, SelectItem, SelectTrigger, SelectValue, Table, TableBody, TableCell, TableHead, TableHeader, TableRow, } from '@object-ui/components'; -import { Upload, FileSpreadsheet, CheckCircle2, AlertCircle, X, ArrowRight, ArrowLeft, Save, Trash2, ClipboardPaste } from 'lucide-react'; +import { Upload, FileSpreadsheet, CheckCircle2, AlertCircle, X, ArrowRight, ArrowLeft, Save, Trash2, ClipboardPaste, Download, Undo2 } from 'lucide-react'; import { useObjectTranslation } from '@object-ui/react'; +import type { + DataSource, + ImportRequestOptions, + ImportRecordsResult, + ImportWriteMode, + CreateImportJobResult, + ImportJobProgressInfo, + ImportJobResultsInfo, + ImportJobStatus, + ImportJobSummaryInfo, +} from '@object-ui/types'; import { parseSpreadsheetFile, parseClipboardTable, inferColumnType, isTypeCompatible, - ImportParseError, type InferredType, + suggestColumnMappings, ImportParseError, + type InferredType, type ColumnSuggestion, type MappingConfidence, } from './importParsers'; /** Default English fallback strings used when no I18nProvider is mounted @@ -29,6 +41,8 @@ const IMPORT_DEFAULT_TRANSLATIONS: Record = { 'grid.import.previewDescription': 'Review data before importing.', 'grid.import.dragDrop': 'Drag & drop a CSV or Excel file here, or click to browse', 'grid.import.browseFiles': 'Browse Files', + 'grid.import.downloadTemplate': 'Download template', + 'grid.import.downloadTemplateHint': 'Get a CSV with the right columns (required fields marked *).', 'grid.import.parsing': 'Parsing…', 'grid.import.pasteHint': 'or paste (Ctrl/⌘+V) rows copied from Excel or Google Sheets', 'grid.import.legacyXls': "Legacy .xls files aren't supported — please re-save as .xlsx.", @@ -46,6 +60,11 @@ const IMPORT_DEFAULT_TRANSLATIONS: Record = { 'grid.import.csvColumn': 'Column', 'grid.import.mapsTo': 'Maps To', 'grid.import.typeMismatch': 'Looks like {{type}}', + 'grid.import.autoMatched': 'Auto-matched', + 'grid.import.autoMatchedSummary': 'Auto-matched {{count}} column(s) — review and adjust below.', + 'grid.import.confidence.high': 'High confidence', + 'grid.import.confidence.medium': 'Medium confidence', + 'grid.import.confidence.low': 'Low confidence', 'grid.import.type.number': 'Number', 'grid.import.type.boolean': 'Boolean', 'grid.import.type.date': 'Date', @@ -61,10 +80,67 @@ const IMPORT_DEFAULT_TRANSLATIONS: Record = { 'grid.import.clickToFix': '— click a highlighted cell to fix it inline.', 'grid.import.showingRows': 'Showing {{shown}} of {{total}} rows', 'grid.import.importing': 'Importing… {{progress}}%', + // Async (large-file) import — job queued + processed server-side. + 'grid.import.asyncQueued': 'Queued — preparing to import…', + 'grid.import.asyncProcessing': 'Importing {{processed}} of {{total}} rows… {{progress}}%', + 'grid.import.asyncLargeHint': 'This file is large, so it will be imported in the background.', + 'grid.import.largeSampleNotice': 'Previewing the first {{shown}} of {{total}} rows.', + 'grid.import.cancelImport': 'Cancel import', + 'grid.import.importCancelled': 'Import cancelled', + 'grid.import.resultsTruncated': 'Showing the first {{count}} row results (of {{total}}).', 'grid.import.importComplete': 'Import Complete', 'grid.import.imported': '{{count}} imported', + 'grid.import.createdCount': '{{count}} created', + 'grid.import.updatedCount': '{{count}} updated', 'grid.import.skippedCount': '{{count}} skipped', 'grid.import.moreErrors': '…and {{count}} more errors', + 'grid.import.downloadFailed': 'Download failed rows', + // Write-mode / options (preview step) + 'grid.import.options': 'Import options', + 'grid.import.writeMode': 'When a row matches an existing record', + 'grid.import.writeModeOpt.insert': 'Always create new', + 'grid.import.writeModeOpt.update': 'Update existing (skip if no match)', + 'grid.import.writeModeOpt.upsert': 'Update if matched, else create', + 'grid.import.matchFields': 'Match on', + 'grid.import.matchFieldsPlaceholder': 'Choose match field(s)…', + 'grid.import.matchFieldsHint': 'Rows are matched to existing records by these field(s).', + 'grid.import.needMatchFields': 'Select at least one field to match on.', + 'grid.import.optCreateOptions': 'Keep unknown option values', + 'grid.import.optRunAutomations': 'Run automations & triggers', + 'grid.import.optSkipBlankKey': 'Skip rows with a blank match value', + 'grid.import.optBackground': 'Import in the background', + 'grid.import.optBackgroundHint': '(runs as an undoable job)', + // Server dry-run pre-check (small files, preview step) + 'grid.import.validate': 'Validate data', + 'grid.import.validating': 'Validating…', + 'grid.import.validateHint': 'Check every row against the server before importing.', + 'grid.import.validatePassed': 'All {{ok}} rows are valid.', + 'grid.import.validateFailed': '{{ok}} valid, {{errors}} with errors.', + 'grid.import.errorRowPrefix': 'Row {{row}}: ', + // Import-job history + 'grid.import.history': 'History', + 'grid.import.historyBack': 'Back to import', + 'grid.import.historyDescription': 'Recent imports for this object.', + 'grid.import.historyHint': 'Background import jobs, newest first.', + 'grid.import.historyRefresh': 'Refresh', + 'grid.import.historyLoading': 'Loading…', + 'grid.import.historyEmpty': 'No imports yet.', + 'grid.import.historyUnsupported': 'Import history isn’t available for this data source.', + 'grid.import.historyColStatus': 'Status', + 'grid.import.historyColRows': 'Rows', + 'grid.import.historyColResult': 'Result', + 'grid.import.historyColTime': 'When', + 'grid.import.errorCount': '{{count}} errors', + // Undo / logical rollback + 'grid.import.undoImport': 'Undo import', + 'grid.import.undoing': 'Undoing…', + 'grid.import.undoConfirm': 'Undo this import? Records it created will be deleted and records it updated will be restored to their previous values.', + 'grid.import.reverted': 'Undone', + 'grid.import.jobStatus.pending': 'Pending', + 'grid.import.jobStatus.running': 'Running', + 'grid.import.jobStatus.succeeded': 'Succeeded', + 'grid.import.jobStatus.failed': 'Failed', + 'grid.import.jobStatus.cancelled': 'Cancelled', 'grid.import.cancel': 'Cancel', 'grid.import.back': 'Back', 'grid.import.next': 'Next', @@ -114,6 +190,14 @@ export const __testables = { get loadTemplates() { return loadTemplates; }, get saveTemplates() { return saveTemplates; }, get autoMapColumns() { return autoMapColumns; }, + get isUnsupportedImport() { return isUnsupportedImport; }, + get isUnsupportedImportJob() { return isUnsupportedImportJob; }, + get jobResultToImportResult() { return jobResultToImportResult; }, + get buildFailedRowsCsv() { return buildFailedRowsCsv; }, + get buildImportTemplateCsv() { return buildImportTemplateCsv; }, + get assembleImportRequest() { return assembleImportRequest; }, + get isImportJobActive() { return isImportJobActive; }, + get isImportJobUndoable() { return isImportJobUndoable; }, }; /** A reusable column-mapping template, persisted across sessions. Keys are @@ -137,7 +221,15 @@ export interface ImportTemplateStorage { export interface ImportWizardProps { objectName: string; objectLabel?: string; - fields: Array<{ name: string; label: string; type: string; required?: boolean }>; + fields: Array<{ + name: string; + label: string; + type: string; + required?: boolean; + /** Allowed values for select/enum fields — used to seed the downloadable + * template's example row. Accepts option objects or bare strings. */ + options?: Array<{ label?: string; value?: string | number } | string>; + }>; dataSource: any; onComplete?: (result: ImportResult) => void; onCancel?: () => void; @@ -158,6 +250,16 @@ export interface ImportResult { importedRows: number; skippedRows: number; errors: Array<{ row: number; field: string; message: string }>; + /** Rows that created a new record (server-side import). */ + createdRows?: number; + /** Rows that updated an existing record (server-side import). */ + updatedRows?: number; + /** The raw per-row server result, when the server `/import` path was used. */ + serverResult?: ImportRecordsResult; + /** True when an async job's per-row `errors` were capped by the server. */ + resultsTruncated?: boolean; + /** True when the user cancelled an in-flight async import job. */ + cancelled?: boolean; } type WizardStep = 'upload' | 'mapping' | 'preview'; @@ -165,24 +267,48 @@ type WizardStep = 'upload' | 'mapping' | 'preview'; /** Maximum number of rows to show in the preview step */ const PREVIEW_ROW_COUNT = 10; +/** + * Row count above which the wizard prefers an asynchronous import job (when the + * data source supports it) instead of the synchronous single-call import. Kept + * in step with the server's synchronous `/import` ceiling (`maxRows: 5000`), so + * files the sync route would reject with 413 are routed to a background job. + */ +const ASYNC_IMPORT_THRESHOLD = 5000; + +/** How often (ms) to poll an in-flight import job for progress. */ +const IMPORT_JOB_POLL_INTERVAL = 800; + +/** Text colour for the auto-match confidence hint, keyed by confidence bucket. */ +const CONFIDENCE_CLASS: Record = { + high: 'text-emerald-600', + medium: 'text-sky-600', + low: 'text-muted-foreground', +}; + +/** Boolean tokens the server's import coercion accepts (import-coerce.ts). + * Kept in sync so the preview step doesn't flag a cell the server would take + * (e.g. Chinese 是/否, on/off, ✓/×). Compared case-insensitively. */ +const BOOLEAN_IMPORT_TOKENS = new Set([ + 'true', 't', 'yes', 'y', '1', 'on', '是', '对', '✓', '√', + 'false', 'f', 'no', 'n', '0', 'off', '否', '错', '✗', '×', +]); + function validateValue(value: string, type: string): boolean { if (!value) return true; switch (type) { case 'number': case 'currency': case 'percent': return !isNaN(Number(value)); - case 'boolean': return ['true', 'false', '1', '0', 'yes', 'no'].includes(value.toLowerCase()); + case 'boolean': return BOOLEAN_IMPORT_TOKENS.has(value.trim().toLowerCase()); case 'date': case 'datetime': return !isNaN(Date.parse(value)); default: return true; } } -const normalizeKey = (s: string) => s.toLowerCase().replace(/[_\s-]/g, ''); - /** - * Auto-map source columns to object fields. Pass 1 matches by normalized - * name/label (exact). Pass 2 fills still-unmapped columns by fuzzy name - * containment *gated on type compatibility* with the column's inferred type — - * the type gate keeps the fuzzy pass from confidently mis-mapping. `rows` is - * optional; without it only the exact pass runs. + * Auto-map source columns to object fields, Airtable-style. Delegates to + * {@link suggestColumnMappings} (name/label similarity + bilingual synonyms + + * token overlap + content-inferred type gating, assigned globally by + * confidence) and keeps only the confidently-matched columns. `rows` is + * optional; without it only name-based signals fire. */ function autoMapColumns( headers: string[], @@ -190,29 +316,8 @@ function autoMapColumns( rows?: string[][], ): Record { const mapping: Record = {}; - const used = new Set(); - // Pass 1 — exact normalized name/label match. - headers.forEach((header, idx) => { - const h = normalizeKey(header); - const match = fields.find((f) => normalizeKey(f.name) === h || normalizeKey(f.label) === h); - if (match && !used.has(match.name)) { mapping[idx] = match.name; used.add(match.name); } - }); - // Pass 2 — fuzzy containment, gated on inferred-type compatibility. - if (rows && rows.length) { - headers.forEach((header, idx) => { - if (mapping[idx]) return; - const h = normalizeKey(header); - if (h.length < 3) return; - const inferred = inferColumnType(rows.map((r) => r[idx])); - const match = fields.find((f) => { - if (used.has(f.name)) return false; - if (!isTypeCompatible(inferred, f.type)) return false; - const name = normalizeKey(f.name); - const label = normalizeKey(f.label); - return name.includes(h) || h.includes(name) || label.includes(h) || h.includes(label); - }); - if (match) { mapping[idx] = match.name; used.add(match.name); } - }); + for (const s of suggestColumnMappings(headers, fields, rows)) { + if (s.fieldName) mapping[s.columnIndex] = s.fieldName; } return mapping; } @@ -285,6 +390,169 @@ function validateRow(row: string[], mappedCols: MappedCol[], rowIndex: number) { return { record, errors }; } +/** Assemble the server `/import` request from mapping-applied raw rows plus the + * current write-mode + coercion options. Kept pure (no component state) so the + * real import and the dry-run pre-check send byte-identical payloads and it can + * be unit-tested. `matchFields` is only sent when the write-mode consults it. */ +function assembleImportRequest( + rows: Record[], + opts: { + writeMode: ImportWriteMode; + matchFields: string[]; + createMissingOptions: boolean; + runAutomations: boolean; + skipBlankMatchKey: boolean; + dryRun?: boolean; + }, +): ImportRequestOptions { + return { + format: 'json', + rows, + writeMode: opts.writeMode, + ...(opts.writeMode !== 'insert' ? { matchFields: opts.matchFields } : {}), + createMissingOptions: opts.createMissingOptions, + runAutomations: opts.runAutomations, + skipBlankMatchKey: opts.skipBlankMatchKey, + ...(opts.dryRun ? { dryRun: true } : {}), + }; +} + +/** True when the adapter/client can't speak the server `/import` route, so the + * wizard should transparently fall back to a per-row `create` loop. */ +function isUnsupportedImport(err: unknown): boolean { + const code = (err as { code?: unknown })?.code; + if (code === 'UNSUPPORTED_OPERATION') return true; + const msg = err instanceof Error ? err.message : ''; + return /does not support data\.import|importRecords is not a function|\.import is not a function/i.test(msg); +} + +/** True when the data source lacks the async import-job API (older + * adapter/client/server), so the wizard should fall back to the sync path. */ +function isUnsupportedImportJob(err: unknown): boolean { + const code = (err as { code?: unknown })?.code; + if (code === 'UNSUPPORTED_OPERATION') return true; + const msg = err instanceof Error ? err.message : ''; + return /does not support async import|createImportJob is not a function|import\/jobs|404/i.test(msg); +} + +/** Map an async import-job's final results payload onto the wizard's + * {@link ImportResult} shape — identical to the synchronous mapping so the + * completion screen renders the same regardless of which path ran. */ +function jobResultToImportResult(res: ImportJobResultsInfo): ImportResult { + return { + totalRows: res.total, + importedRows: res.created + res.updated, + skippedRows: res.skipped + res.errors, + createdRows: res.created, + updatedRows: res.updated, + errors: (res.results ?? []) + .filter((r) => !r.ok) + .map((r) => ({ row: r.row, field: r.field ?? '', message: r.error ?? r.code ?? 'Import failed' })), + resultsTruncated: res.resultsTruncated, + }; +} + +/** True while an import job is still in flight — it can be cancelled and the + * history list should keep polling it. Terminal states are the rest. */ +function isImportJobActive(status: ImportJobStatus): boolean { + return status === 'pending' || status === 'running'; +} + +/** Whether to show the "Undo import" button for a history row: the adapter must + * support undo, the job must be terminal, still undoable, and not already + * reverted. Mirrors the server's `importJobUndoable`. */ +function isImportJobUndoable(job: Pick, canUndo: boolean): boolean { + return canUndo && !!job.undoable && !job.revertedAt && !isImportJobActive(job.status); +} + +/** Build a CSV blob of failed rows for re-export: the original mapped columns + * plus an `_error` column, so a user can fix and re-import just the failures. */ +function buildFailedRowsCsv( + headers: string[], + rows: string[][], + mapping: Record, + errorsByRow: Map, +): string { + const cols = Object.keys(mapping).map(Number).sort((a, b) => a - b); + const esc = (v: string) => (/[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v); + const head = [...cols.map((c) => headers[c] ?? `col${c}`), '_error']; + const lines = [head.map(esc).join(',')]; + // errorsByRow is keyed by 1-based row number. + for (const [rowNum, message] of errorsByRow) { + const src = rows[rowNum - 1]; + if (!src) continue; + lines.push([...cols.map((c) => esc(src[c] ?? '')), esc(message)].join(',')); + } + return lines.join('\n'); +} + +/** Pick a representative allowed value from a select field's options, for the + * template example row. Prefers the stored value over the display label. */ +function firstOptionValue( + options: ImportWizardProps['fields'][number]['options'], +): string | undefined { + const first = options?.[0]; + if (first === undefined || first === null) return undefined; + if (typeof first === 'string') return first; + if (first.value !== undefined && first.value !== null) return String(first.value); + if (first.label) return first.label; + return undefined; +} + +/** A type-appropriate example cell for the downloadable import template. Kept + * format-oriented (dates, emails) rather than prose so it reads the same in + * any locale; text-ish fields are left blank so the row is obviously a sample. */ +function exampleForField(field: ImportWizardProps['fields'][number]): string { + switch (field.type) { + case 'number': + case 'currency': + case 'percent': + return '0'; + case 'date': + return '2024-01-31'; + case 'datetime': + return '2024-01-31 09:00'; + case 'time': + return '09:00'; + case 'boolean': + return 'true'; + case 'email': + return 'name@example.com'; + case 'url': + return 'https://example.com'; + case 'select': + case 'multiselect': + case 'lookup': + case 'reference': + return firstOptionValue(field.options) ?? ''; + default: + return ''; + } +} + +/** Build a downloadable CSV import template for the given fields: a header row + * of field labels (required fields marked with `*`, which re-import tolerates) + * plus a single example row. Not persisted — a convenience starting point. */ +function buildImportTemplateCsv(fields: ImportWizardProps['fields']): string { + const esc = (v: string) => (/[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v); + const header = fields.map((f) => `${f.label}${f.required ? ' *' : ''}`); + const example = fields.map((f) => exampleForField(f)); + return [header.map(esc).join(','), example.map(esc).join(',')].join('\n'); +} + +/** Trigger a client-side text file download (prepends a UTF-8 BOM so Excel + * reads non-ASCII correctly). No-op in non-DOM environments. */ +function downloadTextFile(filename: string, text: string, mime = 'text/csv;charset=utf-8'): void { + if (typeof document === 'undefined' || typeof URL?.createObjectURL !== 'function') return; + const blob = new Blob([`${text}`], { type: mime }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = filename; + a.click(); + URL.revokeObjectURL(url); +} + /** Map a thrown import-parse error code to a translated, user-facing message. */ function parseErrorMessage(err: unknown, t: (k: string, v?: Record) => string): string { const code = err instanceof Error ? err.message : ''; @@ -294,7 +562,11 @@ function parseErrorMessage(err: unknown, t: (k: string, v?: Record void }> = ({ onFileLoaded }) => { +const StepUpload: React.FC<{ + onFileLoaded: (headers: string[], rows: string[][]) => void; + fields: ImportWizardProps['fields']; + objectName: string; +}> = ({ onFileLoaded, fields, objectName }) => { const { t } = useImportTranslation(); const [dragOver, setDragOver] = useState(false); const [error, setError] = useState(null); @@ -360,6 +632,20 @@ const StepUpload: React.FC<{ onFileLoaded: (headers: string[], rows: string[][]) {t('grid.import.pasteHint')}

+ {fields.length > 0 && ( +
+ +

{t('grid.import.downloadTemplateHint')}

+
+ )} {error && (

{error} @@ -462,14 +748,26 @@ const StepMapping: React.FC<{ mapping: Record; onMappingChange: (mapping: Record) => void; inferredTypes: InferredType[]; + suggestions: ColumnSuggestion[]; templates: ImportMappingTemplate[]; selectedTemplateId: string | null; onSelectTemplate: (id: string) => void; onSaveTemplate: (name: string) => void; onDeleteTemplate: () => void; -}> = ({ headers, fields, mapping, onMappingChange, inferredTypes, templates, selectedTemplateId, onSelectTemplate, onSaveTemplate, onDeleteTemplate }) => { +}> = ({ headers, fields, mapping, onMappingChange, inferredTypes, suggestions, templates, selectedTemplateId, onSelectTemplate, onSaveTemplate, onDeleteTemplate }) => { const { t } = useImportTranslation(); const usedFields = useMemo(() => new Set(Object.values(mapping)), [mapping]); + const suggestionByCol = useMemo(() => { + const m = new Map(); + suggestions.forEach((s) => m.set(s.columnIndex, s)); + return m; + }, [suggestions]); + // How many columns were auto-matched (vs. the current, possibly-edited mapping). + const autoMatchedCount = useMemo(() => { + let n = 0; + suggestionByCol.forEach((s, idx) => { if (s.fieldName && mapping[idx] === s.fieldName) n++; }); + return n; + }, [suggestionByCol, mapping]); const handleChange = useCallback((colIdx: number, fieldName: string) => { const next = { ...mapping }; if (fieldName === '__skip__') delete next[colIdx]; else next[colIdx] = fieldName; @@ -486,6 +784,12 @@ const StepMapping: React.FC<{ onDelete={onDeleteTemplate} disabled={Object.keys(mapping).length === 0} /> + {autoMatchedCount > 0 && ( +

+ + {t('grid.import.autoMatchedSummary', { count: String(autoMatchedCount) })} +

+ )}
@@ -500,6 +804,10 @@ const StepMapping: React.FC<{ const inferred = inferredTypes[idx] ?? 'text'; const mappedField = mapping[idx] ? fields.find((f) => f.name === mapping[idx]) : undefined; const typeMismatch = !!mappedField && !isTypeCompatible(inferred, mappedField.type); + const suggestion = suggestionByCol.get(idx); + // Badge only while the user's choice still matches what we auto-suggested. + const autoMatched = !!suggestion?.fieldName && mapping[idx] === suggestion.fieldName + ? suggestion.confidence : null; return ( @@ -524,6 +832,11 @@ const StepMapping: React.FC<{ ))} + {autoMatched && ( +

+ {t('grid.import.autoMatched')} · {t(`grid.import.confidence.${autoMatched}`)} +

+ )} {typeMismatch && (

{t('grid.import.typeMismatch', { type: t(`grid.import.type.${inferred}`) })} @@ -652,6 +965,273 @@ const StepPreview: React.FC<{ ); }; +/** Options controlling how the server commits each row (insert/update/upsert + * + toggles). Rendered above the preview so the choices are visible before the + * import runs. */ +const ImportOptions: React.FC<{ + fields: ImportWizardProps['fields']; + mapping: Record; + writeMode: ImportWriteMode; + onWriteMode: (m: ImportWriteMode) => void; + matchFields: string[]; + onToggleMatchField: (name: string) => void; + createMissingOptions: boolean; + onCreateMissingOptions: (v: boolean) => void; + runAutomations: boolean; + onRunAutomations: (v: boolean) => void; + skipBlankMatchKey: boolean; + onSkipBlankMatchKey: (v: boolean) => void; + showBackground: boolean; + backgroundImport: boolean; + onBackgroundImport: (v: boolean) => void; +}> = ({ + fields, mapping, writeMode, onWriteMode, matchFields, onToggleMatchField, + createMissingOptions, onCreateMissingOptions, runAutomations, onRunAutomations, + skipBlankMatchKey, onSkipBlankMatchKey, + showBackground, backgroundImport, onBackgroundImport, +}) => { + const { t } = useImportTranslation(); + // Only fields that are actually mapped can serve as match keys. + const mappedFieldNames = useMemo(() => new Set(Object.values(mapping)), [mapping]); + const matchable = useMemo(() => fields.filter((f) => mappedFieldNames.has(f.name)), [fields, mappedFieldNames]); + const needsMatch = writeMode !== 'insert'; + + return ( +

+

{t('grid.import.options')}

+
+
+ + +
+ + {needsMatch && ( +
+ +
+ {matchable.length === 0 && ( + {t('grid.import.matchFieldsPlaceholder')} + )} + {matchable.map((f) => ( + + ))} +
+

+ {matchFields.length === 0 ? t('grid.import.needMatchFields') : t('grid.import.matchFieldsHint')} +

+
+ )} + +
+ + {needsMatch && ( + + )} + + {showBackground && ( + + )} +
+
+
+ ); +}; + +/** Colour intent for each import-job status badge. */ +const IMPORT_JOB_STATUS_VARIANT: Record = { + pending: 'outline', + running: 'secondary', + succeeded: 'default', + failed: 'destructive', + cancelled: 'outline', +}; + +/** Format an ISO timestamp compactly for the history table; falls back to the + * raw string (or a dash) when it isn't a parseable date. */ +function formatImportJobTime(iso?: string): string { + if (!iso) return '—'; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return iso; + return d.toLocaleString(); +} + +/** + * Import-job history for one object: lists prior async jobs (status, counts, + * time), lets the user cancel an in-flight job, and refresh. Degrades to an + * empty state when the data source lacks `listImportJobs` (older adapter). + */ +const ImportHistoryPanel: React.FC<{ + objectName: string; + dataSource: unknown; + t: (key: string, vars?: Record) => string; +}> = ({ objectName, dataSource, t }) => { + const [jobs, setJobs] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + // Job id currently being undone (disables its row's Undo button + confirm). + const [undoingId, setUndoingId] = useState(null); + + const ds = dataSource as Partial | undefined; + const supported = typeof ds?.listImportJobs === 'function'; + const canUndo = typeof ds?.undoImportJob === 'function'; + + const load = useCallback(async () => { + if (typeof ds?.listImportJobs !== 'function') return; + setLoading(true); setError(null); + try { + const list = await ds.listImportJobs({ object: objectName, limit: 50 }); + setJobs(list); + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + setJobs([]); + } finally { + setLoading(false); + } + }, [ds, objectName]); + + useEffect(() => { void load(); }, [load]); + + const handleCancel = useCallback(async (jobId: string) => { + if (typeof ds?.cancelImportJob !== 'function') return; + try { await ds.cancelImportJob(jobId); } catch { /* best-effort */ } + void load(); + }, [ds, load]); + + // Logical rollback: delete created records + restore updated ones. Confirms + // first (destructive + irreversible), then reloads so the row flips to + // "reverted" and its Undo button disappears. + const handleUndo = useCallback(async (jobId: string) => { + if (typeof ds?.undoImportJob !== 'function') return; + // eslint-disable-next-line no-alert + if (typeof window !== 'undefined' && !window.confirm(t('grid.import.undoConfirm'))) return; + setUndoingId(jobId); setError(null); + try { + await ds.undoImportJob(jobId); + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + } finally { + setUndoingId(null); + void load(); + } + }, [ds, load, t]); + + if (!supported) { + return ( +
+ {t('grid.import.historyUnsupported')} +
+ ); + } + + return ( +
+
+

{t('grid.import.historyHint')}

+ +
+ {error &&

{error}

} + {jobs && jobs.length === 0 && !loading && ( +

+ {t('grid.import.historyEmpty')} +

+ )} + {jobs && jobs.length > 0 && ( +
+ + + {t('grid.import.historyColStatus')} + {t('grid.import.historyColRows')} + {t('grid.import.historyColResult')} + {t('grid.import.historyColTime')} + + + + + {jobs.map((job) => ( + + + + {t(`grid.import.jobStatus.${job.status}`)} + + + {job.processed}/{job.total} + + {t('grid.import.createdCount', { count: job.created })} + {job.updated > 0 && · {t('grid.import.updatedCount', { count: job.updated })}} + {job.skipped > 0 && · {t('grid.import.skippedCount', { count: job.skipped })}} + {job.errors > 0 && · {t('grid.import.errorCount', { count: job.errors })}} + + {formatImportJobTime(job.completedAt ?? job.createdAt)} + + {isImportJobActive(job.status) && typeof ds?.cancelImportJob === 'function' && ( + + )} + {job.revertedAt && ( + + {t('grid.import.reverted')} + + )} + {isImportJobUndoable(job, canUndo) && ( + + )} + + + ))} + +
+ )} +
+ ); +}; + // Main wizard component export const ImportWizard: React.FC = ({ objectName, objectLabel, fields, dataSource, onComplete, onCancel, open, onOpenChange, onErrorMode = 'skip', @@ -665,8 +1245,53 @@ export const ImportWizard: React.FC = ({ const [progress, setProgress] = useState(0); const [result, setResult] = useState(null); const [corrections, setCorrections] = useState>>({}); + // Import-job history view (swaps the wizard body for a list of prior jobs). + const [showHistory, setShowHistory] = useState(false); + // Async (large-file) import job — jobId + live processed/total, plus a ref the + // poll loop reads so a mid-flight Cancel stops polling without a re-render race. + const [jobId, setJobId] = useState(null); + const [asyncCounts, setAsyncCounts] = useState<{ processed: number; total: number } | null>(null); + const cancelPollRef = React.useRef(false); + // Small-file server dry-run pre-check — validates the exact payload without + // writing, so the summary/error list reflect real coercion outcomes. + const [validating, setValidating] = useState(false); + const [dryRunResult, setDryRunResult] = useState(null); + // Write-mode + coercion options (drive the server-side /import request). + const [writeMode, setWriteMode] = useState('insert'); + const [matchFields, setMatchFields] = useState([]); + const [createMissingOptions, setCreateMissingOptions] = useState(false); + const [runAutomations, setRunAutomations] = useState(false); + const [skipBlankMatchKey, setSkipBlankMatchKey] = useState(false); + // Opt-in: route this import through a background job even when the row count + // is under the async threshold. This is the only way to obtain an undoable + // job for a small import — the sync path never captures undo state. + const [backgroundImport, setBackgroundImport] = useState(false); const label = objectLabel ?? objectName; + // The background-import toggle only makes sense when the data source can + // actually run jobs (create + poll + fetch results). Mirrors the guard in + // runAsyncImport so the checkbox never promises an unsupported path. + const supportsImportJob = useMemo(() => { + const ds = dataSource as Partial | undefined; + return typeof ds?.createImportJob === 'function' + && typeof ds?.getImportJobProgress === 'function' + && typeof ds?.getImportJobResults === 'function'; + }, [dataSource]); + + const toggleMatchField = useCallback((name: string) => { + setMatchFields((prev) => (prev.includes(name) ? prev.filter((n) => n !== name) : [...prev, name])); + }, []); + + // Keep matchFields consistent with the current column mapping — drop any + // match key whose column was unmapped so we never send a stale key. + useEffect(() => { + const mapped = new Set(Object.values(mapping)); + setMatchFields((prev) => { + const next = prev.filter((n) => mapped.has(n)); + return next.length === prev.length ? prev : next; + }); + }, [mapping]); + // Template storage — resolved once; `null` opts out of persistence. const storage = useMemo( () => (templateStorage === undefined ? defaultTemplateStorage() : templateStorage), @@ -738,8 +1363,34 @@ export const ImportWizard: React.FC = ({ [headers, rows], ); - const handleImport = useCallback(async () => { - setImporting(true); setProgress(0); + // Airtable-style auto-mapping suggestions (with confidence), computed once per + // file. Drives the "auto-matched" badges: a badge shows only while the user's + // current mapping for a column still equals what we suggested. + const suggestions = useMemo( + () => suggestColumnMappings(headers, fields, rows), + [headers, fields, rows], + ); + + // Build raw, mapping-applied rows keyed by target field name (inline + // corrections applied). Values stay RAW strings — the server coerces them to + // storage values from field metadata, so booleans / dates / lookups / selects + // are all handled uniformly server-side rather than guessed on the client. + const buildRawRows = useCallback((): Array> => { + const mappedCols = Object.entries(mapping).map(([idx, name]) => ({ csvIdx: Number(idx), field: name })); + return rows.map((original, i) => { + const fixes = corrections[i]; + const out: Record = {}; + for (const col of mappedCols) { + const v = fixes && fixes[col.csvIdx] !== undefined ? fixes[col.csvIdx] : (original[col.csvIdx] ?? ''); + out[col.field] = v; + } + return out; + }); + }, [rows, mapping, corrections]); + + // Legacy fallback — per-row `create` with light client-side validation. Used + // only when the adapter/client can't reach the server `/import` route. + const legacyImport = useCallback(async () => { const errors: ImportResult['errors'] = []; let importedRows = 0, skippedRows = 0; const mappedCols = Object.entries(mapping).map(([idx, name]) => ({ @@ -747,8 +1398,6 @@ export const ImportWizard: React.FC = ({ })); for (let i = 0; i < rows.length; i++) { - // Apply inline corrections (only available for the visible preview rows) - // before validation so users can fix issues without re-uploading the file. const original = rows[i]; const fixes = corrections[i]; const effectiveRow = fixes @@ -775,11 +1424,256 @@ export const ImportWizard: React.FC = ({ setResult(importResult); setImporting(false); onComplete?.(importResult); }, [rows, mapping, fields, dataSource, objectName, onComplete, onErrorMode, corrections]); + // Large-file path: hand the rows to a server-side background job and poll it + // to completion. Returns `true` when the async path handled the import + // (success / failure / cancel) and `false` when the data source can't run + // jobs, signalling the caller to fall back to the synchronous route. + const runAsyncImport = useCallback(async (request: ImportRequestOptions): Promise => { + const ds = dataSource as Partial | undefined; + if ( + typeof ds?.createImportJob !== 'function' || + typeof ds?.getImportJobProgress !== 'function' || + typeof ds?.getImportJobResults !== 'function' + ) { + return false; + } + + cancelPollRef.current = false; + let created: CreateImportJobResult; + try { + created = await ds.createImportJob(objectName, request); + } catch (err) { + if (isUnsupportedImportJob(err)) return false; + throw err; + } + setJobId(created.jobId); + setAsyncCounts({ processed: 0, total: created.total }); + + const terminal = new Set(['succeeded', 'failed', 'cancelled']); + let consecutivePollErrors = 0; + // Poll until the job reaches a terminal state (or the user cancels, in + // which case the cancel handler owns producing the result). + for (;;) { + if (cancelPollRef.current) return true; + await new Promise((resolve) => setTimeout(resolve, IMPORT_JOB_POLL_INTERVAL)); + if (cancelPollRef.current) return true; + + let prog: ImportJobProgressInfo; + try { + prog = await ds.getImportJobProgress(created.jobId); + consecutivePollErrors = 0; + } catch (err) { + // Tolerate transient poll blips; give up only after several in a row so + // a network hiccup doesn't abort an import that's still running server-side. + if (++consecutivePollErrors >= 5) throw err; + continue; + } + + setAsyncCounts({ processed: prog.processed, total: prog.total }); + setProgress(prog.percentComplete); + + if (!terminal.has(prog.status)) continue; + + if (prog.status === 'cancelled') { + const importResult: ImportResult = { + totalRows: prog.total, + importedRows: prog.created + prog.updated, + skippedRows: prog.skipped + prog.errors, + createdRows: prog.created, + updatedRows: prog.updated, + errors: [], + cancelled: true, + }; + setResult(importResult); setImporting(false); onComplete?.(importResult); + return true; + } + + const results = await ds.getImportJobResults(created.jobId); + const importResult = jobResultToImportResult(results); + if (prog.status === 'failed' && importResult.errors.length === 0) { + importResult.errors.push({ row: 0, field: '', message: prog.error ?? 'Import failed' }); + } + setProgress(100); + setResult(importResult); setImporting(false); onComplete?.(importResult); + return true; + } + }, [dataSource, objectName, onComplete]); + + // Assemble the server import request from the current mapping + options. + // `dryRun` reuses the exact same payload the real import will send, so the + // pre-check validates precisely what would be written. + const buildImportRequest = useCallback((dryRun = false): ImportRequestOptions => + assembleImportRequest(buildRawRows(), { + writeMode, matchFields, createMissingOptions, runAutomations, skipBlankMatchKey, dryRun, + }), + [buildRawRows, writeMode, matchFields, createMissingOptions, runAutomations, skipBlankMatchKey]); + + const handleImport = useCallback(async () => { + setImporting(true); setProgress(0); + cancelPollRef.current = false; + setJobId(null); setAsyncCounts(null); + + const request = buildImportRequest(); + + // Route large files through a background job so they neither block the UI + // nor trip the sync route's row ceiling. Small files can also opt into the + // background path (the "background import" toggle) — that's the only way to + // get an undoable job for a sub-threshold import. Any unsupported signal + // (older adapter / client / server) falls through to the synchronous path. + if (rows.length > ASYNC_IMPORT_THRESHOLD || backgroundImport) { + try { + const handled = await runAsyncImport(request); + if (handled) return; + } catch (err) { + if (!isUnsupportedImportJob(err)) { + const msg = err instanceof Error ? err.message : 'Import failed'; + const importResult: ImportResult = { + totalRows: rows.length, importedRows: 0, skippedRows: rows.length, + errors: [{ row: 0, field: '', message: msg }], + }; + setResult(importResult); setImporting(false); onComplete?.(importResult); + return; + } + // Unsupported — fall through to the synchronous path below. + } + } + + // Prefer the single-call server import: it coerces special values and + // routes each row to insert / update / upsert. Fall back to the per-row + // create loop only when the adapter can't speak `/import`. + const serverImport = (dataSource as { + importRecords?: (o: string, r: ImportRequestOptions) => Promise; + } | undefined)?.importRecords; + + if (typeof serverImport === 'function') { + try { + const res = await serverImport.call(dataSource, objectName, request); + const importResult: ImportResult = { + totalRows: res.total, + importedRows: res.ok, + skippedRows: res.skipped + res.errors, + createdRows: res.created, + updatedRows: res.updated, + errors: res.results + .filter((r) => !r.ok) + .map((r) => ({ row: r.row, field: r.field ?? '', message: r.error ?? r.code ?? 'Import failed' })), + serverResult: res, + }; + setProgress(100); + setResult(importResult); setImporting(false); onComplete?.(importResult); + return; + } catch (err) { + if (!isUnsupportedImport(err)) { + // A real server failure — surface it rather than silently retrying + // via the legacy loop (which could double-import partial successes). + const msg = err instanceof Error ? err.message : 'Import failed'; + const importResult: ImportResult = { + totalRows: rows.length, importedRows: 0, skippedRows: rows.length, + errors: [{ row: 0, field: '', message: msg }], + }; + setResult(importResult); setImporting(false); onComplete?.(importResult); + return; + } + // Unsupported — fall through to the legacy path below. + } + } + + await legacyImport(); + }, [ + dataSource, objectName, buildImportRequest, onComplete, rows.length, legacyImport, runAsyncImport, + backgroundImport, + ]); + + // Small-file server dry-run pre-check: validate + coerce every row without + // persisting, so mapping / type / required errors are caught before import. + // Large files skip this (they're validated row-by-row during the async job). + const handleValidate = useCallback(async () => { + const serverImport = (dataSource as { + importRecords?: (o: string, r: ImportRequestOptions) => Promise; + } | undefined)?.importRecords; + if (typeof serverImport !== 'function') return; + setValidating(true); + try { + const res = await serverImport.call(dataSource, objectName, buildImportRequest(true)); + setDryRunResult(res); + } catch (err) { + // Older adapter/client without /import — silently fall back to the + // client-side cell validation that StepPreview already shows. + if (!isUnsupportedImport(err)) { + const msg = err instanceof Error ? err.message : 'Validation failed'; + setDryRunResult({ + object: objectName, dryRun: true, writeMode, total: rows.length, + ok: 0, errors: rows.length, created: 0, updated: 0, skipped: 0, + results: [{ row: 0, ok: false, error: msg }], + }); + } + } finally { + setValidating(false); + } + }, [dataSource, objectName, buildImportRequest, writeMode, rows.length]); + + // A prior dry-run becomes stale the moment the payload changes (mapping, + // write-mode, options, or an inline cell correction) — drop it so the summary + // never reflects data the user has since edited. + useEffect(() => { + setDryRunResult(null); + }, [mapping, corrections, writeMode, matchFields, createMissingOptions, runAutomations, skipBlankMatchKey]); + + // User-initiated cancel of an in-flight async job. Stops the poll loop, asks + // the server to cancel (best-effort), and shows a cancelled result. + const handleCancelImport = useCallback(async () => { + cancelPollRef.current = true; + const id = jobId; + const ds = dataSource as Partial | undefined; + if (id && typeof ds?.cancelImportJob === 'function') { + try { await ds.cancelImportJob(id); } catch { /* best-effort — the poll loop already stopped */ } + } + const importResult: ImportResult = { + totalRows: asyncCounts?.total ?? rows.length, + importedRows: 0, + skippedRows: 0, + errors: [], + cancelled: true, + }; + setResult(importResult); setImporting(false); + }, [jobId, dataSource, asyncCounts, rows.length]); + const reset = useCallback(() => { + cancelPollRef.current = false; setStep('upload'); setHeaders([]); setRows([]); setMapping({}); setProgress(0); setResult(null); setCorrections({}); setSelectedTemplateId(null); + setWriteMode('insert'); setMatchFields([]); + setCreateMissingOptions(false); setRunAutomations(false); setSkipBlankMatchKey(false); + setJobId(null); setAsyncCounts(null); + setValidating(false); setDryRunResult(null); + setShowHistory(false); }, []); + /** Download a CSV of just the failed rows (original values + `_error`). */ + const handleDownloadFailed = useCallback(() => { + if (!result || result.errors.length === 0) return; + const errorsByRow = new Map(); + for (const e of result.errors) { + if (e.row < 1) continue; // top-level (row 0) errors have no source row + const prefix = e.field ? `${e.field}: ` : ''; + const existing = errorsByRow.get(e.row); + errorsByRow.set(e.row, existing ? `${existing}; ${prefix}${e.message}` : `${prefix}${e.message}`); + } + if (errorsByRow.size === 0) return; + const csv = buildFailedRowsCsv(headers, rows, mapping, errorsByRow); + try { + const blob = new Blob([`${csv}`], { type: 'text/csv;charset=utf-8' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `${objectName}-import-errors.csv`; + document.body.appendChild(a); + a.click(); + a.remove(); + setTimeout(() => URL.revokeObjectURL(url), 0); + } catch { /* non-browser env */ } + }, [result, headers, rows, mapping, objectName]); + const handleClose = useCallback(() => { reset(); onOpenChange?.(false); onCancel?.(); }, [reset, onOpenChange, onCancel]); const { t } = useImportTranslation(); @@ -795,32 +1689,51 @@ export const ImportWizard: React.FC = ({ onInteractOutside={(e) => e.preventDefault()} > - - {t('grid.import.title', { object: label })} - +
+ + {t('grid.import.title', { object: label })} + + {step === 'upload' && !result && !importing + && typeof (dataSource as Partial | undefined)?.listImportJobs === 'function' && ( + + )} +
- {step === 'upload' && t('grid.import.uploadDescription')} - {step === 'mapping' && t('grid.import.mappingDescription')} - {step === 'preview' && t('grid.import.previewDescription')} + {showHistory + ? t('grid.import.historyDescription') + : step === 'upload' ? t('grid.import.uploadDescription') + : step === 'mapping' ? t('grid.import.mappingDescription') + : t('grid.import.previewDescription')} {/* Step indicators */} -
- {(['upload', 'mapping', 'preview'] as WizardStep[]).map((s, i) => ( - - {i > 0 && } - - {i + 1}. {s === 'upload' ? t('grid.import.stepUpload') : s === 'mapping' ? t('grid.import.stepMapping') : t('grid.import.stepPreview')} - - - ))} -
+ {!showHistory && ( +
+ {(['upload', 'mapping', 'preview'] as WizardStep[]).map((s, i) => ( + + {i > 0 && } + + {i + 1}. {s === 'upload' ? t('grid.import.stepUpload') : s === 'mapping' ? t('grid.import.stepMapping') : t('grid.import.stepPreview')} + + + ))} +
+ )}
- {!result ? ( + {showHistory ? ( + + ) : !result ? ( <> - {step === 'upload' && } + {step === 'upload' && } {step === 'mapping' && ( = ({ mapping={mapping} onMappingChange={setMapping} inferredTypes={inferredTypes} + suggestions={suggestions} templates={templates} selectedTemplateId={selectedTemplateId} onSelectTemplate={handleSelectTemplate} @@ -836,45 +1750,157 @@ export const ImportWizard: React.FC = ({ /> )} {step === 'preview' && ( - + <> + + + {rows.length > ASYNC_IMPORT_THRESHOLD && ( +
+ + {t('grid.import.largeSampleNotice', { shown: PREVIEW_ROW_COUNT, total: rows.length })} {t('grid.import.asyncLargeHint')} +
+ )} + {rows.length <= ASYNC_IMPORT_THRESHOLD + && typeof (dataSource as Partial | undefined)?.importRecords === 'function' && ( +
+
+

{t('grid.import.validateHint')}

+ +
+ {dryRunResult && ( +
+

0 ? 'text-destructive' : 'text-emerald-600'}`}> + {dryRunResult.errors > 0 + ? t('grid.import.validateFailed', { ok: dryRunResult.ok, errors: dryRunResult.errors }) + : t('grid.import.validatePassed', { ok: dryRunResult.ok })} +

+ {dryRunResult.errors > 0 && ( +
    + {dryRunResult.results.filter((r) => !r.ok).slice(0, 20).map((r, i) => ( +
  • + {r.row > 0 ? t('grid.import.errorRowPrefix', { row: r.row }) : ''} + {r.field ? `${r.field}: ` : ''}{r.error ?? r.code ?? ''} +
    • + ))} +
+ )} +
+ )} +
+ )} + )} {importing && ( -
- -

{t('grid.import.importing', { progress })}

+
+ +

+ {jobId + ? asyncCounts + ? t('grid.import.asyncProcessing', { processed: asyncCounts.processed, total: asyncCounts.total, progress }) + : t('grid.import.asyncQueued') + : t('grid.import.importing', { progress })} +

+ {jobId && typeof (dataSource as Partial | undefined)?.cancelImportJob === 'function' && ( + + )}
)} ) : (
- -

{t('grid.import.importComplete')}

-
- {t('grid.import.imported', { count: result.importedRows })} + {result.cancelled ? ( + <> + +

{t('grid.import.importCancelled')}

+ + ) : ( + <> + +

{t('grid.import.importComplete')}

+ + )} +
+ {/* Prefer the finer created/updated breakdown when the server + reports it; otherwise fall back to a single "imported" count. */} + {result.createdRows !== undefined || result.updatedRows !== undefined ? ( + <> + {(result.createdRows ?? 0) > 0 && {t('grid.import.createdCount', { count: result.createdRows })}} + {(result.updatedRows ?? 0) > 0 && {t('grid.import.updatedCount', { count: result.updatedRows })}} + {(result.createdRows ?? 0) === 0 && (result.updatedRows ?? 0) === 0 && ( + {t('grid.import.imported', { count: result.importedRows })} + )} + + ) : ( + {t('grid.import.imported', { count: result.importedRows })} + )} {result.skippedRows > 0 && {t('grid.import.skippedCount', { count: result.skippedRows })}}
{result.errors.length > 0 && ( -
- {result.errors.slice(0, 10).map((err, i) => ( -

Row {err.row}{err.field ? ` (${err.field})` : ''}: {err.message}

- ))} - {result.errors.length > 10 &&

{t('grid.import.moreErrors', { count: result.errors.length - 10 })}

} -
+ <> +
+ {result.errors.slice(0, 10).map((err, i) => ( +

Row {err.row}{err.field ? ` (${err.field})` : ''}: {err.message}

+ ))} + {result.errors.length > 10 &&

{t('grid.import.moreErrors', { count: result.errors.length - 10 })}

} +
+ {result.errors.some((e) => e.row >= 1) && ( + + )} + + )} + {result.resultsTruncated && ( +

+ {t('grid.import.resultsTruncated', { count: result.errors.length, total: result.skippedRows })} +

)}
)}
- {result ? ( - + {showHistory ? ( + + ) : result ? ( + ) : ( <> @@ -884,12 +1910,16 @@ export const ImportWizard: React.FC = ({ )} {step === 'mapping' && ( - )} {step === 'preview' && ( - )} diff --git a/packages/plugin-grid/src/importAsyncPath.test.ts b/packages/plugin-grid/src/importAsyncPath.test.ts new file mode 100644 index 000000000..05ba3169f --- /dev/null +++ b/packages/plugin-grid/src/importAsyncPath.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect } from 'vitest'; +import { __testables } from './ImportWizard'; +import type { ImportJobResultsInfo } from '@object-ui/types'; + +const { isUnsupportedImportJob, jobResultToImportResult } = __testables; + +describe('isUnsupportedImportJob', () => { + it('matches the adapter UNSUPPORTED_OPERATION code', () => { + expect(isUnsupportedImportJob({ code: 'UNSUPPORTED_OPERATION' })).toBe(true); + }); + + it('matches a "createImportJob is not a function" message', () => { + expect(isUnsupportedImportJob(new Error('dataSource.createImportJob is not a function'))).toBe(true); + }); + + it('matches a 404 / missing import/jobs route (older server)', () => { + expect(isUnsupportedImportJob(new Error('POST /data/task/import/jobs 404 Not Found'))).toBe(true); + expect(isUnsupportedImportJob(new Error('Request failed with status 404'))).toBe(true); + }); + + it('does NOT match a genuine server/validation error', () => { + expect(isUnsupportedImportJob(new Error('Row 3: value out of range'))).toBe(false); + expect(isUnsupportedImportJob({ code: 'PAYLOAD_TOO_LARGE' })).toBe(false); + expect(isUnsupportedImportJob(null)).toBe(false); + expect(isUnsupportedImportJob(undefined)).toBe(false); + }); +}); + +describe('jobResultToImportResult', () => { + const base: ImportJobResultsInfo = { + jobId: 'imp_1', + object: 'task', + status: 'succeeded', + total: 5, + processed: 5, + created: 3, + updated: 1, + skipped: 0, + errors: 1, + percentComplete: 100, + resultsTruncated: false, + results: [ + { row: 1, ok: true, action: 'created', id: 'a' }, + { row: 2, ok: true, action: 'created', id: 'b' }, + { row: 3, ok: true, action: 'created', id: 'c' }, + { row: 4, ok: true, action: 'updated', id: 'd' }, + { row: 5, ok: false, action: 'failed', field: 'amount', error: 'not a number', code: 'COERCE' }, + ], + }; + + it('maps job counters onto the wizard ImportResult shape (created+updated = imported)', () => { + const r = jobResultToImportResult(base); + expect(r.totalRows).toBe(5); + expect(r.importedRows).toBe(4); // created 3 + updated 1 + expect(r.createdRows).toBe(3); + expect(r.updatedRows).toBe(1); + expect(r.skippedRows).toBe(1); // skipped 0 + errors 1 + expect(r.resultsTruncated).toBe(false); + }); + + it('projects only failed rows into the errors list, preferring error over code', () => { + const r = jobResultToImportResult(base); + expect(r.errors).toEqual([{ row: 5, field: 'amount', message: 'not a number' }]); + }); + + it('carries the resultsTruncated flag through when the server capped the report', () => { + const r = jobResultToImportResult({ ...base, resultsTruncated: true, results: [] }); + expect(r.resultsTruncated).toBe(true); + expect(r.errors).toEqual([]); + }); +}); diff --git a/packages/plugin-grid/src/importDryRun.test.ts b/packages/plugin-grid/src/importDryRun.test.ts new file mode 100644 index 000000000..9e9213b3b --- /dev/null +++ b/packages/plugin-grid/src/importDryRun.test.ts @@ -0,0 +1,54 @@ +import { describe, it, expect } from 'vitest'; +import { __testables } from './ImportWizard'; + +const { assembleImportRequest } = __testables; + +const rows = [{ name: 'Acme' }, { name: 'Beta' }]; +const baseOpts = { + writeMode: 'insert' as const, + matchFields: [] as string[], + createMissingOptions: false, + runAutomations: false, + skipBlankMatchKey: false, +}; + +describe('assembleImportRequest', () => { + it('omits dryRun on a real import request', () => { + const req = assembleImportRequest(rows, baseOpts); + expect(req).not.toHaveProperty('dryRun'); + expect(req.format).toBe('json'); + expect(req.rows).toBe(rows); + expect(req.writeMode).toBe('insert'); + }); + + it('sets dryRun:true when validating, keeping the rest of the payload identical', () => { + const live = assembleImportRequest(rows, baseOpts); + const dry = assembleImportRequest(rows, { ...baseOpts, dryRun: true }); + expect(dry.dryRun).toBe(true); + // dryRun is the ONLY difference — the pre-check validates the exact payload. + expect({ ...dry, dryRun: undefined }).toEqual({ ...live, dryRun: undefined }); + }); + + it('drops matchFields for insert mode but sends them for update/upsert', () => { + const insert = assembleImportRequest(rows, { ...baseOpts, writeMode: 'insert', matchFields: ['name'] }); + expect(insert).not.toHaveProperty('matchFields'); + + const upsert = assembleImportRequest(rows, { ...baseOpts, writeMode: 'upsert', matchFields: ['name'] }); + expect(upsert.matchFields).toEqual(['name']); + + const update = assembleImportRequest(rows, { ...baseOpts, writeMode: 'update', matchFields: ['email'] }); + expect(update.matchFields).toEqual(['email']); + }); + + it('threads coercion options through verbatim', () => { + const req = assembleImportRequest(rows, { + ...baseOpts, + createMissingOptions: true, + runAutomations: true, + skipBlankMatchKey: true, + }); + expect(req.createMissingOptions).toBe(true); + expect(req.runAutomations).toBe(true); + expect(req.skipBlankMatchKey).toBe(true); + }); +}); diff --git a/packages/plugin-grid/src/importHistory.test.ts b/packages/plugin-grid/src/importHistory.test.ts new file mode 100644 index 000000000..22d662347 --- /dev/null +++ b/packages/plugin-grid/src/importHistory.test.ts @@ -0,0 +1,42 @@ +import { describe, it, expect } from 'vitest'; +import { __testables } from './ImportWizard'; +import type { ImportJobStatus } from '@object-ui/types'; + +const { isImportJobActive, isImportJobUndoable } = __testables; + +describe('isImportJobActive', () => { + it('is true only while a job is still in flight', () => { + expect(isImportJobActive('pending')).toBe(true); + expect(isImportJobActive('running')).toBe(true); + }); + + it('is false for every terminal status', () => { + const terminal: ImportJobStatus[] = ['succeeded', 'failed', 'cancelled']; + for (const s of terminal) expect(isImportJobActive(s)).toBe(false); + }); +}); + +describe('isImportJobUndoable', () => { + const base = { status: 'succeeded' as ImportJobStatus, undoable: true, revertedAt: undefined }; + + it('shows Undo for a terminal, undoable, not-yet-reverted job', () => { + expect(isImportJobUndoable(base, true)).toBe(true); + }); + + it('hides Undo when the adapter cannot undo', () => { + expect(isImportJobUndoable(base, false)).toBe(false); + }); + + it('hides Undo when the job did not capture an undo log', () => { + expect(isImportJobUndoable({ ...base, undoable: false }, true)).toBe(false); + }); + + it('hides Undo once the job has been reverted', () => { + expect(isImportJobUndoable({ ...base, revertedAt: '2026-07-01T00:00:00Z' }, true)).toBe(false); + }); + + it('hides Undo while the job is still active', () => { + expect(isImportJobUndoable({ ...base, status: 'running' }, true)).toBe(false); + expect(isImportJobUndoable({ ...base, status: 'pending' }, true)).toBe(false); + }); +}); diff --git a/packages/plugin-grid/src/importParsers.test.ts b/packages/plugin-grid/src/importParsers.test.ts index e852731ff..4eb3b1cd7 100644 --- a/packages/plugin-grid/src/importParsers.test.ts +++ b/packages/plugin-grid/src/importParsers.test.ts @@ -7,6 +7,7 @@ import ExcelJS from 'exceljs'; import { parseDelimited, parseCSV, parseExcelArrayBuffer, parseClipboardTable, inferColumnType, isTypeCompatible, ImportParseError, parseSpreadsheetFile, + suggestColumnMappings, scoreToConfidence, type MappableField, } from './importParsers'; describe('parseDelimited', () => { @@ -109,3 +110,67 @@ describe('parseSpreadsheetFile', () => { expect(grid).toEqual([['a', 'b'], ['1', '2']]); }); }); + +describe('suggestColumnMappings', () => { + const FIELDS: MappableField[] = [ + { name: 'full_name', label: '姓名', type: 'text' }, + { name: 'email', label: 'Email', type: 'email' }, + { name: 'phone', label: '手机', type: 'text' }, + { name: 'amount', label: '金额', type: 'currency' }, + { name: 'active', label: '启用', type: 'boolean' }, + { name: 'due_date', label: '截止日期', type: 'date' }, + ]; + const byCol = (s: ReturnType) => + Object.fromEntries(s.map((x) => [x.columnIndex, x.fieldName])); + + it('matches exact and normalized header names with high confidence', () => { + const s = suggestColumnMappings(['Full Name', 'email'], FIELDS); + expect(s[0]).toMatchObject({ fieldName: 'full_name', confidence: 'high' }); + expect(s[1]).toMatchObject({ fieldName: 'email', reason: 'exact', confidence: 'high' }); + }); + + it('resolves bilingual synonyms (邮箱→email, 电话→phone)', () => { + const s = suggestColumnMappings(['邮箱', '电话'], FIELDS); + expect(byCol(s)).toMatchObject({ 0: 'email', 1: 'phone' }); + expect(s[0].reason).toBe('synonym'); + }); + + it('assigns each field to at most one column (global greedy)', () => { + // Two columns both look like email; only the better one wins the field. + const s = suggestColumnMappings(['email', 'e-mail address'], FIELDS, + [['a@x.com', 'b@x.com']]); + const assigned = s.map((x) => x.fieldName).filter(Boolean); + expect(assigned).toContain('email'); + expect(new Set(assigned).size).toBe(assigned.length); // no dup field + }); + + it('uses content type to gate a fuzzy match (numeric column → currency)', () => { + const s = suggestColumnMappings(['金额'], FIELDS, [['1200'], ['3400']]); + expect(s[0]).toMatchObject({ fieldName: 'amount' }); + expect(s[0].inferredType).toBe('number'); + }); + + it('discounts a name match when the content type is incompatible', () => { + // Header rhymes with a boolean field, but the sampled data is text → no confident map. + const s = suggestColumnMappings(['active'], [{ name: 'active', label: 'Active', type: 'boolean' }], + [['some free text'], ['more prose here']]); + // exact name still wins (exact matches aren't type-gated)… + expect(s[0].fieldName).toBe('active'); + // …but a merely-fuzzy header with bad type stays unmapped: + const s2 = suggestColumnMappings(['is_it_on'], [{ name: 'active', label: 'Active', type: 'boolean' }], + [['free text'], ['prose']]); + expect(s2[0].fieldName).toBeNull(); + }); + + it('leaves an unknown column unmapped', () => { + const s = suggestColumnMappings(['xyzzy_42'], FIELDS); + expect(s[0]).toMatchObject({ fieldName: null, confidence: null, reason: 'none' }); + }); + + it('scoreToConfidence buckets by threshold', () => { + expect(scoreToConfidence(1)).toBe('high'); + expect(scoreToConfidence(0.6)).toBe('medium'); + expect(scoreToConfidence(0.3)).toBe('low'); + expect(scoreToConfidence(0)).toBeNull(); + }); +}); diff --git a/packages/plugin-grid/src/importParsers.ts b/packages/plugin-grid/src/importParsers.ts index 539ac44bd..5e9a4b79f 100644 --- a/packages/plugin-grid/src/importParsers.ts +++ b/packages/plugin-grid/src/importParsers.ts @@ -192,3 +192,179 @@ export function isTypeCompatible(inferred: InferredType, fieldType: string): boo default: return true; } } + +// ── Airtable-style column → field mapping suggestions ─────────────────── + +/** Normalize a header/field key: lower-case, strip separators/punctuation. + * Also strips the `*`/`＊` required-marker our downloaded templates append to + * header labels, so a filled-in template round-trips back to the same field. */ +const normalizeKey = (s: string): string => + s.toLowerCase().replace(/[\s_\-.]+/g, '').replace(/[()（）[\]{}:：,，、/*＊]/g, ''); + +/** Split a header/label into comparable tokens (space/underscore/case/CJK aware). */ +function tokenize(s: string): string[] { + return s + .replace(/([a-z])([A-Z])/g, '$1 $2') + .toLowerCase() + .split(/[\s_\-./()（）[\]{}:：,，、*＊]+/) + .map((t) => t.trim()) + .filter(Boolean); +} + +/** + * Bilingual synonym groups — headers and field names in the same group are + * treated as strong matches even when neither contains the other (e.g. a + * `邮箱` column onto an `email` field). Deliberately conservative: only + * unambiguous, common CRM/spreadsheet concepts. + */ +const SYNONYM_GROUPS: string[][] = [ + ['name', 'fullname', 'displayname', '姓名', '名称', '名字'], + ['email', 'emailaddress', 'mail', '邮箱', '电子邮箱', '邮件'], + ['phone', 'tel', 'telephone', 'mobile', 'cell', '手机', '电话', '手机号', '联系电话'], + ['date', '日期'], + ['datetime', 'timestamp', '时间', '日期时间'], + ['amount', 'price', 'total', 'cost', '金额', '价格', '总额', '总价', '费用'], + ['status', 'state', '状态'], + ['address', '地址'], + ['company', 'organization', 'org', '公司', '单位', '组织', '企业'], + ['description', 'desc', 'note', 'notes', 'remark', 'remarks', '备注', '描述', '说明'], + ['id', 'code', 'number', 'no', '编号', '编码', '代码'], + ['quantity', 'qty', 'count', '数量'], + ['country', '国家'], ['city', '城市'], ['province', 'state', '省份', '省'], + ['gender', 'sex', '性别'], ['age', '年龄'], + ['title', '标题', '职位', '头衔'], + ['owner', 'assignee', 'assignedto', '负责人', '负责', '所有者', '归属人'], + ['createdat', 'createdon', 'createtime', '创建时间'], + ['updatedat', 'updatedon', 'modifiedtime', '更新时间'], +]; + +const SYNONYM_INDEX: Map = (() => { + const m = new Map(); + SYNONYM_GROUPS.forEach((group, gi) => group.forEach((k) => m.set(normalizeKey(k), gi))); + return m; +})(); + +/** Confidence bucket used for the mapping UI badge. */ +export type MappingConfidence = 'high' | 'medium' | 'low'; + +/** Why a column was matched to a field — drives the UI hint text. */ +export type MappingReason = 'exact' | 'normalized' | 'synonym' | 'contains' | 'token' | 'none'; + +/** A per-column mapping suggestion with a confidence score, à la Airtable. */ +export interface ColumnSuggestion { + columnIndex: number; + /** The suggested field name, or `null` when nothing matched confidently. */ + fieldName: string | null; + /** Match strength in [0, 1]. */ + score: number; + confidence: MappingConfidence | null; + reason: MappingReason; + /** Content-inferred type of the column (for the type-mismatch hint). */ + inferredType: InferredType; +} + +/** Minimal field descriptor the mapper needs (keeps this module React-free). */ +export interface MappableField { + name: string; + label?: string; + type: string; +} + +/** Map a raw score to a confidence bucket (null → not assigned). */ +export function scoreToConfidence(score: number): MappingConfidence | null { + if (score >= 0.85) return 'high'; + if (score >= 0.55) return 'medium'; + if (score > 0) return 'low'; + return null; +} + +/** Minimum score for a column/field pair to be auto-applied. */ +const MIN_MATCH_SCORE = 0.4; + +function jaccard(a: string[], b: string[]): number { + if (a.length === 0 || b.length === 0) return 0; + const sa = new Set(a); + const sb = new Set(b); + let inter = 0; + sa.forEach((x) => { if (sb.has(x)) inter++; }); + return inter / (sa.size + sb.size - inter); +} + +/** Best (score, reason) for a single header ↔ field pair. */ +function scorePair(header: string, field: MappableField, inferred: InferredType): { score: number; reason: MappingReason } { + const targets = [field.name, field.label].filter((s): s is string => !!s); + const hNorm = normalizeKey(header); + const hTokens = tokenize(header); + let score = 0; + let reason: MappingReason = 'none'; + const bump = (s: number, r: MappingReason) => { if (s > score) { score = s; reason = r; } }; + + for (const target of targets) { + if (header.trim() === target.trim()) { bump(1, 'exact'); continue; } + const tNorm = normalizeKey(target); + if (hNorm && tNorm && hNorm === tNorm) { bump(0.95, 'normalized'); continue; } + if (hNorm && tNorm && (hNorm.includes(tNorm) || tNorm.includes(hNorm))) { + const ratio = Math.min(hNorm.length, tNorm.length) / Math.max(hNorm.length, tNorm.length); + bump(0.5 + ratio * 0.35, 'contains'); + } + const j = jaccard(hTokens, tokenize(target)); + if (j > 0) bump(0.4 + j * 0.45, 'token'); + } + + // Synonym signal — header and any target land in the same concept group. + const hGroup = SYNONYM_INDEX.get(hNorm); + if (hGroup !== undefined && targets.some((t) => SYNONYM_INDEX.get(normalizeKey(t)) === hGroup)) { + bump(0.82, 'synonym'); + } + + // Type gate: for softer (non-exact) matches, reward a compatible inferred + // type and heavily discount an incompatible one so we don't confidently map + // a text column onto a number field just because the names rhyme. + if (reason !== 'exact' && reason !== 'normalized' && inferred !== 'text' && score > 0) { + if (isTypeCompatible(inferred, field.type)) score = Math.min(1, score + 0.05); + else score *= 0.5; + } + return { score, reason }; +} + +/** + * Suggest a field for every source column, Airtable-style: score each + * column/field pair on name/label similarity, bilingual synonyms, token + * overlap and (content-inferred) type compatibility, then assign globally by + * descending score so each column and each field is used at most once. `rows` + * is optional — without sample data only name-based signals fire (type gates + * are skipped). Returns one entry per column, in column order. + */ +export function suggestColumnMappings( + headers: string[], + fields: MappableField[], + rows?: string[][], +): ColumnSuggestion[] { + const inferred = headers.map((_, ci) => inferColumnType(rows ? rows.map((r) => r[ci]) : [])); + + type Pair = { ci: number; field: string; score: number; reason: MappingReason }; + const pairs: Pair[] = []; + headers.forEach((header, ci) => { + fields.forEach((field) => { + const { score, reason } = scorePair(header, field, inferred[ci]); + if (score > 0) pairs.push({ ci, field: field.name, score, reason }); + }); + }); + // Greedy global assignment: highest-scoring pairs win their column + field. + pairs.sort((a, b) => b.score - a.score); + const usedCol = new Set(); + const usedField = new Set(); + const chosen = new Map(); + for (const p of pairs) { + if (p.score < MIN_MATCH_SCORE || usedCol.has(p.ci) || usedField.has(p.field)) continue; + chosen.set(p.ci, p); + usedCol.add(p.ci); + usedField.add(p.field); + } + + return headers.map((_, ci) => { + const c = chosen.get(ci); + if (!c) return { columnIndex: ci, fieldName: null, score: 0, confidence: null, reason: 'none' as MappingReason, inferredType: inferred[ci] }; + return { columnIndex: ci, fieldName: c.field, score: c.score, confidence: scoreToConfidence(c.score), reason: c.reason, inferredType: inferred[ci] }; + }); +} diff --git a/packages/plugin-grid/src/importServerPath.test.ts b/packages/plugin-grid/src/importServerPath.test.ts new file mode 100644 index 000000000..7866977e0 --- /dev/null +++ b/packages/plugin-grid/src/importServerPath.test.ts @@ -0,0 +1,61 @@ +import { describe, it, expect } from 'vitest'; +import { __testables } from './ImportWizard'; + +const { isUnsupportedImport, buildFailedRowsCsv } = __testables; + +describe('isUnsupportedImport', () => { + it('matches the adapter UNSUPPORTED_OPERATION code', () => { + expect(isUnsupportedImport({ code: 'UNSUPPORTED_OPERATION' })).toBe(true); + }); + + it('matches the "does not support data.import" message', () => { + expect(isUnsupportedImport(new Error('The connected @objectstack/client does not support data.import().'))).toBe(true); + }); + + it('matches a "importRecords is not a function" message', () => { + expect(isUnsupportedImport(new Error('dataSource.importRecords is not a function'))).toBe(true); + }); + + it('does NOT match a genuine server/validation error', () => { + expect(isUnsupportedImport(new Error('Row 3: value out of range'))).toBe(false); + expect(isUnsupportedImport({ code: 'VALIDATION_ERROR' })).toBe(false); + expect(isUnsupportedImport(null)).toBe(false); + expect(isUnsupportedImport(undefined)).toBe(false); + }); +}); + +describe('buildFailedRowsCsv', () => { + const headers = ['Name', 'Owner', 'Amount']; + // Only columns 0 and 2 are mapped (Owner is dropped). + const mapping: Record = { 0: 'name', 2: 'amount' }; + const rows = [ + ['Acme', 'alice', '100'], // row 1 + ['Beta', 'bob', 'oops'], // row 2 (fails) + ['Gamma', 'carol', '300'], // row 3 + ]; + + it('emits only the mapped columns plus an _error column, keyed by 1-based row number', () => { + const errorsByRow = new Map([[2, 'amount: not a number']]); + const csv = buildFailedRowsCsv(headers, rows, mapping, errorsByRow); + const lines = csv.split('\n'); + expect(lines[0]).toBe('Name,Amount,_error'); + // Only the failed row 2 is exported, with its mapped columns (Owner dropped). + expect(lines).toHaveLength(2); + expect(lines[1]).toBe('Beta,oops,amount: not a number'); + }); + + it('escapes commas, quotes and newlines per RFC 4180', () => { + const csvRows = [['a,b', 'x', 'has "quote"']]; + const errorsByRow = new Map([[1, 'bad, value']]); + const csv = buildFailedRowsCsv(headers, csvRows, { 0: 'name', 2: 'note' }, errorsByRow); + const lines = csv.split('\n'); + expect(lines[1]).toBe('"a,b","has ""quote""","bad, value"'); + }); + + it('skips error rows whose source row is missing', () => { + const errorsByRow = new Map([[99, 'ghost row']]); + const csv = buildFailedRowsCsv(headers, rows, mapping, errorsByRow); + // header only — no data line for the non-existent row + expect(csv.split('\n')).toHaveLength(1); + }); +}); diff --git a/packages/plugin-grid/src/importTemplate.test.ts b/packages/plugin-grid/src/importTemplate.test.ts new file mode 100644 index 000000000..9a65aeb51 --- /dev/null +++ b/packages/plugin-grid/src/importTemplate.test.ts @@ -0,0 +1,46 @@ +import { describe, it, expect } from 'vitest'; +import { __testables } from './ImportWizard'; + +const { buildImportTemplateCsv, autoMapColumns } = __testables; + +describe('buildImportTemplateCsv', () => { + it('emits a header row of labels (required marked with *) plus one example row', () => { + const csv = buildImportTemplateCsv([ + { name: 'name', label: '客户名称', type: 'text', required: true }, + { name: 'email', label: 'Email', type: 'email' }, + { name: 'amount', label: 'Amount', type: 'number' }, + { name: 'due', label: 'Due', type: 'date' }, + { name: 'active', label: 'Active', type: 'boolean' }, + ]); + const [header, example] = csv.split('\n'); + expect(header).toBe('客户名称 *,Email,Amount,Due,Active'); + expect(example).toBe(',name@example.com,0,2024-01-31,true'); + }); + + it('seeds select example from the first option value, preferring value over label', () => { + const csv = buildImportTemplateCsv([ + { name: 'stage', label: 'Stage', type: 'select', options: [{ label: 'New Lead', value: 'new' }, { label: 'Won', value: 'won' }] }, + { name: 'tier', label: 'Tier', type: 'select', options: ['gold', 'silver'] }, + { name: 'empty', label: 'Empty', type: 'select' }, + ]); + const example = csv.split('\n')[1]; + expect(example).toBe('new,gold,'); + }); + + it('escapes labels containing commas per RFC 4180', () => { + const csv = buildImportTemplateCsv([ + { name: 'n', label: 'Name, Full', type: 'text', required: true }, + ]); + expect(csv.split('\n')[0]).toBe('"Name, Full *"'); + }); + + it('round-trips: a filled-in template (required * header) re-maps to the field', () => { + const fields = [ + { name: 'name', label: '客户名称', type: 'text', required: true }, + { name: 'email', label: 'Email', type: 'email' }, + ]; + const header = buildImportTemplateCsv(fields).split('\n')[0].split(','); + // The `*` required-marker must not defeat auto-mapping on re-import. + expect(autoMapColumns(header, fields)).toEqual({ 0: 'name', 1: 'email' }); + }); +}); diff --git a/packages/types/src/data.ts b/packages/types/src/data.ts index 49fadbfa4..1ce892993 100644 --- a/packages/types/src/data.ts +++ b/packages/types/src/data.ts @@ -551,6 +551,314 @@ export interface DataSource { resource: string, request: ExportDownloadRequest, ): Promise; + + /** + * Bulk-import rows into an object in a single server call. + * + * Callers send **raw** spreadsheet values (CSV text or JSON row objects) plus + * an optional `mapping` from source column → target field. The server coerces + * every cell to its storage value from the object's field metadata (booleans, + * numbers, dates→ISO, select label→code, lookup name→id), so the client does + * NOT pre-convert special values. `writeMode` selects insert / update / + * upsert (the latter two require `matchFields`); `dryRun` validates + previews + * without persisting. The result carries per-row outcomes for an import + * report + failed-row re-export. + * + * Optional — adapters without a server-side `/import` primitive may omit this + * (the wizard falls back to a per-row `create` loop). + * + * @param resource - Object/table name + * @param request - Import payload + options (see {@link ImportRequestOptions}) + * @returns Promise resolving to the aggregate + per-row import result + */ + importRecords?( + resource: string, + request: ImportRequestOptions, + ): Promise; + + /** + * Initiate an **asynchronous** import job — the large-file counterpart to + * {@link importRecords}. The whole payload is posted once; the server persists + * a job, returns immediately with a `jobId`, and processes rows in the + * background (up to its row ceiling, typically 50,000). Callers poll + * {@link getImportJobProgress} for live counters and + * {@link getImportJobResults} for the capped per-row report. + * + * Optional — adapters whose backend lacks async import jobs omit this (the + * wizard then keeps every file on the synchronous {@link importRecords} path). + * Feature-detect with `typeof dataSource.createImportJob === 'function'`. + * + * @param resource - Object/table name + * @param request - Same payload shape as {@link importRecords} + * @returns Promise resolving to job tracking info ({ jobId, status, total, … }) + */ + createImportJob?( + resource: string, + request: ImportRequestOptions, + ): Promise; + + /** + * Poll the progress of a previously-created import job. + * Optional — required only if {@link createImportJob} is implemented. + * + * @param jobId - The job identifier returned by {@link createImportJob}. + * @returns Promise resolving to current counters / terminal status. + */ + getImportJobProgress?(jobId: string): Promise; + + /** + * Fetch the per-row results of an import job (server-capped; failures first). + * Optional — required only if {@link createImportJob} is implemented. + * + * @param jobId - The job identifier. + * @returns Progress fields plus `results` and a `resultsTruncated` flag. + */ + getImportJobResults?(jobId: string): Promise; + + /** + * List recent import jobs (history), newest first. + * Optional — implementations without a history endpoint omit this. + * + * @param options - Optional filters (object, status) + pagination. + */ + listImportJobs?(options?: ListImportJobsOptions): Promise; + + /** + * Cancel a pending/running import job (cooperative — the worker stops at its + * next progress boundary). Optional; the UI hides Cancel when omitted. + * + * @param jobId - The job identifier to cancel. + */ + cancelImportJob?(jobId: string): Promise; + + /** + * Logically roll back a finished import job: delete the records it created + * and restore the records it updated to their pre-import field values. + * Optional — only jobs the server captured an undo log for are undoable + * (see {@link ImportJobProgressInfo.undoable}). The UI hides Undo when this + * is omitted or the job reports `undoable: false`. + * + * @param jobId - The job identifier to undo. + * @returns Counts of deleted / restored / failed reversal operations. + */ + undoImportJob?(jobId: string): Promise; +} + +/** + * How each incoming import row is committed against existing data. Mirrors the + * server's `ImportWriteMode` (`@objectstack/spec`). + * - `insert` — always create a new record (default; ignores `matchFields`) + * - `update` — update the record matched by `matchFields`; skip when none match + * - `upsert` — update when matched, else create + */ +export type ImportWriteMode = 'insert' | 'update' | 'upsert'; + +/** + * A single source-column → target-field mapping with optional per-column + * transform metadata. Mirrors the server's `FieldMappingEntry`. + */ +export interface ImportFieldMappingEntry { + sourceField: string; + targetField: string; + transform?: 'none' | 'uppercase' | 'lowercase' | 'trim' | 'date_format' | 'lookup'; + defaultValue?: unknown; + required?: boolean; +} + +/** + * Options + payload for {@link DataSource.importRecords}. Mirrors the server's + * `ImportRequest` (`POST /api/v1/data/:object/import`). + */ +export interface ImportRequestOptions { + /** Payload shape — inferred from `csv`/`rows` when omitted. */ + format?: 'csv' | 'json'; + /** CSV text (when `format = 'csv'`). */ + csv?: string; + /** Row objects (when `format = 'json'`). */ + rows?: Array>; + /** Source column → target field mapping (compact record or entry array). */ + mapping?: Record | ImportFieldMappingEntry[]; + /** Validate + coerce every row without persisting. @default false */ + dryRun?: boolean; + /** insert / update / upsert semantics. @default 'insert' */ + writeMode?: ImportWriteMode; + /** Fields that identify an existing record (required for update/upsert). */ + matchFields?: string[]; + /** Fire triggers/hooks for each imported row (off by default for bulk). */ + runAutomations?: boolean; + /** Trim leading/trailing whitespace from string cells. @default true */ + trimWhitespace?: boolean; + /** Strings treated as null/blank besides the empty string. */ + nullValues?: string[]; + /** Keep unmatched select values instead of failing the row. @default false */ + createMissingOptions?: boolean; + /** Skip rows whose `matchFields` are blank. @default false */ + skipBlankMatchKey?: boolean; +} + +/** + * Outcome of one imported row. Mirrors the server's `ImportRowResult`. + */ +export interface ImportRowResult { + /** 1-based row number in the source data. */ + row: number; + /** Whether the row succeeded. */ + ok: boolean; + /** What happened to the row. */ + action?: 'created' | 'updated' | 'skipped' | 'failed'; + /** Record id (created/updated rows). */ + id?: string; + /** Field that caused a coercion/validation error (failed rows). */ + field?: string; + /** Error code (failed rows). */ + code?: string; + /** Human-readable error message (failed rows). */ + error?: string; +} + +/** + * Aggregate summary + per-row results from {@link DataSource.importRecords}. + * Mirrors the server's `ImportResponse`. + */ +export interface ImportRecordsResult { + object: string; + dryRun: boolean; + writeMode: ImportWriteMode; + total: number; + ok: number; + errors: number; + created: number; + updated: number; + skipped: number; + results: ImportRowResult[]; +} + +/** + * Lifecycle status of an asynchronous import job. Mirrors the server's + * `ImportJobStatus` enum (`@objectstack/spec`). + */ +export type ImportJobStatus = + | 'pending' + | 'running' + | 'succeeded' + | 'failed' + | 'cancelled'; + +/** + * Result of {@link DataSource.createImportJob}. `jobId` is the polling key. + * Mirrors the server's `CreateImportJobResponse`. + */ +export interface CreateImportJobResult { + /** Server-assigned job identifier. */ + jobId: string; + /** Object the job imports into. */ + object: string; + /** Initial status (usually 'pending'). */ + status: ImportJobStatus; + /** Total rows accepted for processing. */ + total: number; + /** ISO-8601 creation timestamp. */ + createdAt?: string; +} + +/** + * Live progress of an import job, returned by + * {@link DataSource.getImportJobProgress}. Mirrors the server's + * `ImportJobProgress`. + */ +export interface ImportJobProgressInfo { + jobId: string; + object: string; + status: ImportJobStatus; + dryRun?: boolean; + writeMode?: ImportWriteMode; + /** Total rows in the job. */ + total: number; + /** Rows processed so far. */ + processed: number; + created: number; + updated: number; + skipped: number; + errors: number; + /** 0–100 completion. */ + percentComplete: number; + /** Whether this job can still be logically rolled back (see {@link DataSource.undoImportJob}). */ + undoable?: boolean; + /** ISO-8601 timestamp of when the job was undone / rolled back. */ + revertedAt?: string; + /** Failure detail when `status === 'failed'`. */ + error?: string; + /** ISO-8601 start timestamp. */ + startedAt?: string; + /** ISO-8601 completion timestamp. */ + completedAt?: string; + /** ISO-8601 creation timestamp. */ + createdAt?: string; +} + +/** + * Import-job progress plus the capped per-row report, returned by + * {@link DataSource.getImportJobResults}. Mirrors the server's + * `ImportJobResults`. + */ +export interface ImportJobResultsInfo extends ImportJobProgressInfo { + /** Per-row outcomes (server-capped; failures first). */ + results: ImportRowResult[]; + /** True when `results` omits rows because the cap was exceeded. */ + resultsTruncated: boolean; +} + +/** + * One row in the import-job history list, returned by + * {@link DataSource.listImportJobs}. Mirrors the server's `ImportJobSummary`. + */ +export interface ImportJobSummaryInfo { + jobId: string; + object: string; + status: ImportJobStatus; + total: number; + processed: number; + created: number; + updated: number; + skipped: number; + errors: number; + createdAt?: string; + completedAt?: string; + /** Whether this job can still be logically rolled back. */ + undoable?: boolean; + /** ISO-8601 timestamp of when the job was undone / rolled back. */ + revertedAt?: string; +} + +/** + * Outcome of {@link DataSource.undoImportJob} — a logical rollback. Mirrors the + * server's `UndoImportJobResponse`. + */ +export interface ImportJobUndoResult { + /** Whether the undo completed. */ + success: boolean; + jobId: string; + object: string; + /** Created records deleted. */ + deleted: number; + /** Updated records restored to their pre-import values. */ + restored: number; + /** Reversal operations that failed. */ + failed: number; +} + +/** + * Filters + pagination for {@link DataSource.listImportJobs}. + */ +export interface ListImportJobsOptions { + /** Only jobs importing into this object. */ + object?: string; + /** Only jobs in this status. */ + status?: ImportJobStatus; + /** Page size (server clamps; default 50). */ + limit?: number; + /** Offset for pagination. */ + offset?: number; } /** diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 53b53e548..90b0cf558 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -279,6 +279,18 @@ export type { CreateExportJobRequest, CreateExportJobResult, ExportJobProgressInfo, + ImportWriteMode, + ImportFieldMappingEntry, + ImportRequestOptions, + ImportRowResult, + ImportRecordsResult, + ImportJobStatus, + CreateImportJobResult, + ImportJobProgressInfo, + ImportJobResultsInfo, + ImportJobSummaryInfo, + ImportJobUndoResult, + ListImportJobsOptions, ExportDownloadRequest, } from './data'; diff --git a/playwright.import-console.config.ts b/playwright.import-console.config.ts new file mode 100644 index 000000000..11e4a36f6 --- /dev/null +++ b/playwright.import-console.config.ts @@ -0,0 +1,56 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * REAL-CONSOLE import-wizard E2E — the most faithful "how a user actually does + * it" test: it logs into the real `apps/console`, navigates to an object's list + * view (ObjectView), clicks the real toolbar Import button, and drives the + * wizard through a BACKGROUND import → History → Undo, asserting the backend + * record count before/after each step. No hand-mounted harness — the wizard is + * reached exactly the way the product surfaces it. + * + * This differs from: + * - playwright.live.config.ts — general live console specs + * - playwright.import-harness.config.ts — a standalone hand-mounted wizard + * + * Prereqs (long-lived dev processes — NOT started by this config): + * 1. An ObjectStack backend with the async import-job routes + * (`/api/v1/data/import/jobs`), e.g. the framework `data-import` build: + * objectstack serve --dev --port 3002 (from examples/app-crm) + * 2. The console dev server pointed at that backend AND at an + * import-job-capable `@objectstack/client`. The published client (11.2.0) + * predates the async import API, so the console must alias it via + * OBJECTSTACK_CLIENT_DIST (see apps/console/vite.config.ts): + * cd apps/console && VITE_SERVER_URL= VITE_USE_MOCK_SERVER=false \ + * DEV_PROXY_TARGET=http://localhost:3002 \ + * OBJECTSTACK_CLIENT_DIST=/packages/client \ + * pnpm dev --port 5180 --strictPort + * + * Because the flow only works once that unshipped client is wired in, the spec + * is gated on IMPORT_CONSOLE_LIVE=1 and additionally skips if the backend has + * no import-job route — so an unconfigured CI run reports skipped, not failed. + * + * Run: IMPORT_CONSOLE_LIVE=1 pnpm test:e2e:import-console + * IMPORT_CONSOLE_LIVE=1 pnpm test:e2e:import-console --headed + * + * Overrides: LIVE_APP_URL (console, default :5180), LIVE_API_URL (backend, + * default :3000 — set :3002 for app-crm), LIVE_EMAIL / LIVE_PASSWORD, + * LIVE_IMPORT_APP (default crm_app), LIVE_IMPORT_OBJECT (default crm_lead). + */ +const APP = process.env.LIVE_APP_URL || 'http://localhost:5180'; + +export default defineConfig({ + testDir: './e2e/import-console', + fullyParallel: false, + workers: 1, + retries: 0, + reporter: [['list']], + globalSetup: './e2e/live/global-setup.ts', + use: { + baseURL: APP, + storageState: 'e2e/live/.auth/state.json', + trace: 'on-first-retry', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, + projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], +}); diff --git a/playwright.import-harness.config.ts b/playwright.import-harness.config.ts new file mode 100644 index 000000000..d5651f62d --- /dev/null +++ b/playwright.import-harness.config.ts @@ -0,0 +1,45 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * LIVE import-wizard E2E — drives the REAL Import Wizard UI against a REAL + * ObjectStack backend, end to end (upload → background import → History → Undo), + * asserting record counts at the backend before and after each step. + * + * Unlike `playwright.live.config.ts` (which auths into the full console), this + * config targets the standalone import harness served by + * `packages/plugin-grid/demo/vite.live.config.ts`. That harness self-authenticates + * its adapter in-page, so there is no globalSetup / storageState here. + * + * Prereqs (long-lived dev processes — NOT started by this config): + * - An ObjectStack backend the harness proxies to (default app-crm on :3002). + * - The harness dev server: + * pnpm --filter @object-ui/plugin-grid exec \ + * vite --config demo/vite.live.config.ts + * serving http://localhost:5200/live.html + * + * The single spec SKIPS itself when the harness origin isn't reachable, so this + * config is safe to run in CI (where the harness isn't up) — it reports 0 run, + * 1 skipped rather than failing. + * + * Run: pnpm test:e2e:import-harness (headless) + * pnpm test:e2e:import-harness --headed (watch it drive the UI) + * + * Override the target via IMPORT_HARNESS_ORIGIN (default http://localhost:5200) + * and the object under test via IMPORT_HARNESS_OBJECT (default crm_lead). + */ +const ORIGIN = process.env.IMPORT_HARNESS_ORIGIN || 'http://localhost:5200'; + +export default defineConfig({ + testDir: './e2e/import-harness', + fullyParallel: false, + workers: 1, + retries: 0, + reporter: [['list']], + use: { + baseURL: ORIGIN, + trace: 'on-first-retry', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, + projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], +});