diff --git a/.claude/skills/writing-data-transfer-config/SKILL.md b/.claude/skills/writing-data-transfer-config/SKILL.md index e4424f20..19ed132d 100644 --- a/.claude/skills/writing-data-transfer-config/SKILL.md +++ b/.claude/skills/writing-data-transfer-config/SKILL.md @@ -182,6 +182,7 @@ Post-run inspection: `cat .transfer//logs/*.log | pino-pretty`. Default p ```ts tuning: { + flushEvery: numberFromEnv("FLUSH_EVERY", 500), // records per shard flush — bounds peak memory ddb: { maxRetries: 3, initialBackoffMs: 100 }, s3: { concurrency: 10, maxRetries: 3, initialBackoffMs: 100 }, os: { @@ -194,6 +195,8 @@ tuning: { All optional; absent = built-in defaults. AWS SDK `retryMode: "adaptive"` is always on for DDB + S3 — it self-tunes backoff based on real throttle signals, so you usually don't need to tune these. +**`flushEvery`** caps peak per-shard memory. The runner calls `processor.execute()` every N records and resets the pending-commands buffer. Default 500 (≈ 5 MB at a 10 KB average record). Lower to 100 for tables with very large records (approaching the 400 KB DDB max). + ## Running it From the user project root: diff --git a/.gitignore b/.gitignore index 068859d1..e61a28a4 100644 --- a/.gitignore +++ b/.gitignore @@ -26,11 +26,11 @@ logs.txt !.yarn/sdks !.yarn/versions .pnp.* -projects/**/.env -projects/v5-to-v6/models/ -# User project directories — not committed; v5-to-v6 is the committed example -projects/*/ +# Projects — only the two reference files below are committed; everything else is local +projects/** !projects/v5-to-v6/ +!projects/v5-to-v6/config.ts +!projects/v5-to-v6/.env.example CLAUDE.*.md # Ignore .claude/ except the two shipped skills — config/preset writing diff --git a/AGENTS.md b/AGENTS.md index fd4408fe..2f9a3f03 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,9 +14,10 @@ This document is read by AI agents when working on this codebase. It describes t **Runtime flow (when deployed):** -1. User writes a config file: `createDdbConfig({ source, target, pipeline })` or `createOsConfig(...)`. -2. CLI `transfer` command bootstraps a DI container, loads the named preset, spawns worker processes per segment. -3. Each worker runs one or more shards: scans source → for each record, first-match-wins pipeline runs: filters → transformers → each processor's `onEnd?` hook (sequential, array order) → commands accumulate in a shared shard buffer. At shard end, each processor's `execute()` drains its own keys from that buffer (sequential, array order). `Commands.unclaimedKeys()` surfaces commands no processor claimed. +1. User writes a single `config.ts`: `createConfig({ source, target, pipeline })`. One file covers DDB, S3, and optional OpenSearch. +2. CLI `transfer` command (no `--config`): the `TransferWizard` selects a project, writes `.env`, then on subsequent runs prompts for a preset and returns `WizardResult { configPath, preset }`. With `--config`: skips wizard, preset passed as `--preset` flag. +3. Bootstrap loads the config, registers all features (DDB + S3 always; OS conditional on `config.target.opensearch != null`), loads the named preset, spawns worker processes per segment. +4. Each worker runs one or more shards: scans source → for each record, first-match-wins pipeline runs: filters → transformers → each processor's `onEnd?` hook (sequential, array order) → commands accumulate in a pending buffer. Every `tuning.flushEvery` records (default 500) each processor's `execute()` drains its own keys from that buffer (sequential, array order) and the buffer resets — this bounds peak memory to `flushEvery × avg_record_size`. A final flush at shard end drains any remainder. `Commands.unclaimedKeys()` surfaces commands no processor claimed. **Read before big refactors:** @@ -29,7 +30,7 @@ This document is read by AI agents when working on this codebase. It describes t Everything users import lives in `src/index.ts`. The surface is **infrastructure-only** — no built-in transformers or pipelines are re-exported. The package ships two built-in presets: **`v5-to-v6-ddb`** (full DDB + S3 migration) and **`v5-to-v6-os`** (OpenSearch companion table migration). `PresetLoader` scans `src/presets/` (resolved relative to its own `import.meta.url`, works from source or `node_modules/`) — convention is **filename = preset name**, drop a `.ts` file in there and it ships, no other code change. The authoring reference lives in `templates/presets/example.ts` (scaffolded into user projects by `init`). -- **Config builders:** `createDdbConfig`, `createOsConfig` +- **Config builder:** `createConfig` — single unified builder; replaces the old `createDdbConfig` / `createOsConfig` (both deleted 2026-05-10). DDB + S3 always required; `source.opensearch` / `target.opensearch` optional. - **Env helpers:** `loadEnv` (dotenv loader), `fromEnv(name, default?)` (required string env, throws on missing), `numberFromEnv(name, default?)` (typed numeric, throws on parse failure). Empty string counts as missing in both — `.env`'s `KEY=` is almost always a forgotten value, not an intentional empty override. - **AWS credential helpers:** re-exports from `@aws-sdk/credential-providers` so users don't need the direct dep. `fromAwsProfile` (= `fromIni`) binds an explicit profile from `~/.aws/credentials` — best for local dev where a stray env var shouldn't hijack auth. `fromAwsCredentialChain` (= `fromNodeProviderChain`) runs the AWS SDK default chain (env → ini → SSO → EC2/ECS IAM) — best for CI / cloud. `credentials` in config also accepts a literal `{accessKeyId, secretAccessKey, sessionToken?}`; the union is schema-validated at `createDdbConfig` / `createOsConfig` time. - **Snapshot (debugging):** `config.debug.snapshot` (boolean or `{dir?, compress?}`) dumps per-record JSONL files at `//segment-.{source,post-transform,commands}.jsonl[.gz]` + `/dropped/segment-.jsonl[.gz]`. Default dir: `.transfer//snapshot`, gzipped. Opt-in, no-op when disabled — PipelineRunner depends on SnapshotWriter unconditionally so the hot path has no branching. @@ -50,6 +51,8 @@ Everything users import lives in `src/index.ts`. The surface is **infrastructure **Pipeline construction:** inside a preset's `configure({ runner, pipelineBuilderFactory, container })` callback, users call `pipelineBuilderFactory.create({ name, scanner, processors: [...] })`. `processors` is a `NonEmptyArray` — TS rejects empty arrays AND rejects processors whose slice keys collide (`DisjointKeys<...>`). Returns a typed `PipelineBuilder` whose `ctx` is `BaseTransformContext & (union of processor slices)`. Chain `.filter()` / `.use()` / `.beforeExecuteCommands()` / `.afterExecuteCommands()` in any order; `.build()` takes no arguments (terminal behavior comes from each processor's `onEnd?` hook). Pass the built pipeline to `runner.register(...pipelines)` (variadic, chainable, throws on duplicate name). The legacy `createPipeline` / `createDdbPipeline` / `createOsPipeline` factories were deleted on 2026-04-20; `runner.pipeline()` was moved to `PipelineBuilderFactory.create()` shortly after. +**Preset selection:** `pipeline.preset` was **removed** from the config schema on 2026-05-10. Preset is chosen at runtime — interactively by `TransferWizard` (returns `WizardResult { configPath: string; preset: string; dryRun: boolean }`), or passed as `--preset ` directly. Workers receive `--preset` on their CLI argv. Do NOT add `preset` back to `pipelineSettingsSchema`. `dryRun: true` causes `DdbProcessor`, `OsProcessor`, and `S3Processor` to skip their `execute()` bodies entirely — reads still happen, but nothing is written to the target. + **User-side custom DI — `setup.ts`:** CLI looks for `setup.ts` next to the user's config file. If present, dynamic-imports its default export and awaits `fn({ container })` BEFORE `preset.configure({...})` runs. Use the `initDataTransfer` typed helper to export it. Optional — pure-config users skip the file entirely. **Rule:** when adding something to `src/index.ts`, it must be something a user building their own transformers/pipelines/presets genuinely needs. Filter predicates and stable, widely-useful transformers (`copyFileToTarget`) are appropriate. Domain-specific migration transformers (CMS transformers, `createMetadata`, etc.) remain internal — they are tightly coupled to the v5→v6 schema and subject to change. The full transformer and filter catalogue lives in `.claude/skills/data-transfer-transformers/SKILL.md`. @@ -74,11 +77,14 @@ src/ │ │ ├── handler.ts # Unchanged — runs a resolved config path │ │ └── wizard/ # Guided .env setup + config selection │ │ ├── TransferWizard.ts # Orchestrator: project select → JSON extract → write .env -│ │ │ # OR (re-run, .env exists, no JSON) → config select → return path +│ │ │ # OR (re-run, .env exists, no JSON) → preset select → return WizardResult │ │ ├── projectDiscovery.ts # Scans projects/, returns sorted names -│ │ ├── configDiscovery.ts # Scans *.config.ts, imports each, reads storage field +│ │ ├── configDiscovery.ts # Finds config.ts in project dir; returns path or null +│ │ ├── presetDiscovery.ts # listAvailablePresets(presetsDir?) — names only (sync) +│ # listAvailablePresetsWithDescriptions(presetsDir?) — async, +│ # dynamically imports each preset to read .description │ │ ├── envWriter.ts # {{TOKEN}} substitution from .env.example → writes .env -│ │ ├── types.ts # RawOutputValues + EnvValues interfaces +│ │ ├── types.ts # RawOutputValues + EnvValues + WizardResult interfaces │ │ ├── sources/ │ │ │ ├── WebinyOutputSource.ts # Reads source/target.webiny.json → RawOutputValues │ │ │ └── PulumiStateSource.ts # Reads source/target.pulumi.json → RawOutputValues @@ -249,6 +255,7 @@ Optional `tuning` section on `MigrationConfig`: ```typescript tuning?: { + flushEvery?: number; // records per shard flush (default 500); bounds peak memory ddb?: { maxRetries?: number; initialBackoffMs?: number }; s3?: { concurrency?: number; maxRetries?: number; initialBackoffMs?: number }; os?: { maxRetries?: number; retryScheduleMs?: number[]; gzipConcurrency?: number }; @@ -257,6 +264,8 @@ tuning?: { Fields flow to the respective client/executor; absent = module-level defaults. `BATCH_SIZE = 25` in DDB is AWS-enforced, NOT a user knob. +`flushEvery` caps peak per-shard memory: at default 500 × 10 KB avg = ~5 MB/shard. For tables with very large records (approaching the 400 KB DDB max) lower this to 100. Set via `tuning: { flushEvery: numberFromEnv("FLUSH_EVERY", 500) }` in the config. + ### AWS retry + error classification All AWS-facing code shares one classifier: `src/base/isRetryableAwsError.ts` (duck-typed, no SDK import). Retry path per client: @@ -290,18 +299,25 @@ No custom token-bucket pacing — the AWS SDK's adaptive mode handles remote-sig Verification before any commit: ```bash -yarn format:fix # oxfmt -yarn ts-check # expect 0 errors -yarn test # expect all green -git status # include ALL modified files +yarn format:fix # oxfmt — must be clean before ts-check +yarn ts-check # expect 0 errors +yarn test:coverage # expect all green (use :coverage to keep thresholds enforced) +yarn lint # expect 0 errors +yarn check:imports # expect 0 errors +git status # include ALL modified files ``` +All five checks are required. Missing any one of them has broken CI in the past. + --- ## 6. Hard-won decisions (read before changing) These are one-line summaries. Each links to a spec or PR if fuller context is needed. +- **Periodic shard flush (`flushEvery`, 2026-05-11)** — `PipelineRunner.runShard` no longer buffers all commands for the entire shard. Every `tuning.flushEvery` records (default 500) `processor.execute()` is called and the buffer resets. A final flush drains the remainder. This bounds memory to `flushEvery × avg_record_size` regardless of table size. `afterShard` still fires exactly once per shard, after all flushes. Env var: `FLUSH_EVERY`. +- **One `createConfig`, no `createDdbConfig`/`createOsConfig`** (2026-05-10) — a single unified config covers all storage types. `source.opensearch` / `target.opensearch` are optional; omit or set to `null` to skip OpenSearch. Bootstrap registers DDB + S3 unconditionally; OS features only when `config.target.opensearch != null`. Storage guards (`storage !== "ddb"`) in `DdbScanner`, `DdbProcessor`, `S3Processor` were deleted. `OsScanner`/`OsProcessor` check `!config.source.opensearch` / `!config.target.opensearch`. Never reintroduce a `storage` discriminator field or per-storage config builders. +- **Preset selected at runtime, not in config** (2026-05-10) — `pipeline.preset` is gone from the schema. `TransferWizard` prompts the user and returns `WizardResult { configPath, preset }`. Workers receive `--preset ` on their argv. Preset name is never derived from config. Never add `preset` back to `pipelineSettingsSchema`. - **Zero transformers must work** — infra supports pure data-transfer (prod→dev seeding). `PipelineBuilder.build()` never throws for missing `.filter()`; if the pipeline includes a processor with `onEnd` (e.g. `DdbProcessor`), the terminal put fires via that hook for every matching record. - **Record carries everything** — processors + executors trust `ctx.record` at execute time; no side-channel queues or pre-transform snapshot passing. The OS refactor on 2026-04-19 made this explicit. - **`ctx.original` always present** — frozen pre-transform snapshot, on every context, permanently. Don't remove even if no built-in code consumes it. @@ -340,7 +356,7 @@ Built on top of `bruno/feat/di-features`. Adds: `v5-to-v6-os` built-in preset (` ### Broader open work 1. **npm publish story** — the package isn't on npm yet. Needs version strategy, publish script, CI. `npx @webiny/data-transfer init` in the README won't work until this lands. -2. **Init scaffolding smoke** — `init` scaffolds from `templates/`. All three scaffold files exist (`stampMigratedAt.ts`, `presets/example.ts`, `ddb.transfer.config.ts` + optional `setup.ts`). Do a smoke run to verify a scaffolded project compiles + runs against a live sandbox. +2. **Init scaffolding smoke** — `init` scaffolds from `templates/`. Scaffold output: `config.ts`, `presets/example.ts`, optional `setup.ts`. Do a smoke run to verify a scaffolded project compiles + runs against a live sandbox. 3. **End-to-end AWS smoke** — no test has ever run against real AWS. Day-long sandbox exercise. Catches real issues mocks can't. 4. **Public API audit pass (post-refactor)** — `src/index.ts` grew with `Processor`, `NonEmptyArray`, `InitDataTransferContext`, `BaseTransformContext`, `DdbTransformContext`, `OsTransformContext`, `initDataTransfer`. Re-audit before publish to confirm the surface matches user-authoring intent (e.g., should `DdbTransformContext` stay as-is or split into the narrower `BaseTransformContext & DdbProcessorSlice` for users who don't include S3Processor?). @@ -353,25 +369,26 @@ Built on top of `bruno/feat/di-features`. Adds: `v5-to-v6-os` built-in preset (` - Type-check: `yarn ts-check` - Test: `yarn test` (or `yarn test:coverage`) - Scaffold a standalone user project: `npx @webiny/data-transfer init my-transfer-folder` -- Add a project folder to this repo: `yarn transfer init-project ` — creates `projects//` with `ddb.transfer.config.ts`, `os.transfer.config.ts`, `.env.example`, `models/`, and `presets/` (with `presetsDir` pre-wired in both configs). Template lives in `templates/internal-project/`. New project folders are **gitignored** (`projects/*/` except `projects/v5-to-v6/`) — credentials stay local. -- **Guided setup (first-time use):** `yarn transfer` (no `--config`) launches `TransferWizard`. It selects the project, validates the Webiny output or Pulumi state JSON files the user drops into `projects//`, and writes the `.env` from `.env.example`. After writing the `.env` it exits — user reviews the file and runs `yarn transfer` again to run the transfer. On the second run (no JSON files, `.env` exists) the wizard skips to config selection. -- **Dry-run the preset against real AWS (dev use, from this repo):** +- Add a project folder to this repo: `yarn transfer init-project ` — creates `projects//` with `config.ts`, `.env.example`, `models/`, and `presets/`. Template lives in `templates/internal-project/`. New project folders are **gitignored** (`projects/*/` except `projects/v5-to-v6/`) — credentials stay local. +- **Guided setup (recommended):** `yarn transfer` (no `--config`) launches `TransferWizard`: + - Selects a project from `projects/`. + - If JSON output files are present (`source/target.webiny.json` or `.pulumi.json`): + - If `.env` also exists, asks whether to **repopulate** it from the JSON files or **use the existing** `.env`. + - If `.env` does not exist, extracts values and writes `.env`, then exits (user reviews and re-runs). + - Account IDs extracted from `primaryDynamodbTableArn` in the JSON files. If source and target account IDs differ, the wizard warns and advises setting `SOURCE_PROFILE` / `TARGET_PROFILE`. + - Preset selection: lists available presets with their one-line `description` in the prompt (`v5-to-v6-ddb — Full DDB migration`). User-supplied presets in `presetsDir` appear alongside built-ins. + - Dry-run prompt: after selecting a preset, the wizard asks if this is a dry run. Dry-run mode reads the source but skips all writes (`DdbProcessor`, `OsProcessor`, `S3Processor` skip `execute()`). Useful for validation passes. + - Returns `WizardResult { configPath, preset, dryRun }`. Workers receive `--preset ` and optionally `--dry-run`. +- **Direct run with config:** ```bash - # Option A — guided (recommended): - yarn transfer - # Wizard prompts for project, validates JSON files, writes .env, exits. - # After reviewing .env, run again: - yarn transfer - - # Option B — manual .env then direct config: - cp projects/v5-to-v6/.env.example projects/v5-to-v6/.env - # edit .env — set region, DDB/S3/OS tables, optional profiles - yarn transfer --config=./projects/v5-to-v6/ddb.transfer.config.ts # DDB + S3 first - yarn transfer --config=./projects/v5-to-v6/os.transfer.config.ts # OS table second + # After .env is written: + yarn transfer --config=./projects/v5-to-v6/config.ts --preset=v5-to-v6-ddb + # Then OpenSearch (if needed): + yarn transfer --config=./projects/v5-to-v6/config.ts --preset=v5-to-v6-os ``` - Run DDB transfer first, then OS — they don't share state. `.env*` is gitignored. The OS config additionally needs `SOURCE_OS_TABLE`, `TARGET_OS_TABLE`, `TARGET_OS_ENDPOINT`, and optionally `MODELS_DIR` (defaults to `./models`). + `.env*` is gitignored. One `config.ts` covers both DDB and OS runs — the preset determines which storage operations execute. - **JSON file formats for guided setup:** place in `projects//` before running `yarn transfer`: - `source.webiny.json` / `target.webiny.json` — output of `yarn webiny output core --json` run in the source/target Webiny project. diff --git a/README.md b/README.md index c0814703..9b1a51ea 100644 --- a/README.md +++ b/README.md @@ -8,30 +8,34 @@ A generic data-transfer tool for Webiny environments. Copies DynamoDB + S3 (or O - **Prod → dev seeding** — zero transformers, just copy. - **Custom transfers** — write your own transformers + pipelines + preset for bespoke data moves. -The package ships two built-in presets (`v5-to-v6-ddb`, `v5-to-v6-os`) plus full authoring support for your own. +The package ships four built-in presets (`v5-to-v6-ddb`, `v5-to-v6-os`, `copy-ddb`, `copy-files`) plus full authoring support for your own. ## Quick start ```bash -git clone git@github.com:webiny/v5-to-v6.git -cd v5-to-v6 +git clone git@github.com:webiny/data-transfer.git +cd data-transfer yarn install -yarn transfer init-project my-transfer -# then run the guided setup: yarn transfer ``` -`yarn transfer` (no `--config`) launches the **guided setup wizard**. It walks you through selecting your project, collecting your Webiny output or Pulumi state JSON files, and automatically writing your `.env`. After writing the `.env` it exits — review the file and run `yarn transfer` again to start the transfer. +`yarn transfer` (no `--config`) launches the **guided setup wizard**. It walks you through: -To scaffold a new project folder: +1. Selecting (or creating) a project folder under `projects/` +2. Collecting your Webiny output or Pulumi state JSON files and writing `.env` +3. Selecting a preset and optional dry-run mode, then starting the transfer -```bash -yarn transfer init-project -# e.g. -yarn transfer init-project my-client-prod -``` +**First run (no `.env` yet):** the wizard extracts values from your JSON files, writes `.env`, and exits so you can review it before anything runs. Run `yarn transfer` again to continue. + +**Subsequent runs (`.env` exists, no JSON files):** the wizard skips env setup entirely and goes straight to preset selection. + +**`.env` exists AND JSON files present:** the wizard asks whether to repopulate `.env` from the JSON files or keep the existing values. Choose "repopulate" to refresh after deploying a new environment; choose "use existing" to skip to preset selection. + +**Account ID warning:** the wizard extracts the AWS account ID from `primaryDynamodbTableArn` in the JSON files. If source and target accounts differ, it warns you to set `SOURCE_PROFILE` and `TARGET_PROFILE` in `.env` so the right credentials are used for each side. -This creates `projects//` with `ddb.transfer.config.ts`, `os.transfer.config.ts`, `README.md`, `.env.example`, `models/`, and `presets/` already wired up. +**Preset selection:** each preset is listed with its one-line description (`v5-to-v6-ddb — Full DDB migration`). User-supplied presets in `presetsDir` appear alongside built-ins. + +**Dry-run mode:** after selecting a preset the wizard asks "Dry run?" (default: No). In dry-run mode the tool scans and transforms records normally but skips all writes to the target (DynamoDB, S3, OpenSearch). Useful for validating your pipeline and transformer chain before committing a full transfer. New project folders are **gitignored** by default — credentials and env files stay local. Only `projects/v5-to-v6/` is committed as the reference example. @@ -58,59 +62,16 @@ cp /path/to/target-project/state.json projects//target.pulumi.json Mixed formats are allowed (e.g. `source.webiny.json` + `target.pulumi.json`). -## Storage modes - -The config builder determines which AWS storage the transfer reads from and writes to: - -- **`createDdbConfig(...)`** — DynamoDB primary table (+ S3 files). Handles all record types: CMS entries + models, security, file manager, folder permissions, mailer settings. -- **`createOsConfig(...)`** — OpenSearch companion DynamoDB table. Reads gzipped records, unzips, transforms, zips, writes to target OS DDB table. - -Run DDB transfer first, then OS transfer with a separate config file. They don't share state. +**CMS model exports (optional):** drop your exported model definitions into `projects//models/`. Export them from the Webiny Admin CMS → Models → Export, then copy the file there. See [`modelsDir`](#modelsdir) for accepted formats. ## Config reference -### DDB config - -```typescript -import { - loadEnv, - createDdbConfig, - fromAwsProfile, - fromEnv, - numberFromEnv -} from "@webiny/data-transfer"; - -loadEnv(import.meta.url); - -export default createDdbConfig({ - source: { - region: fromEnv("SOURCE_REGION", "us-east-1"), - credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", "default") }), - dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, - s3: { bucket: fromEnv("SOURCE_S3_BUCKET") } - }, - target: { - region: fromEnv("TARGET_REGION", "us-east-1"), - credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", "default") }), - dynamodb: { tableName: fromEnv("TARGET_DDB_TABLE") }, - s3: { bucket: fromEnv("TARGET_S3_BUCKET") } - }, - pipeline: { - preset: "./presets/my-preset.ts", - segments: numberFromEnv("SEGMENTS", 4), - modelsDir: "./models" // optional - } -}); -``` - -`loadEnv(import.meta.url)` loads the `.env` file sitting next to this config file. Each project folder should have its own `.env` so credentials stay isolated between projects. - -### OS config +One `config.ts` file covers all storage types. DynamoDB and S3 are required; OpenSearch is optional — omit or set to `null` if your environment doesn't use it. The preset you select at runtime determines which storage operations actually run. ```typescript import { loadEnv, - createOsConfig, + createConfig, fromAwsProfile, fromEnv, numberFromEnv @@ -118,16 +79,23 @@ import { loadEnv(import.meta.url); -export default createOsConfig({ +export default createConfig({ source: { - region: fromEnv("SOURCE_REGION", "us-east-1"), + region: fromEnv("SOURCE_REGION", "eu-central-1"), credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", "default") }), dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, + s3: { bucket: fromEnv("SOURCE_S3_BUCKET") }, + // Remove or set to null if your source has no OpenSearch: opensearch: { tableName: fromEnv("SOURCE_OS_TABLE") } }, target: { - region: fromEnv("TARGET_REGION", "us-east-1"), + region: fromEnv("TARGET_REGION", "eu-central-1"), credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", "default") }), + dynamodb: { tableName: fromEnv("TARGET_DDB_TABLE") }, + s3: { bucket: fromEnv("TARGET_S3_BUCKET") }, + // Set tableName to null or omit the block to skip the audit log: + auditLog: { dynamodb: { tableName: fromEnv("TARGET_AUDIT_LOGS_TABLE") } }, + // Remove or set to null if your target has no OpenSearch: opensearch: { endpoint: fromEnv("TARGET_OS_ENDPOINT"), tableName: fromEnv("TARGET_OS_TABLE"), @@ -136,14 +104,17 @@ export default createOsConfig({ } }, pipeline: { - preset: "v5-to-v6-os", segments: numberFromEnv("SEGMENTS", 4), - modelsDir: fromEnv("MODELS_DIR", "./models") + modelsDir: fromEnv("MODELS_DIR", "./models"), + // Optional: point at your own preset files (alongside built-ins): + presetsDir: "./presets" } }); ``` -**Index management** (OS mode): the tool disables `refresh_interval` just-in-time when it first writes to each index, and restores the original value after the transfer completes. Missing indexes are created with the Webiny base mapping. Only touched indexes are affected. +`loadEnv(import.meta.url)` loads the `.env` file sitting next to this config file. Each project folder should have its own `.env` so credentials stay isolated between projects. + +**Index management** (OpenSearch): the tool disables `refresh_interval` just-in-time when it first writes to each index, and restores the original value after the transfer completes. Missing indexes are created with the Webiny base mapping. Only touched indexes are affected. ### Env helpers @@ -186,6 +157,7 @@ JSON models override DB-loaded models when both exist. ```typescript tuning: { + flushEvery: numberFromEnv("FLUSH_EVERY", 500), // records per shard flush — bounds peak memory ddb: { maxRetries: 3, initialBackoffMs: 100 }, s3: { concurrency: 10, maxRetries: 3, initialBackoffMs: 100 }, os: { maxRetries: 3, retryScheduleMs: [5000, 10000, 20000], gzipConcurrency: 16 } @@ -194,6 +166,8 @@ tuning: { All fields are optional; absent = built-in defaults. `BATCH_SIZE` for DynamoDB is NOT tunable (AWS enforces 25 items per `BatchWriteItem`). DDB and S3 clients run in AWS SDK `adaptive` retry mode — `tuning.{ddb,s3}.maxRetries` caps the outer retry envelope on top of the SDK's own self-tuning backoff. +**`tuning.flushEvery`** controls how often accumulated write commands are flushed during a shard scan. The runner calls `processor.execute()` every N records and resets the buffer, so peak memory stays at `flushEvery × avg_record_size` regardless of table size. Default 500 (≈ 5 MB at a 10 KB average). Lower to 100 for tables with very large records. + ### Debug options Add a `debug` block to your config to opt into diagnostics: @@ -269,7 +243,7 @@ export default createTransferPreset({ }); ``` -Point `config.pipeline.preset` at the file path (relative to the config): `"./presets/my-preset.ts"`. Or use a built-in name like `"v5-to-v6-ddb"`. +Drop the file in your `projects//presets/` directory. The wizard will offer it by name alongside built-ins. ### `pipelineBuilderFactory.create({ name, scanner, processors })` @@ -380,12 +354,14 @@ export default createTransferPreset({ ### Built-in presets -Pass by name in `config.pipeline.preset`: +Select by name when the wizard asks "Which preset do you want to run?": - **`"v5-to-v6-ddb"`** — full Webiny v5 → v6 migration of the primary DynamoDB table (CMS entries, file manager, security, mailer, folder permissions, etc.). - **`"v5-to-v6-os"`** — migration of the OpenSearch companion DynamoDB table. Run **after** `v5-to-v6-ddb`. +- **`"copy-ddb"`** — verbatim DynamoDB-only copy (no transformations). +- **`"copy-files"`** — verbatim DynamoDB + S3 file copy. -Custom presets are path-resolved from your config file's directory. +Custom presets placed in your `presetsDir` are listed alongside built-ins. --- @@ -552,6 +528,7 @@ The CLI picks it up automatically and runs it **before** loading your preset, so ## Troubleshooting +- **Out-of-memory on large tables** — each worker buffers write commands between flushes. Reduce `tuning.flushEvery` (default 500) to a smaller value (e.g. `FLUSH_EVERY=100`) so each flush covers fewer records and peak memory stays manageable. - **AWS throttling** — the SDK self-tunes via `retryMode: "adaptive"`. If you still hit the outer cap, bump `tuning.ddb.maxRetries` / `tuning.s3.maxRetries`; lower `tuning.s3.concurrency` for S3-heavy transfers. - **OS indexes not creating** — the transfer aborts if index prep exhausts retries. Tune `tuning.os.maxRetries` and `tuning.os.retryScheduleMs`, or fix the underlying mapping error surfaced in the logs. - **Missing env vars** — run `yarn transfer` (no `--config`) to launch the guided setup wizard, which writes your `.env` automatically. Or copy `.env.example` manually and fill it in. Config files use `loadEnv(import.meta.url)` to load the sibling `.env`. diff --git a/__tests__/base/BaseError.test.ts b/__tests__/base/BaseError.test.ts new file mode 100644 index 00000000..09217464 --- /dev/null +++ b/__tests__/base/BaseError.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect } from "vitest"; +import { BaseError } from "~/base/BaseError.ts"; + +class TestError extends BaseError<{ field: string }> { + public readonly code = "TEST_ERROR"; + + public constructor(message: string, field: string) { + super({ message, data: { field } }); + } +} + +class VoidTestError extends BaseError { + public readonly code = "VOID_ERROR"; + + public constructor(message: string, stack?: string) { + super({ message }, { stack }); + } +} + +describe("BaseError", () => { + it("sets message from input", () => { + const err = new TestError("something broke", "name"); + expect(err.message).toBe("something broke"); + }); + + it("sets data from input", () => { + const err = new TestError("oops", "email"); + expect(err.data).toEqual({ field: "email" }); + }); + + it("exposes code on the instance", () => { + const err = new TestError("x", "y"); + expect(err.code).toBe("TEST_ERROR"); + }); + + it("is an instance of Error", () => { + const err = new TestError("x", "y"); + expect(err).toBeInstanceOf(Error); + }); + + it("stores custom stack when provided", () => { + const err = new VoidTestError("test", "custom stack"); + expect(err.stack).toBe("custom stack"); + }); + + it("data is undefined for void data type", () => { + const err = new VoidTestError("no data"); + expect(err.data).toBeUndefined(); + }); +}); diff --git a/__tests__/base/Result.test.ts b/__tests__/base/Result.test.ts new file mode 100644 index 00000000..b7d896da --- /dev/null +++ b/__tests__/base/Result.test.ts @@ -0,0 +1,98 @@ +import { describe, it, expect, vi } from "vitest"; +import { Result } from "~/base/Result.ts"; + +describe("Result", () => { + describe("ok", () => { + it("creates a successful result with a value", () => { + const r = Result.ok(42); + expect(r.isOk()).toBe(true); + expect(r.isFail()).toBe(false); + expect(r.value).toBe(42); + }); + + it("creates a successful result with no value", () => { + const r = Result.ok(); + expect(r.isOk()).toBe(true); + expect(r.value).toBeUndefined(); + }); + }); + + describe("fail", () => { + it("creates a failed result with an error", () => { + const r = Result.fail("oops"); + expect(r.isFail()).toBe(true); + expect(r.isOk()).toBe(false); + expect(r.error).toBe("oops"); + }); + }); + + describe("value getter", () => { + it("throws when accessed on a failed result", () => { + const r = Result.fail("err"); + expect(() => r.value).toThrow("Tried to get value from a failed Result."); + }); + }); + + describe("error getter", () => { + it("throws when accessed on a successful result", () => { + const r = Result.ok(1); + expect(() => r.error).toThrow("Tried to get error from a successful Result."); + }); + }); + + describe("map", () => { + it("transforms the value on success", () => { + const r = Result.ok(2).map(v => v * 3); + expect(r.value).toBe(6); + }); + + it("passes through the error on failure", () => { + const r = Result.fail("e").map((v: never) => v); + expect(r.error).toBe("e"); + }); + }); + + describe("mapError", () => { + it("transforms the error on failure", () => { + const r = Result.fail("raw").mapError(e => `wrapped:${e}`); + expect(r.error).toBe("wrapped:raw"); + }); + + it("passes through the value on success", () => { + const r = Result.ok(7).mapError(() => "x"); + expect(r.value).toBe(7); + }); + }); + + describe("flatMap", () => { + it("chains a new Result on success", () => { + const r = Result.ok(5).flatMap(v => Result.ok(v + 1)); + expect(r.value).toBe(6); + }); + + it("short-circuits on failure", () => { + const fn = vi.fn(); + const r = Result.fail("e").flatMap(fn); + expect(fn).not.toHaveBeenCalled(); + expect(r.error).toBe("e"); + }); + }); + + describe("match", () => { + it("calls ok handler on success", () => { + const out = Result.ok("hi").match({ + ok: v => `ok:${v}`, + fail: () => "fail" + }); + expect(out).toBe("ok:hi"); + }); + + it("calls fail handler on failure", () => { + const out = Result.fail("boom").match({ + ok: () => "ok", + fail: e => `fail:${e}` + }); + expect(out).toBe("fail:boom"); + }); + }); +}); diff --git a/__tests__/base/ResultAsync.test.ts b/__tests__/base/ResultAsync.test.ts new file mode 100644 index 00000000..ac22de89 --- /dev/null +++ b/__tests__/base/ResultAsync.test.ts @@ -0,0 +1,102 @@ +import { describe, it, expect } from "vitest"; +import { ResultAsync } from "~/base/ResultAsync.ts"; +import { Result } from "~/base/Result.ts"; + +describe("ResultAsync", () => { + describe("from", () => { + it("wraps a Promise", async () => { + const r = ResultAsync.from(() => Promise.resolve(Result.ok(1))); + const result = await r.unwrap(); + expect(result.value).toBe(1); + }); + }); + + describe("ok", () => { + it("wraps a successful value", async () => { + const result = await ResultAsync.ok(42).unwrap(); + expect(result.isOk()).toBe(true); + expect(result.value).toBe(42); + }); + }); + + describe("fail", () => { + it("wraps a failure", async () => { + const result = await ResultAsync.fail("oops").unwrap(); + expect(result.isFail()).toBe(true); + expect(result.error).toBe("oops"); + }); + }); + + describe("mapAsync", () => { + it("transforms the value on success", async () => { + const result = await ResultAsync.ok(3) + .mapAsync(v => v * 2) + .unwrap(); + expect(result.value).toBe(6); + }); + + it("supports async transformation", async () => { + const result = await ResultAsync.ok(3) + .mapAsync(async v => v + 1) + .unwrap(); + expect(result.value).toBe(4); + }); + + it("passes through error on failure", async () => { + const result = await ResultAsync.fail("e") + .mapAsync((v: never) => v) + .unwrap(); + expect(result.error).toBe("e"); + }); + }); + + describe("mapErrorAsync", () => { + it("transforms the error on failure", async () => { + const result = await ResultAsync.fail("raw") + .mapErrorAsync(e => `wrapped:${e}`) + .unwrap(); + expect(result.error).toBe("wrapped:raw"); + }); + + it("passes through value on success", async () => { + const result = await ResultAsync.ok(7) + .mapErrorAsync(() => "x") + .unwrap(); + expect(result.value).toBe(7); + }); + }); + + describe("flatMapAsync", () => { + it("chains on success", async () => { + const result = await ResultAsync.ok(5) + .flatMapAsync(v => ResultAsync.ok(v + 1)) + .unwrap(); + expect(result.value).toBe(6); + }); + + it("short-circuits on failure", async () => { + const result = await ResultAsync.fail("e") + .flatMapAsync(() => ResultAsync.ok(99)) + .unwrap(); + expect(result.error).toBe("e"); + }); + }); + + describe("match", () => { + it("calls ok handler on success", async () => { + const out = await ResultAsync.ok("hi").match({ + ok: v => `ok:${v}`, + fail: () => "fail" + }); + expect(out).toBe("ok:hi"); + }); + + it("calls fail handler on failure", async () => { + const out = await ResultAsync.fail("boom").match({ + ok: () => "ok", + fail: e => `fail:${e}` + }); + expect(out).toBe("fail:boom"); + }); + }); +}); diff --git a/__tests__/base/formatError.test.ts b/__tests__/base/formatError.test.ts index 419f475c..c4388dc5 100644 --- a/__tests__/base/formatError.test.ts +++ b/__tests__/base/formatError.test.ts @@ -61,4 +61,9 @@ describe("formatError", () => { expect(formatError(42)).toBe("42"); expect(formatError({ arbitrary: "object" })).toBe("[object Object]"); }); + + it("returns generic message when ZodError has no issues", () => { + const zodLike = { name: "ZodError", issues: [] }; + expect(formatError(zodLike)).toContain("Validation failed (no details)."); + }); }); diff --git a/__tests__/bootstrap.test.ts b/__tests__/bootstrap.test.ts index 8b1670af..39887c2c 100644 --- a/__tests__/bootstrap.test.ts +++ b/__tests__/bootstrap.test.ts @@ -16,113 +16,92 @@ import { DirectoryTool } from "../src/tools/DirectoryTool/index.ts"; import { FileTool } from "../src/tools/FileTool/index.ts"; import { OpenSearchClient } from "../src/services/OpenSearchClient/index.ts"; -describe("bootstrap", () => { - const ddbConfig: MigrationConfig.Interface = { - storage: "ddb", - source: { - region: "us-east-1", - credentials: { accessKeyId: "test", secretAccessKey: "test" }, - dynamodb: { tableName: "source-table" }, - s3: { bucket: "source-bucket" } - }, - target: { - region: "eu-central-1", - credentials: { accessKeyId: "test", secretAccessKey: "test" }, - dynamodb: { tableName: "target-table" }, - s3: { bucket: "target-bucket" }, - auditLog: null - }, - pipeline: { preset: "v5-to-v6" } - }; +const creds = { accessKeyId: "test", secretAccessKey: "test" }; - const osConfig: MigrationConfig.Interface = { - storage: "os", - source: { - region: "us-east-1", - credentials: { accessKeyId: "test", secretAccessKey: "test" }, - dynamodb: { tableName: "source-primary" }, - opensearch: { tableName: "source-os" } - }, - target: { - region: "eu-central-1", - credentials: { accessKeyId: "test", secretAccessKey: "test" }, - opensearch: { - endpoint: "https://es.example.com", - tableName: "target-os", - service: "opensearch" as const, - indexPrefix: "" - } - }, - pipeline: { preset: "v5-to-v6-os" } - }; +const ddbOnlyConfig: MigrationConfig.Interface = { + source: { + region: "us-east-1", + credentials: creds, + dynamodb: { tableName: "source-table" }, + s3: { bucket: "source-bucket" } + }, + target: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "target-table" }, + s3: { bucket: "target-bucket" }, + auditLog: null + }, + pipeline: {} +}; - describe("ddb mode", () => { - it("should resolve all core features", () => { - const container = bootstrap({ config: ddbConfig }); +const fullConfig: MigrationConfig.Interface = { + source: { + region: "us-east-1", + credentials: creds, + dynamodb: { tableName: "source-primary" }, + s3: { bucket: "source-bucket" }, + opensearch: { tableName: "source-os" } + }, + target: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "target-table" }, + s3: { bucket: "target-bucket" }, + opensearch: { + endpoint: "https://es.example.com", + tableName: "target-os", + service: "opensearch" as const, + indexPrefix: "" + } + }, + pipeline: {} +}; - expect(container.resolve(MigrationConfig)).toBeDefined(); - expect(container.resolve(Logger)).toBeDefined(); - expect(container.resolve(Cache)).toBeDefined(); - expect(container.resolve(DirectoryTool)).toBeDefined(); - expect(container.resolve(FileTool)).toBeDefined(); - expect(container.resolve(SourceDynamoDbClient)).toBeDefined(); - expect(container.resolve(TargetDynamoDbClient)).toBeDefined(); - expect(container.resolve(ModelProvider)).toBeDefined(); - expect(container.resolve(TenantLocales)).toBeDefined(); - expect(container.resolve(SourceS3Client)).toBeDefined(); - expect(container.resolve(TargetS3Client)).toBeDefined(); - expect(container.resolve(PresetLoader)).toBeDefined(); - expect(container.resolve(WorkerSpawner)).toBeDefined(); - }); - - it("should not register OpenSearchClient in ddb mode", () => { - const container = bootstrap({ config: ddbConfig }); - - expect(() => container.resolve(OpenSearchClient)).toThrow(); - }); - - it("should use provided log level", () => { - const container = bootstrap({ config: ddbConfig, logLevel: "debug" }); - const logger = container.resolve(Logger); - expect(logger).toBeDefined(); - }); +describe("bootstrap — DDB-only config", () => { + it("resolves all core features", () => { + const container = bootstrap({ config: ddbOnlyConfig }); + expect(container.resolve(MigrationConfig)).toBeDefined(); + expect(container.resolve(Logger)).toBeDefined(); + expect(container.resolve(Cache)).toBeDefined(); + expect(container.resolve(DirectoryTool)).toBeDefined(); + expect(container.resolve(FileTool)).toBeDefined(); + expect(container.resolve(SourceDynamoDbClient)).toBeDefined(); + expect(container.resolve(TargetDynamoDbClient)).toBeDefined(); + expect(container.resolve(SourceS3Client)).toBeDefined(); + expect(container.resolve(TargetS3Client)).toBeDefined(); + expect(container.resolve(ModelProvider)).toBeDefined(); + expect(container.resolve(TenantLocales)).toBeDefined(); + expect(container.resolve(PresetLoader)).toBeDefined(); + expect(container.resolve(WorkerSpawner)).toBeDefined(); }); - describe("ddb mode - exclusions", () => { - it("should not register S3Client in os mode", () => { - const container = bootstrap({ config: osConfig }); - expect(() => container.resolve(SourceS3Client)).toThrow(); - expect(() => container.resolve(TargetS3Client)).toThrow(); - }); + it("does NOT register OpenSearchClient when opensearch is absent", () => { + const container = bootstrap({ config: ddbOnlyConfig }); + expect(() => container.resolve(OpenSearchClient)).toThrow(); }); +}); - describe("os mode", () => { - it("should resolve all core features including OpenSearchClient", () => { - const container = bootstrap({ config: osConfig }); - - expect(container.resolve(MigrationConfig)).toBeDefined(); - expect(container.resolve(Logger)).toBeDefined(); - expect(container.resolve(Cache)).toBeDefined(); - expect(container.resolve(SourceDynamoDbClient)).toBeDefined(); - expect(container.resolve(TargetDynamoDbClient)).toBeDefined(); - expect(container.resolve(ModelProvider)).toBeDefined(); - expect(container.resolve(TenantLocales)).toBeDefined(); - expect(container.resolve(OpenSearchClient)).toBeDefined(); - }); +describe("bootstrap — full config (DDB + OS)", () => { + it("resolves OpenSearchClient when target.opensearch is set", () => { + const container = bootstrap({ config: fullConfig }); + expect(container.resolve(OpenSearchClient)).toBeDefined(); }); - describe("singleton behavior", () => { - it("should return same instances on multiple resolves", () => { - const container = bootstrap({ config: ddbConfig }); + it("also resolves S3 clients in full config", () => { + const container = bootstrap({ config: fullConfig }); + expect(container.resolve(SourceS3Client)).toBeDefined(); + expect(container.resolve(TargetS3Client)).toBeDefined(); + }); +}); - expect(container.resolve(Logger)).toBe(container.resolve(Logger)); - expect(container.resolve(Cache)).toBe(container.resolve(Cache)); - expect(container.resolve(SourceDynamoDbClient)).toBe( - container.resolve(SourceDynamoDbClient) - ); - expect(container.resolve(TargetDynamoDbClient)).toBe( - container.resolve(TargetDynamoDbClient) - ); - }); +describe("bootstrap — singleton behavior", () => { + it("returns same instance on multiple resolves", () => { + const container = bootstrap({ config: ddbOnlyConfig }); + expect(container.resolve(Logger)).toBe(container.resolve(Logger)); + expect(container.resolve(Cache)).toBe(container.resolve(Cache)); + expect(container.resolve(SourceDynamoDbClient)).toBe( + container.resolve(SourceDynamoDbClient) + ); }); }); diff --git a/__tests__/commands/processSegment.test.ts b/__tests__/commands/processSegment.test.ts index 8dbc84fe..4e794c19 100644 --- a/__tests__/commands/processSegment.test.ts +++ b/__tests__/commands/processSegment.test.ts @@ -12,7 +12,21 @@ const resolveMap = new Map(); const registerInstanceSpy = vi.fn(); vi.mock("~/features/MigrationConfig/loadConfig.ts", () => ({ - loadConfig: vi.fn(async (_path: string) => ({ storage: "ddb", pipeline: { preset: "x" } })) + loadConfig: vi.fn(async (_path: string) => ({ + source: { + region: "us-east-1", + credentials: { accessKeyId: "t", secretAccessKey: "t" }, + dynamodb: { tableName: "src" }, + s3: { bucket: "src-b" } + }, + target: { + region: "us-east-1", + credentials: { accessKeyId: "t", secretAccessKey: "t" }, + dynamodb: { tableName: "tgt" }, + s3: { bucket: "tgt-b" } + }, + pipeline: {} + })) })); vi.mock("~/bootstrap.ts", () => ({ bootstrap: vi.fn(() => ({ @@ -60,14 +74,26 @@ describe("processSegment handler", () => { }); it("loads preset, configures runner, calls run({segment, totalSegments})", async () => { - await handler({ runId: "r1", segment: 2, total: 4, config: "./x.ts" }); + await handler({ + runId: "r1", + segment: 2, + total: 4, + config: "./x.ts", + preset: "test-preset" + }); - expect(loadSpy).toHaveBeenCalledWith("x"); + expect(loadSpy).toHaveBeenCalledWith("test-preset"); expect(runSpy).toHaveBeenCalledWith({ segment: 2, totalSegments: 4 }); }); it("registers TransferContext with the provided runId", async () => { - await handler({ runId: "r-xyz", segment: 0, total: 1, config: "./x.ts" }); + await handler({ + runId: "r-xyz", + segment: 0, + total: 1, + config: "./x.ts", + preset: "test-preset" + }); expect(registerInstanceSpy).toHaveBeenCalledWith( expect.anything(), @@ -78,7 +104,13 @@ describe("processSegment handler", () => { it("calls process.exit(1) on runner failure", async () => { const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); runSpy.mockRejectedValueOnce(new Error("boom")); - await handler({ runId: "r1", segment: 0, total: 1, config: "./x.ts" }); + await handler({ + runId: "r1", + segment: 0, + total: 1, + config: "./x.ts", + preset: "test-preset" + }); expect(exitSpy).toHaveBeenCalledWith(1); exitSpy.mockRestore(); }); diff --git a/__tests__/commands/run/wizard/TransferWizard.test.ts b/__tests__/commands/run/wizard/TransferWizard.test.ts index 501e7027..95a63e85 100644 --- a/__tests__/commands/run/wizard/TransferWizard.test.ts +++ b/__tests__/commands/run/wizard/TransferWizard.test.ts @@ -5,6 +5,7 @@ import type { RawOutputValues } from "../../../../src/commands/run/wizard/types. vi.mock("../../../../src/commands/run/wizard/projectDiscovery.ts"); vi.mock("../../../../src/commands/run/wizard/configDiscovery.ts"); +vi.mock("../../../../src/commands/run/wizard/presetDiscovery.ts"); vi.mock("../../../../src/commands/run/wizard/envWriter.ts"); vi.mock("../../../../src/commands/run/wizard/sources/WebinyOutputSource.ts"); vi.mock("../../../../src/commands/run/wizard/sources/PulumiStateSource.ts"); @@ -16,23 +17,24 @@ vi.mock("../../../../src/commands/initProject/scaffoldProject.ts", () => ({ })); import { discoverProjects } from "../../../../src/commands/run/wizard/projectDiscovery.ts"; -import { discoverConfigs } from "../../../../src/commands/run/wizard/configDiscovery.ts"; +import { discoverConfig } from "../../../../src/commands/run/wizard/configDiscovery.ts"; +import { listAvailablePresetsWithDescriptions } from "../../../../src/commands/run/wizard/presetDiscovery.ts"; import { writeEnv } from "../../../../src/commands/run/wizard/envWriter.ts"; import { extractFromWebinyOutput } from "../../../../src/commands/run/wizard/sources/WebinyOutputSource.ts"; import { extractFromPulumiState } from "../../../../src/commands/run/wizard/sources/PulumiStateSource.ts"; import { input, select } from "@inquirer/prompts"; -import { stat, access } from "node:fs/promises"; +import { stat } from "node:fs/promises"; import { scaffoldProject } from "../../../../src/commands/initProject/scaffoldProject.ts"; const mockDiscoverProjects = vi.mocked(discoverProjects); -const mockDiscoverConfigs = vi.mocked(discoverConfigs); +const mockDiscoverConfig = vi.mocked(discoverConfig); +const mockListAvailablePresetsWithDescriptions = vi.mocked(listAvailablePresetsWithDescriptions); const mockWriteEnv = vi.mocked(writeEnv); const mockExtractFromWebinyOutput = vi.mocked(extractFromWebinyOutput); const mockExtractFromPulumiState = vi.mocked(extractFromPulumiState); const mockInput = vi.mocked(input); const mockSelect = vi.mocked(select); const mockStat = vi.mocked(stat); -const mockAccess = vi.mocked(access); const mockScaffoldProject = vi.mocked(scaffoldProject); const noFile = (): never => { @@ -62,27 +64,9 @@ beforeEach(() => { }); describe("TransferWizard", () => { - it("shows CREATE_NEW option even when no projects are found, and scaffolds on selection", async () => { - mockDiscoverProjects.mockResolvedValue([]); - mockSelect.mockResolvedValue("__create__"); - mockStat.mockRejectedValue(new Error("ENOENT")); - // First input call is for the new project name; second breaks out of the instructions loop. - mockInput.mockResolvedValueOnce("brand-new").mockRejectedValue(new Error("stop")); - - await expect(new TransferWizard(process.cwd()).run()).rejects.toThrow("stop"); - - expect(mockSelect).toHaveBeenCalledOnce(); - const choices = mockSelect.mock.calls[0][0].choices as Array<{ value: string }>; - expect(choices.some((c: { value: string }) => c.value === "__create__")).toBe(true); - expect(mockScaffoldProject).toHaveBeenCalledWith({ name: "brand-new", cwd: process.cwd() }); - }); - - it("create-new happy path: scaffolds project, writes env, returns null", async () => { - mockDiscoverProjects.mockResolvedValue([]); - mockSelect.mockResolvedValue("__create__"); - mockScaffoldProject.mockResolvedValue(undefined); - // First input call: project name. Second: segment count. - mockInput.mockResolvedValueOnce("my-project").mockResolvedValueOnce("4"); + it("env-setup path: writes .env and returns null", async () => { + mockDiscoverProjects.mockResolvedValue(["my-project"]); + mockSelect.mockResolvedValue("my-project"); mockStat.mockImplementation(async (p: unknown) => { const path = String(p); if (path.endsWith("source.webiny.json") || path.endsWith("target.webiny.json")) { @@ -90,63 +74,58 @@ describe("TransferWizard", () => { } return noFile(); }); - mockAccess.mockRejectedValue(Object.assign(new Error("ENOENT"), { code: "ENOENT" })); mockExtractFromWebinyOutput .mockResolvedValueOnce(SOURCE_VALS) .mockResolvedValueOnce(TARGET_VALS); + mockInput.mockResolvedValue("4"); const result = await new TransferWizard(process.cwd()).run(); - expect(mockScaffoldProject).toHaveBeenCalledWith({ - name: "my-project", - cwd: expect.any(String) - }); - expect(mockWriteEnv).toHaveBeenCalledOnce(); expect(result).toBeNull(); + expect(mockWriteEnv).toHaveBeenCalledOnce(); }); - it("routes to config selection when no JSON files and .env exists", async () => { + it("re-run path: .env exists, no JSON → finds config.ts, prompts for preset, returns WizardResult", async () => { + const CONFIG_PATH = "/projects/my-project/config.ts"; mockDiscoverProjects.mockResolvedValue(["my-project"]); - mockSelect.mockResolvedValue("my-project"); + mockSelect.mockResolvedValueOnce("my-project").mockResolvedValueOnce("v5-to-v6-ddb"); mockStat.mockImplementation(async (p: unknown) => { if (String(p).endsWith(".env")) { return { size: 100 } as unknown as Stats; } return noFile(); }); - mockDiscoverConfigs.mockResolvedValue([ - { path: "/projects/my-project/ddb.config.ts", label: "DynamoDB Transfer" } + mockDiscoverConfig.mockResolvedValue(CONFIG_PATH); + mockListAvailablePresetsWithDescriptions.mockResolvedValue([ + { name: "v5-to-v6-ddb", description: "DDB only" }, + { name: "v5-to-v6-os", description: "DDB + OpenSearch" } ]); const result = await new TransferWizard(process.cwd()).run(); - expect(result).toBe("/projects/my-project/ddb.config.ts"); + expect(result).toEqual({ configPath: CONFIG_PATH, preset: "v5-to-v6-ddb" }); expect(mockWriteEnv).not.toHaveBeenCalled(); }); - it("throws when same-side files disagree on osTableName", async () => { + it("re-run path: exits with error when no config.ts found in project", async () => { mockDiscoverProjects.mockResolvedValue(["my-project"]); mockSelect.mockResolvedValue("my-project"); mockStat.mockImplementation(async (p: unknown) => { - const path = String(p); - if (path.endsWith("source.webiny.json") || path.endsWith("source.pulumi.json")) { + if (String(p).endsWith(".env")) { return { size: 100 } as unknown as Stats; } return noFile(); }); - mockExtractFromWebinyOutput.mockResolvedValue({ - ...SOURCE_VALS, - osTableName: "wby-es-webiny" - }); - mockExtractFromPulumiState.mockResolvedValue({ - ...SOURCE_VALS, - osTableName: "wby-es-pulumi" - }); + mockDiscoverConfig.mockResolvedValue(null); - await expect(new TransferWizard(process.cwd()).run()).rejects.toThrow(/osTableName/); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("exit"); + }); + await expect(new TransferWizard(process.cwd()).run()).rejects.toThrow("exit"); + exitSpy.mockRestore(); }); - it("writes .env with correct values and returns null on happy path", async () => { + it("writes .env with correct values from webiny output", async () => { mockDiscoverProjects.mockResolvedValue(["my-project"]); mockSelect.mockResolvedValue("my-project"); mockStat.mockImplementation(async (p: unknown) => { @@ -156,19 +135,135 @@ describe("TransferWizard", () => { } return noFile(); }); - mockAccess.mockRejectedValue(Object.assign(new Error("ENOENT"), { code: "ENOENT" })); mockExtractFromWebinyOutput .mockResolvedValueOnce(SOURCE_VALS) .mockResolvedValueOnce(TARGET_VALS); mockInput.mockResolvedValue("4"); - const result = await new TransferWizard(process.cwd()).run(); + await new TransferWizard(process.cwd()).run(); - expect(result).toBeNull(); expect(mockWriteEnv).toHaveBeenCalledOnce(); const [, envValues] = mockWriteEnv.mock.calls[0]; expect(envValues.sourceRegion).toBe("eu-central-1"); expect(envValues.targetRegion).toBe("us-east-1"); expect(envValues.segments).toBe(4); }); + + it("warns when source and target are in different AWS accounts", async () => { + mockDiscoverProjects.mockResolvedValue(["my-project"]); + mockSelect.mockResolvedValue("my-project"); + mockStat.mockImplementation(async (p: unknown) => { + const path = String(p); + if (path.endsWith("source.webiny.json") || path.endsWith("target.webiny.json")) { + return { size: 100 } as unknown as Stats; + } + return noFile(); + }); + mockExtractFromWebinyOutput + .mockResolvedValueOnce({ ...SOURCE_VALS, accountId: "111111111111" }) + .mockResolvedValueOnce({ ...TARGET_VALS, accountId: "999999999999" }); + mockInput.mockResolvedValue("4"); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await new TransferWizard(process.cwd()).run(); + + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining("111111111111")); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining("999999999999")); + warnSpy.mockRestore(); + }); + + it("does not warn when source and target share the same AWS account", async () => { + mockDiscoverProjects.mockResolvedValue(["my-project"]); + mockSelect.mockResolvedValue("my-project"); + mockStat.mockImplementation(async (p: unknown) => { + const path = String(p); + if (path.endsWith("source.webiny.json") || path.endsWith("target.webiny.json")) { + return { size: 100 } as unknown as Stats; + } + return noFile(); + }); + mockExtractFromWebinyOutput + .mockResolvedValueOnce({ ...SOURCE_VALS, accountId: "111111111111" }) + .mockResolvedValueOnce({ ...TARGET_VALS, accountId: "111111111111" }); + mockInput.mockResolvedValue("4"); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await new TransferWizard(process.cwd()).run(); + + expect(warnSpy).not.toHaveBeenCalled(); + warnSpy.mockRestore(); + }); + + it("prompts for OS index prefix when OS fields are present", async () => { + const OS_SOURCE = { + ...SOURCE_VALS, + osTableName: "wby-es-source", + osEndpoint: "https://es.source" + }; + const OS_TARGET = { + ...TARGET_VALS, + osTableName: "wby-es-target", + osEndpoint: "https://es.target" + }; + mockDiscoverProjects.mockResolvedValue(["my-project"]); + mockSelect.mockResolvedValue("my-project"); + mockStat.mockImplementation(async (p: unknown) => { + const path = String(p); + if (path.endsWith("source.webiny.json") || path.endsWith("target.webiny.json")) { + return { size: 100 } as unknown as Stats; + } + return noFile(); + }); + mockExtractFromWebinyOutput + .mockResolvedValueOnce(OS_SOURCE) + .mockResolvedValueOnce(OS_TARGET); + mockInput.mockResolvedValueOnce("4").mockResolvedValueOnce("v6-"); + + await new TransferWizard(process.cwd()).run(); + + const [, envValues] = mockWriteEnv.mock.calls[0]; + expect(envValues.targetOsIndexPrefix).toBe("v6-"); + }); + + it("exits with error when no presets are available", async () => { + const CONFIG_PATH = "/projects/my-project/config.ts"; + mockDiscoverProjects.mockResolvedValue(["my-project"]); + mockSelect.mockResolvedValue("my-project"); + mockStat.mockImplementation(async (p: unknown) => { + if (String(p).endsWith(".env")) { + return { size: 100 } as unknown as Stats; + } + return noFile(); + }); + mockDiscoverConfig.mockResolvedValue(CONFIG_PATH); + mockListAvailablePresetsWithDescriptions.mockResolvedValue([]); + + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("exit"); + }); + await expect(new TransferWizard(process.cwd()).run()).rejects.toThrow("exit"); + exitSpy.mockRestore(); + }); + + it("throws when same-side files disagree on osTableName", async () => { + mockDiscoverProjects.mockResolvedValue(["my-project"]); + mockSelect.mockResolvedValue("my-project"); + mockStat.mockImplementation(async (p: unknown) => { + const path = String(p); + if (path.endsWith("source.webiny.json") || path.endsWith("source.pulumi.json")) { + return { size: 100 } as unknown as Stats; + } + return noFile(); + }); + mockExtractFromWebinyOutput.mockResolvedValue({ + ...SOURCE_VALS, + osTableName: "wby-es-webiny" + }); + mockExtractFromPulumiState.mockResolvedValue({ + ...SOURCE_VALS, + osTableName: "wby-es-pulumi" + }); + + await expect(new TransferWizard(process.cwd()).run()).rejects.toThrow(/osTableName/); + }); }); diff --git a/__tests__/commands/run/wizard/configDiscovery.test.ts b/__tests__/commands/run/wizard/configDiscovery.test.ts index aa2b4f3c..41ee7ff1 100644 --- a/__tests__/commands/run/wizard/configDiscovery.test.ts +++ b/__tests__/commands/run/wizard/configDiscovery.test.ts @@ -1,46 +1,34 @@ import { describe, it, expect } from "vitest"; import { join } from "node:path"; -import { discoverConfigs } from "../../../../src/commands/run/wizard/configDiscovery.ts"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { discoverConfig } from "../../../../src/commands/run/wizard/configDiscovery.ts"; -const FIXTURES = join(import.meta.dirname, "../../../fixtures/wizard"); - -describe("discoverConfigs", () => { - it("returns labeled configs for each *.config.ts file that imports successfully", async () => { - const configs = await discoverConfigs(FIXTURES); - expect(configs.length).toBe(2); - const labels = configs.map(c => c.label).sort(); - expect(labels).toEqual(["DynamoDB Transfer", "OpenSearch Transfer"]); - }); - - it("returns full resolved paths for each config", async () => { - const configs = await discoverConfigs(FIXTURES); - for (const c of configs) { - expect(c.path).toMatch(/\.config\.ts$/); - } - }); - - it("returns empty array when directory has no *.config.ts files", async () => { - const { mkdtemp } = await import("node:fs/promises"); - const { tmpdir } = await import("node:os"); - const tmp = await mkdtemp(join(tmpdir(), "configdiscovery-")); +describe("discoverConfig", () => { + it("returns the resolved path to config.ts when it exists", async () => { + const tmp = mkdtempSync(join(tmpdir(), "configdiscovery-")); try { - expect(await discoverConfigs(tmp)).toEqual([]); + const configPath = join(tmp, "config.ts"); + writeFileSync(configPath, "export default {};"); + const result = await discoverConfig(tmp); + expect(result).toBe(configPath); } finally { - const { rm } = await import("node:fs/promises"); - await rm(tmp, { recursive: true }); + rmSync(tmp, { recursive: true }); } }); - it("skips a config file that throws on import (does not crash)", async () => { - const { writeFile, mkdtemp, rm } = await import("node:fs/promises"); - const { tmpdir } = await import("node:os"); - const tmp = await mkdtemp(join(tmpdir(), "configdiscovery-bad-")); - await writeFile(join(tmp, "broken.config.ts"), "throw new Error('oops')"); + it("returns null when config.ts does not exist", async () => { + const tmp = mkdtempSync(join(tmpdir(), "configdiscovery-empty-")); try { - const configs = await discoverConfigs(tmp); - expect(configs).toEqual([]); + const result = await discoverConfig(tmp); + expect(result).toBeNull(); } finally { - await rm(tmp, { recursive: true }); + rmSync(tmp, { recursive: true }); } }); + + it("returns null for nonexistent directory", async () => { + const result = await discoverConfig("/nonexistent/path/xyz"); + expect(result).toBeNull(); + }); }); diff --git a/__tests__/commands/run/wizard/envWriter.test.ts b/__tests__/commands/run/wizard/envWriter.test.ts index 154ba2c3..ceed105d 100644 --- a/__tests__/commands/run/wizard/envWriter.test.ts +++ b/__tests__/commands/run/wizard/envWriter.test.ts @@ -9,13 +9,17 @@ const SAMPLE_VALUES: EnvValues = { sourceRegion: "eu-central-1", sourceDdbTable: "wby-source-primary", sourceS3Bucket: "wby-source-bucket", + sourceAuditLogTable: "", sourceOsTable: "wby-source-es", + sourceAccountId: "111111111111", targetRegion: "us-east-1", targetDdbTable: "wby-target-primary", targetS3Bucket: "wby-target-bucket", + targetAuditLogTable: "wby-target-audit-logs", targetOsTable: "wby-target-os", targetOsEndpoint: "search-target.us-east-1.es.amazonaws.com", targetOsIndexPrefix: "my-prefix", + targetAccountId: "222222222222", segments: 8 }; @@ -46,10 +50,12 @@ describe("writeEnv", () => { "SOURCE_REGION={{SOURCE_REGION}}", "SOURCE_DDB_TABLE={{SOURCE_DDB_TABLE}}", "SOURCE_S3_BUCKET={{SOURCE_S3_BUCKET}}", + "SOURCE_AUDIT_LOGS_TABLE={{SOURCE_AUDIT_LOGS_TABLE}}", "SOURCE_OS_TABLE={{SOURCE_OS_TABLE}}", "TARGET_REGION={{TARGET_REGION}}", "TARGET_DDB_TABLE={{TARGET_DDB_TABLE}}", "TARGET_S3_BUCKET={{TARGET_S3_BUCKET}}", + "TARGET_AUDIT_LOGS_TABLE={{TARGET_AUDIT_LOGS_TABLE}}", "TARGET_OS_TABLE={{TARGET_OS_TABLE}}", "TARGET_OS_ENDPOINT={{TARGET_OS_ENDPOINT}}", "TARGET_OS_INDEX_PREFIX={{TARGET_OS_INDEX_PREFIX}}", @@ -67,6 +73,7 @@ describe("writeEnv", () => { expect(content).toContain("TARGET_REGION=us-east-1"); expect(content).toContain("TARGET_DDB_TABLE=wby-target-primary"); expect(content).toContain("TARGET_S3_BUCKET=wby-target-bucket"); + expect(content).toContain("TARGET_AUDIT_LOGS_TABLE=wby-target-audit-logs"); expect(content).toContain("TARGET_OS_TABLE=wby-target-os"); expect(content).toContain("TARGET_OS_ENDPOINT=search-target.us-east-1.es.amazonaws.com"); expect(content).toContain("TARGET_OS_INDEX_PREFIX=my-prefix"); diff --git a/__tests__/commands/run/wizard/presetDiscovery.test.ts b/__tests__/commands/run/wizard/presetDiscovery.test.ts new file mode 100644 index 00000000..9cc91967 --- /dev/null +++ b/__tests__/commands/run/wizard/presetDiscovery.test.ts @@ -0,0 +1,106 @@ +import { describe, it, expect } from "vitest"; +import { join } from "node:path"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { + listAvailablePresets, + listAvailablePresetsWithDescriptions +} from "../../../../src/commands/run/wizard/presetDiscovery.ts"; + +describe("listAvailablePresets", () => { + it("returns built-in preset names (at minimum v5-to-v6-ddb and v5-to-v6-os)", () => { + const presets = listAvailablePresets(); + expect(presets).toContain("v5-to-v6-ddb"); + expect(presets).toContain("v5-to-v6-os"); + }); + + it("includes user presets from presetsDir when provided", () => { + const tmp = mkdtempSync(join(tmpdir(), "presetdiscovery-")); + try { + writeFileSync(join(tmp, "my-preset.ts"), "export default {}"); + writeFileSync(join(tmp, "another.ts"), "export default {}"); + const presets = listAvailablePresets(tmp); + expect(presets).toContain("my-preset"); + expect(presets).toContain("another"); + } finally { + rmSync(tmp, { recursive: true }); + } + }); + + it("deduplicates when user preset name matches a built-in", () => { + const tmp = mkdtempSync(join(tmpdir(), "presetdiscovery-dup-")); + try { + writeFileSync(join(tmp, "v5-to-v6-ddb.ts"), "export default {}"); + const presets = listAvailablePresets(tmp); + const count = presets.filter(p => p === "v5-to-v6-ddb").length; + expect(count).toBe(1); + } finally { + rmSync(tmp, { recursive: true }); + } + }); + + it("returns empty list when presetsDir does not exist", () => { + const presets = listAvailablePresets("/nonexistent/path/xyz"); + expect(Array.isArray(presets)).toBe(true); + }); + + it("returns sorted list", () => { + const presets = listAvailablePresets(); + const sorted = [...presets].sort(); + expect(presets).toEqual(sorted); + }); + + it("ignores files without .ts or .js extension", () => { + const tmp = mkdtempSync(join(tmpdir(), "presetdiscovery-ext-")); + try { + writeFileSync(join(tmp, "my-preset.ts"), "export default {}"); + writeFileSync(join(tmp, "readme.md"), "ignore me"); + const presets = listAvailablePresets(tmp); + expect(presets).toContain("my-preset"); + expect(presets).not.toContain("readme.md"); + expect(presets).not.toContain("readme"); + } finally { + rmSync(tmp, { recursive: true }); + } + }); +}); + +describe("listAvailablePresetsWithDescriptions", () => { + it("returns entries with name and description for built-in presets", async () => { + const entries = await listAvailablePresetsWithDescriptions(); + expect(entries.length).toBeGreaterThan(0); + for (const entry of entries) { + expect(typeof entry.name).toBe("string"); + expect(typeof entry.description).toBe("string"); + } + const ddb = entries.find(e => e.name === "copy-ddb"); + expect(ddb?.description).toBeTruthy(); + }); + + it("returns empty description for a preset whose file cannot be imported", async () => { + const tmp = mkdtempSync(join(tmpdir(), "presetdiscovery-broken-")); + try { + writeFileSync(join(tmp, "broken.js"), "this is not valid js export syntax %%%"); + const entries = await listAvailablePresetsWithDescriptions(tmp); + const broken = entries.find(e => e.name === "broken"); + expect(broken?.description).toBe(""); + } finally { + rmSync(tmp, { recursive: true }); + } + }); + + it("returns empty description when preset exports no description field", async () => { + const tmp = mkdtempSync(join(tmpdir(), "presetdiscovery-nodesc-")); + try { + writeFileSync( + join(tmp, "nodesc.js"), + "export default { name: 'nodesc', configure() {} }" + ); + const entries = await listAvailablePresetsWithDescriptions(tmp); + const nodesc = entries.find(e => e.name === "nodesc"); + expect(nodesc?.description).toBe(""); + } finally { + rmSync(tmp, { recursive: true }); + } + }); +}); diff --git a/__tests__/commands/run/wizard/schemas/webinyOutput.schema.test.ts b/__tests__/commands/run/wizard/schemas/webinyOutput.schema.test.ts index 7426a65f..18adc053 100644 --- a/__tests__/commands/run/wizard/schemas/webinyOutput.schema.test.ts +++ b/__tests__/commands/run/wizard/schemas/webinyOutput.schema.test.ts @@ -109,4 +109,33 @@ describe("normalizeOutputs", () => { expect(result.osTableName).toBe(""); expect(result.osEndpoint).toBe(""); }); + + it("extracts accountId from primaryDynamodbTableArn", () => { + const result = normalizeOutputs({ + region: "eu-central-1", + primaryDynamodbTableName: "wby-primary", + fileManagerBucketId: "wby-bucket", + primaryDynamodbTableArn: "arn:aws:dynamodb:eu-central-1:250532744892:table/wby-primary" + }); + expect(result.accountId).toBe("250532744892"); + }); + + it("sets accountId to undefined when primaryDynamodbTableArn is absent", () => { + const result = normalizeOutputs({ + region: "eu-central-1", + primaryDynamodbTableName: "wby-primary", + fileManagerBucketId: "wby-bucket" + }); + expect(result.accountId).toBeUndefined(); + }); + + it("sets accountId to undefined for a malformed ARN", () => { + const result = normalizeOutputs({ + region: "eu-central-1", + primaryDynamodbTableName: "wby-primary", + fileManagerBucketId: "wby-bucket", + primaryDynamodbTableArn: "not-an-arn" + }); + expect(result.accountId).toBeUndefined(); + }); }); diff --git a/__tests__/containers/ddb.ts b/__tests__/containers/ddb.ts index 82867e9d..c457b14f 100644 --- a/__tests__/containers/ddb.ts +++ b/__tests__/containers/ddb.ts @@ -33,6 +33,7 @@ import { DdbProcessorFeature } from "../../src/features/DdbProcessor/index.ts"; import { DdbExecutorFeature } from "../../src/features/DdbExecutor/index.ts"; import { S3ProcessorFeature } from "../../src/features/S3Processor/index.ts"; import { AuditLogProcessorFeature } from "../../src/features/AuditLogProcessor/index.ts"; +import { AccessCheckerFeature } from "../../src/features/AccessChecker/index.ts"; import { MockDynamoDbClient } from "../services/DynamoDbClient/MockDynamoDbClient.ts"; import { MockS3Client } from "../services/S3Client/MockS3Client.ts"; import { CompressionFeature } from "@webiny/utils/features/compression/feature.js"; @@ -53,6 +54,7 @@ export interface DdbContainerOptions { targetRecords?: Record; modelsDir?: string; presetsDir?: string; + auditLogTable?: string; logLevel?: "debug" | "info" | "warn" | "error"; pipelineOverride?: DdbContainerPipelineOverride; } @@ -62,7 +64,6 @@ export function createDdbContainer(options: DdbContainerOptions = {}): Container const targetDb = new MockDynamoDbClient(options.targetRecords || {}); const config: MigrationConfig.Interface = { - storage: "ddb", source: { region: "us-east-1", credentials: DEFAULT_CREDS, @@ -74,10 +75,11 @@ export function createDdbContainer(options: DdbContainerOptions = {}): Container credentials: DEFAULT_CREDS, dynamodb: { tableName: "target-table" }, s3: { bucket: "target-bucket" }, - auditLog: null + auditLog: options.auditLogTable + ? { dynamodb: { tableName: options.auditLogTable } } + : null }, pipeline: { - preset: "v5-to-v6", modelsDir: options.modelsDir, presetsDir: options.presetsDir, ...(options.pipelineOverride?.segments !== undefined @@ -123,6 +125,7 @@ export function createDdbContainer(options: DdbContainerOptions = {}): Container DdbScannerFeature.register(container); DdbProcessorFeature.register(container); AuditLogProcessorFeature.register(container); + AccessCheckerFeature.register(container); return container; } diff --git a/__tests__/containers/os.ts b/__tests__/containers/os.ts index 22eedb30..df4a0029 100644 --- a/__tests__/containers/os.ts +++ b/__tests__/containers/os.ts @@ -31,6 +31,7 @@ import { TouchedIndexesFeature } from "../../src/features/TouchedIndexes/index.t import { OsRecordDecompressorFeature } from "../../src/features/OsRecordDecompressor/index.ts"; import { OsScannerFeature } from "../../src/features/OsScanner/index.ts"; import { OsProcessorFeature } from "../../src/features/OsProcessor/index.ts"; +import { AccessCheckerFeature } from "../../src/features/AccessChecker/index.ts"; import { MockDynamoDbClient } from "../services/DynamoDbClient/MockDynamoDbClient.ts"; import { MockOpenSearchClient } from "../services/OpenSearchClient/MockOpenSearchClient.ts"; @@ -47,6 +48,7 @@ export interface OsContainerOptions { logLevel?: "debug" | "info" | "warn" | "error"; pipelineOverride?: OsContainerPipelineOverride; indexPrefix?: string; + noOpenSearch?: boolean; } export function createOsContainer(options: OsContainerOptions = {}): Container { @@ -55,25 +57,28 @@ export function createOsContainer(options: OsContainerOptions = {}): Container { const osClient = new MockOpenSearchClient(); const config: MigrationConfig.Interface = { - storage: "os", source: { region: "us-east-1", credentials: DEFAULT_CREDS, dynamodb: { tableName: "source-primary" }, + s3: { bucket: "source-bucket" }, opensearch: { tableName: "source-os" } }, target: { region: "eu-central-1", credentials: DEFAULT_CREDS, - opensearch: { - endpoint: "https://es.example.com", - tableName: "target-os", - service: "opensearch" as const, - indexPrefix: options.indexPrefix ?? "" - } + dynamodb: { tableName: "target-table" }, + s3: { bucket: "target-bucket" }, + opensearch: options.noOpenSearch + ? undefined + : { + endpoint: "https://es.example.com", + tableName: "target-os", + service: "opensearch" as const, + indexPrefix: options.indexPrefix ?? "" + } }, pipeline: { - preset: "v5-to-v6-os", modelsDir: options.modelsDir, presetsDir: options.presetsDir, ...(options.pipelineOverride?.segments !== undefined @@ -118,6 +123,7 @@ export function createOsContainer(options: OsContainerOptions = {}): Container { OsRecordDecompressorFeature.register(container); OsScannerFeature.register(container); OsProcessorFeature.register(container); + AccessCheckerFeature.register(container); return container; } diff --git a/__tests__/domain/pipeline/Processor.test.ts b/__tests__/domain/pipeline/Processor.test.ts index dc8faeea..26bbb49c 100644 --- a/__tests__/domain/pipeline/Processor.test.ts +++ b/__tests__/domain/pipeline/Processor.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from "vitest"; import { Container } from "@webiny/di"; import { Processor } from "~/domain/pipeline/index.ts"; +import { AccessCheck } from "~/domain/pipeline/abstractions/Processor.ts"; import { Commands } from "~/domain/transform/commands/Commands.ts"; import { PutRecord } from "~/domain/transform/commands/PutRecord.ts"; import type { BaseTransformContext } from "~/features/TransformContext/abstractions/BaseTransformContext.ts"; @@ -53,6 +54,10 @@ class FakeProcessor implements Processor.Interface { this.executed.push(commands); } + public async checkAccess(): Promise { + return []; + } + public afterShard(ctx: Processor.AfterShardContext): void { this.afterShardCalls.push(ctx); } diff --git a/__tests__/domain/pipeline/fixtures/fakes.ts b/__tests__/domain/pipeline/fixtures/fakes.ts index aa3af3fc..f657039a 100644 --- a/__tests__/domain/pipeline/fixtures/fakes.ts +++ b/__tests__/domain/pipeline/fixtures/fakes.ts @@ -2,6 +2,7 @@ import { Container } from "@webiny/di"; import { Commands } from "~/domain/transform/commands/Commands.ts"; import { PutRecord } from "~/domain/transform/commands/PutRecord.ts"; import { Scanner, Processor, Hook } from "~/domain/pipeline/index.ts"; +import { AccessCheck } from "~/domain/pipeline/abstractions/Processor.ts"; import type { BaseTransformContext } from "~/features/TransformContext/abstractions/BaseTransformContext.ts"; import type { FakeRecord, FakeShard, FakeSlice, FakeContext } from "./types.ts"; @@ -62,6 +63,10 @@ export class FakeProcessor implements Processor.Interface< this.executed.push(commands); } + public async checkAccess(): Promise { + return []; + } + public afterShardCalls: Processor.AfterShardContext[] = []; public afterShard(ctx: Processor.AfterShardContext): void { diff --git a/__tests__/features/AccessChecker/AccessChecker.test.ts b/__tests__/features/AccessChecker/AccessChecker.test.ts new file mode 100644 index 00000000..0d93bafe --- /dev/null +++ b/__tests__/features/AccessChecker/AccessChecker.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it, vi } from "vitest"; +import { Container } from "@webiny/di"; +import { ContainerToken } from "~/base/index.ts"; +import { Processor } from "~/domain/pipeline/index.ts"; +import { PipelineRunner } from "~/features/PipelineRunner/index.ts"; +import { AccessChecker, AccessCheckerFeature } from "~/features/AccessChecker/index.ts"; + +type StubProcessor = Pick; + +function makeRunner(processors: StubProcessor[]): PipelineRunner.Interface { + return { + register: vi.fn(), + run: vi.fn(), + getProcessors: vi.fn().mockReturnValue(processors), + getShardStats: vi.fn().mockReturnValue(null) + } as unknown as PipelineRunner.Interface; +} + +describe("AccessChecker", () => { + it("returns a flat report from all processor checkAccess results", async () => { + const p1 = { + checkAccess: vi.fn().mockResolvedValue([{ label: "DynamoDB source", status: "ok" }]), + execute: vi.fn() + }; + const p2 = { + checkAccess: vi.fn().mockResolvedValue([ + { label: "S3 source bucket: sb", status: "ok" }, + { label: "S3 target bucket: tb", status: "denied" } + ]), + execute: vi.fn() + }; + + const container = new Container(); + container.registerInstance(ContainerToken, container); + container.registerInstance(PipelineRunner, makeRunner([p1, p2])); + AccessCheckerFeature.register(container); + + const checker = container.resolve(AccessChecker); + const report = await checker.run(); + + expect(report).toHaveLength(3); + expect(report[0]).toEqual({ label: "DynamoDB source", status: "ok" }); + expect(report[1]).toEqual({ label: "S3 source bucket: sb", status: "ok" }); + expect(report[2]).toEqual({ label: "S3 target bucket: tb", status: "denied" }); + }); + + it("returns empty report when no processors are registered", async () => { + const container = new Container(); + container.registerInstance(ContainerToken, container); + container.registerInstance(PipelineRunner, makeRunner([])); + AccessCheckerFeature.register(container); + + const checker = container.resolve(AccessChecker); + const report = await checker.run(); + + expect(report).toHaveLength(0); + }); + + it("returns empty report when all processors return empty arrays", async () => { + const p = { + checkAccess: vi.fn().mockResolvedValue([]), + execute: vi.fn() + }; + const container = new Container(); + container.registerInstance(ContainerToken, container); + container.registerInstance(PipelineRunner, makeRunner([p])); + AccessCheckerFeature.register(container); + + const checker = container.resolve(AccessChecker); + const report = await checker.run(); + + expect(report).toHaveLength(0); + }); + + it("returns unknown entry for a processor that throws instead of returning a result", async () => { + const throwing = { + checkAccess: vi.fn().mockRejectedValue(new Error("unexpected SDK error")), + execute: vi.fn() + }; + const good = { + checkAccess: vi + .fn() + .mockResolvedValue([{ label: "DynamoDB source", status: "ok" as const }]), + execute: vi.fn() + }; + + const container = new Container(); + container.registerInstance(ContainerToken, container); + container.registerInstance(PipelineRunner, makeRunner([throwing, good])); + AccessCheckerFeature.register(container); + + const checker = container.resolve(AccessChecker); + const report = await checker.run(); + + expect(report).toHaveLength(2); + expect(report[0].status).toBe("unknown"); + expect(report[1]).toEqual({ label: "DynamoDB source", status: "ok" }); + }); +}); diff --git a/__tests__/features/AuditLogProcessor/AuditLogProcessor.test.ts b/__tests__/features/AuditLogProcessor/AuditLogProcessor.test.ts index f240b50c..46adb2a0 100644 --- a/__tests__/features/AuditLogProcessor/AuditLogProcessor.test.ts +++ b/__tests__/features/AuditLogProcessor/AuditLogProcessor.test.ts @@ -1,56 +1,20 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { Processor } from "~/domain/pipeline/abstractions/Processor.ts"; import { AuditLogProcessor } from "~/features/AuditLogProcessor/AuditLogProcessor.ts"; import { AuditLogPutRecord } from "~/domain/transform/commands/AuditLogPutRecord.ts"; import { Commands } from "~/domain/transform/commands/Commands.ts"; -import { MigrationConfig } from "~/features/MigrationConfig/abstractions/MigrationConfig.ts"; -import { Container } from "@webiny/di"; -import { ContainerToken } from "~/base/index.ts"; -import { MigrationConfigFeature } from "~/features/MigrationConfig/index.ts"; -import { DdbExecutorFeature } from "~/features/DdbExecutor/index.ts"; -import { AuditLogProcessorFeature } from "~/features/AuditLogProcessor/index.ts"; -import { - TargetDynamoDbClient, - SourceDynamoDbClient -} from "~/services/DynamoDbClient/abstractions/DynamoDbClient.ts"; -import { MockDynamoDbClient } from "../../services/DynamoDbClient/MockDynamoDbClient.ts"; import type { BaseTransformContext } from "~/features/TransformContext/abstractions/BaseTransformContext.ts"; +import { createDdbContainer } from "../../containers/ddb.ts"; +import { DynamoDB } from "@aws-sdk/client-dynamodb"; -const DEFAULT_CREDS = { accessKeyId: "test", secretAccessKey: "test" }; +vi.mock("@aws-sdk/client-dynamodb", () => ({ + DynamoDB: vi.fn() +})); interface AuditLogSlice { putAuditLog(record: Record): void; } -function makeContainer(auditLogTableName: string | null = "audit-log-table"): Container { - const container = new Container(); - container.registerInstance(ContainerToken, container); - MigrationConfigFeature.register(container, { - config: { - storage: "ddb", - source: { - region: "us-east-1", - credentials: DEFAULT_CREDS, - dynamodb: { tableName: "source-table" }, - s3: { bucket: "source-bucket" } - }, - target: { - region: "eu-central-1", - credentials: DEFAULT_CREDS, - dynamodb: { tableName: "target-table" }, - s3: { bucket: "target-bucket" }, - auditLog: auditLogTableName ? { dynamodb: { tableName: auditLogTableName } } : null - }, - pipeline: { preset: "v5-to-v6" } - } as MigrationConfig.Interface - }); - container.registerInstance(SourceDynamoDbClient, new MockDynamoDbClient({})); - container.registerInstance(TargetDynamoDbClient, new MockDynamoDbClient({})); - DdbExecutorFeature.register(container); - AuditLogProcessorFeature.register(container); - return container; -} - function makeBase(): { base: BaseTransformContext.Interface; captured: unknown[] } { const captured: unknown[] = []; const base = { @@ -64,8 +28,12 @@ function makeBase(): { base: BaseTransformContext.Interface; captured: } describe("AuditLogProcessor.putAuditLog", () => { + afterEach(() => { + vi.resetAllMocks(); + }); + it("emits AuditLogPutRecord when TYPE is auditLog.log", () => { - const container = makeContainer(); + const container = createDdbContainer({ auditLogTable: "audit-log-table" }); const processor = container .resolveAll(Processor) .find(p => p.constructor === AuditLogProcessor) as unknown as Processor.Interface< @@ -82,7 +50,7 @@ describe("AuditLogProcessor.putAuditLog", () => { }); it("does not emit when TYPE is not auditLog.log (raw CMS entry bypassed storageShape)", () => { - const container = makeContainer(); + const container = createDdbContainer({ auditLogTable: "audit-log-table" }); const processor = container .resolveAll(Processor) .find(p => p.constructor === AuditLogProcessor) as unknown as Processor.Interface< @@ -98,7 +66,7 @@ describe("AuditLogProcessor.putAuditLog", () => { }); it("does not emit when auditLog table is null regardless of TYPE", () => { - const container = makeContainer(null); + const container = createDdbContainer(); const processor = container .resolveAll(Processor) .find(p => p.constructor === AuditLogProcessor) as unknown as Processor.Interface< @@ -115,8 +83,12 @@ describe("AuditLogProcessor.putAuditLog", () => { }); describe("AuditLogProcessor.execute", () => { + afterEach(() => { + vi.resetAllMocks(); + }); + it("drains AuditLogPutRecord commands via DdbExecutor", async () => { - const container = makeContainer(); + const container = createDdbContainer({ auditLogTable: "audit-log-table" }); const processor = container .resolveAll(Processor) .find(p => p.constructor === AuditLogProcessor) as unknown as Processor.Interface< @@ -135,3 +107,98 @@ describe("AuditLogProcessor.execute", () => { await processor.execute(commands); }); }); + +describe("checkAccess", () => { + let mockDescribeTable: ReturnType; + + afterEach(() => { + vi.resetAllMocks(); + }); + + beforeEach(() => { + mockDescribeTable = vi.fn(); + vi.mocked(DynamoDB).mockImplementation(function (this: { + describeTable: typeof mockDescribeTable; + destroy: ReturnType; + }) { + this.describeTable = mockDescribeTable; + this.destroy = vi.fn(); + } as unknown as typeof DynamoDB); + }); + + it("returns empty array when audit log is not configured", async () => { + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === AuditLogProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries).toHaveLength(0); + }); + + it("returns ok when DescribeTable succeeds for the audit log table", async () => { + mockDescribeTable.mockResolvedValue({}); + const container = createDdbContainer({ auditLogTable: "audit-log-table" }); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === AuditLogProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries).toHaveLength(1); + expect(entries[0]).toEqual({ + label: "DynamoDB audit log table: audit-log-table", + status: "ok" + }); + }); + + it("returns denied when DescribeTable throws AccessDeniedException", async () => { + mockDescribeTable.mockRejectedValue( + Object.assign(new Error("Access denied"), { name: "AccessDeniedException" }) + ); + const container = createDdbContainer({ auditLogTable: "audit-log-table" }); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === AuditLogProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "DynamoDB audit log table: audit-log-table", + status: "denied" + }); + }); + + it("returns missing when DescribeTable throws ResourceNotFoundException", async () => { + mockDescribeTable.mockRejectedValue( + Object.assign(new Error("Table not found"), { name: "ResourceNotFoundException" }) + ); + const container = createDdbContainer({ auditLogTable: "audit-log-table" }); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === AuditLogProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "DynamoDB audit log table: audit-log-table", + status: "missing" + }); + }); + + it("returns unknown for non-access errors", async () => { + mockDescribeTable.mockRejectedValue(new Error("connection refused")); + const container = createDdbContainer({ auditLogTable: "audit-log-table" }); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === AuditLogProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "DynamoDB audit log table: audit-log-table", + status: "unknown" + }); + }); +}); diff --git a/__tests__/features/DdbProcessor/DdbProcessor.test.ts b/__tests__/features/DdbProcessor/DdbProcessor.test.ts index 51d4d9a1..0a075f81 100644 --- a/__tests__/features/DdbProcessor/DdbProcessor.test.ts +++ b/__tests__/features/DdbProcessor/DdbProcessor.test.ts @@ -1,4 +1,9 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { DynamoDB } from "@aws-sdk/client-dynamodb"; + +vi.mock("@aws-sdk/client-dynamodb", () => ({ + DynamoDB: vi.fn() +})); import { Logger } from "~/tools/Logger/abstractions/Logger.ts"; import { createDdbContainer } from "../../containers/index.ts"; import { Processor } from "~/domain/pipeline/abstractions/Processor.ts"; @@ -74,6 +79,10 @@ function makeBase(record: TRecord): BaseStub { } describe("DdbProcessor", () => { + afterEach(() => { + vi.resetAllMocks(); + }); + describe("extendContext", () => { it("returns a slice with putRecord that pushes PutRecord commands via ctx.addCommand", () => { const container = createDdbContainer(); @@ -164,4 +173,101 @@ describe("DdbProcessor", () => { expect(processor.afterShard).toBeUndefined(); }); }); + + describe("checkAccess", () => { + let mockDescribeTable: ReturnType; + + beforeEach(() => { + mockDescribeTable = vi.fn(); + vi.mocked(DynamoDB).mockImplementation(function (this: Record) { + this["describeTable"] = mockDescribeTable; + this["destroy"] = vi.fn(); + } as unknown as typeof DynamoDB); + }); + + it("returns ok entries for source and target tables when DescribeTable succeeds", async () => { + mockDescribeTable.mockResolvedValue({}); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === DdbProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries).toHaveLength(2); + expect(entries[0]).toEqual({ + label: "DynamoDB source table: source-table", + status: "ok" + }); + expect(entries[1]).toEqual({ + label: "DynamoDB target table: target-table", + status: "ok" + }); + }); + + it("returns denied when DescribeTable throws AccessDeniedException on source", async () => { + mockDescribeTable + .mockRejectedValueOnce( + Object.assign(new Error("Access denied"), { name: "AccessDeniedException" }) + ) + .mockResolvedValue({}); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === DdbProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "DynamoDB source table: source-table", + status: "denied" + }); + expect(entries[1]).toEqual({ + label: "DynamoDB target table: target-table", + status: "ok" + }); + }); + + it("returns unknown when DescribeTable throws a non-access error", async () => { + mockDescribeTable.mockRejectedValue( + Object.assign(new Error("connection timeout"), { name: "ETIMEDOUT" }) + ); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === DdbProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "DynamoDB source table: source-table", + status: "unknown" + }); + expect(entries[1]).toEqual({ + label: "DynamoDB target table: target-table", + status: "unknown" + }); + }); + + it("returns missing when DescribeTable throws ResourceNotFoundException", async () => { + mockDescribeTable.mockRejectedValue( + Object.assign(new Error("Table not found"), { name: "ResourceNotFoundException" }) + ); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === DdbProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "DynamoDB source table: source-table", + status: "missing" + }); + expect(entries[1]).toEqual({ + label: "DynamoDB target table: target-table", + status: "missing" + }); + }); + }); }); diff --git a/__tests__/features/DroppedRecordLog/DroppedRecordLog.test.ts b/__tests__/features/DroppedRecordLog/DroppedRecordLog.test.ts index 123113f5..773399cd 100644 --- a/__tests__/features/DroppedRecordLog/DroppedRecordLog.test.ts +++ b/__tests__/features/DroppedRecordLog/DroppedRecordLog.test.ts @@ -117,6 +117,21 @@ describe("DroppedRecordLog", () => { ).rejects.toThrow(/ENOENT/); }); + it("uses empty string for missing PK and SK in label", async () => { + const log = createContainer().resolve(DroppedRecordLog); + log.add( + { TYPE: "cms.entry.l" } as Record, + new RecordDisposition.Unmatched() + ); + log.flush(0); + + const content = await readFile( + join(workDir, ".transfer", "test-run-id", "segment-0-unmatched.log"), + "utf-8" + ); + expect(content.trim()).toBe("[cms.entry.l] :"); + }); + it("clears buffer after flush — second flush with no new adds is a no-op", async () => { const log = createContainer().resolve(DroppedRecordLog); log.add({ PK: "PK1", SK: "SK1", TYPE: "t1" }, new RecordDisposition.Unmatched()); diff --git a/__tests__/features/MigrationConfig/MigrationConfig.test.ts b/__tests__/features/MigrationConfig/MigrationConfig.test.ts index 80bb8b2f..1044d00a 100644 --- a/__tests__/features/MigrationConfig/MigrationConfig.test.ts +++ b/__tests__/features/MigrationConfig/MigrationConfig.test.ts @@ -9,201 +9,154 @@ import { loadConfig } from "../../../src/features/MigrationConfig/index.ts"; -describe("MigrationConfig Feature", () => { +describe("loadConfig", () => { let tmpDir: string; beforeEach(() => { - tmpDir = mkdtempSync(join(tmpdir(), "migration-config-test-")); + tmpDir = mkdtempSync(join(tmpdir(), "mc-test-")); }); - afterEach(() => { rmSync(tmpDir, { recursive: true, force: true }); }); - function writeConfig(config: object): string { - const filePath = join(tmpDir, "config.ts"); - writeFileSync(filePath, `export default ${JSON.stringify(config, null, 2)};`); - return filePath; - } - const creds = { accessKeyId: "AKIA", secretAccessKey: "secret" }; - describe("loadConfig", () => { - it("should load and validate a ddb config", async () => { - const configPath = writeConfig({ - storage: "ddb", - source: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "src" }, - s3: { bucket: "src-bucket" } - }, - target: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "tgt" }, - s3: { bucket: "tgt-bucket" } - }, - pipeline: { preset: "v5-to-v6" } - }); - - const config = await loadConfig(configPath); - expect(config.storage).toBe("ddb"); - }); - - it("should load and validate an os config", async () => { - const configPath = writeConfig({ - storage: "os", - source: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "src" }, - opensearch: { tableName: "src-es" } - }, - target: { - region: "eu-central-1", - credentials: creds, - opensearch: { - endpoint: "https://es.example.com", - tableName: "tgt-es", - service: "opensearch" - } - }, - pipeline: { preset: "v5-to-v6-os" } - }); - - const config = await loadConfig(configPath); - expect(config.storage).toBe("os"); - }); + function writeConfig(config: object): string { + const p = join(tmpDir, "config.ts"); + writeFileSync(p, `export default ${JSON.stringify(config, null, 2)};`); + return p; + } - it("should reject invalid config", async () => { - const configPath = writeConfig({ invalid: true }); - await expect(loadConfig(configPath)).rejects.toThrow(); + it("loads a valid unified config", async () => { + const p = writeConfig({ + source: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "src" }, + s3: { bucket: "src-b" } + }, + target: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "tgt" }, + s3: { bucket: "tgt-b" } + }, + pipeline: {} }); + const config = await loadConfig(p); + expect(config.source.dynamodb.tableName).toBe("src"); + expect((config as any).storage).toBeUndefined(); + }); - it("resolves a file-path preset relative to the config file's directory", async () => { - const configPath = writeConfig({ - storage: "ddb", - source: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "src" }, - s3: { bucket: "src-bucket" } - }, - target: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "tgt" }, - s3: { bucket: "tgt-bucket" } - }, - pipeline: { preset: "./my-preset.ts" } - }); - - const config = await loadConfig(configPath); - - expect(config.pipeline.preset).toBe(join(tmpDir, "my-preset.ts")); + it("loads a config with opensearch fields", async () => { + const p = writeConfig({ + source: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "src" }, + s3: { bucket: "src-b" }, + opensearch: { tableName: "src-os" } + }, + target: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "tgt" }, + s3: { bucket: "tgt-b" }, + opensearch: { + endpoint: "https://es.example.com", + tableName: "tgt-os", + service: "opensearch", + indexPrefix: "" + } + }, + pipeline: {} }); + const config = await loadConfig(p); + expect(config.source.opensearch?.tableName).toBe("src-os"); + }); - it("leaves built-in preset names unchanged", async () => { - const configPath = writeConfig({ - storage: "ddb", - source: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "src" }, - s3: { bucket: "src-bucket" } - }, - target: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "tgt" }, - s3: { bucket: "tgt-bucket" } - }, - pipeline: { preset: "v5-to-v6" } - }); - - const config = await loadConfig(configPath); + it("rejects invalid config", async () => { + const p = writeConfig({ invalid: true }); + await expect(loadConfig(p)).rejects.toThrow(); + }); - expect(config.pipeline.preset).toBe("v5-to-v6"); - }); + it("rejects config missing required fields", async () => { + const p = writeConfig({ source: { region: "us-east-1" } }); + await expect(loadConfig(p)).rejects.toThrow(); + }); - it("resolves presetsDir relative to the config file's directory", async () => { - const configPath = writeConfig({ - storage: "ddb", - source: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "src" }, - s3: { bucket: "src-bucket" } - }, - target: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "tgt" }, - s3: { bucket: "tgt-bucket" } - }, - pipeline: { preset: "v5-to-v6", presetsDir: "./custom-presets" } - }); + it("rejects file with no default export", async () => { + const p = join(tmpDir, "config.ts"); + writeFileSync(p, "export const x = 1;"); + await expect(loadConfig(p)).rejects.toThrow(/default export/); + }); - const config = await loadConfig(configPath); - expect(config.pipeline.presetsDir).toBe(join(tmpDir, "custom-presets")); + it("resolves presetsDir relative to config file directory", async () => { + const p = writeConfig({ + source: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "src" }, + s3: { bucket: "src-b" } + }, + target: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "tgt" }, + s3: { bucket: "tgt-b" } + }, + pipeline: { presetsDir: "./custom-presets" } }); + const config = await loadConfig(p); + expect(config.pipeline?.presetsDir).toBe(join(tmpDir, "custom-presets")); }); - describe("DI registration", () => { - it("should register config and resolve it from container", async () => { - const configPath = writeConfig({ - storage: "ddb", - source: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "src" }, - s3: { bucket: "src-bucket" } - }, - target: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "tgt" }, - s3: { bucket: "tgt-bucket" } - }, - pipeline: { preset: "v5-to-v6" } - }); - - const config = await loadConfig(configPath); - const container = new Container(); - - MigrationConfigFeature.register(container, { config }); - - const resolved = container.resolve(MigrationConfig); - expect(resolved.storage).toBe("ddb"); + it("resolves modelsDir relative to config file directory", async () => { + const p = writeConfig({ + source: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "src" }, + s3: { bucket: "src-b" } + }, + target: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "tgt" }, + s3: { bucket: "tgt-b" } + }, + pipeline: { modelsDir: "./models" } }); + const config = await loadConfig(p); + expect(config.pipeline?.modelsDir).toBe(join(tmpDir, "models")); + }); +}); - it("should resolve same instance on multiple resolves", async () => { - const configPath = writeConfig({ - storage: "ddb", - source: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "src" }, - s3: { bucket: "src-bucket" } - }, - target: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "tgt" }, - s3: { bucket: "tgt-bucket" } - }, - pipeline: { preset: "v5-to-v6" } - }); - - const config = await loadConfig(configPath); - const container = new Container(); - - MigrationConfigFeature.register(container, { config }); - - const first = container.resolve(MigrationConfig); - const second = container.resolve(MigrationConfig); - expect(first).toBe(second); +describe("MigrationConfig DI registration", () => { + it("registers and resolves the config", async () => { + const creds = { accessKeyId: "AKIA", secretAccessKey: "secret" }; + const { migrationConfigSchema } = + await import("../../../src/features/MigrationConfig/validation.ts"); + const config = migrationConfigSchema.parse({ + source: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "src" }, + s3: { bucket: "src-b" } + }, + target: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "tgt" }, + s3: { bucket: "tgt-b" } + }, + pipeline: {} }); + const container = new Container(); + MigrationConfigFeature.register(container, { config }); + const resolved = container.resolve(MigrationConfig); + expect(resolved.source.dynamodb.tableName).toBe("src"); + const second = container.resolve(MigrationConfig); + expect(resolved).toBe(second); }); }); diff --git a/__tests__/features/MigrationConfig/createConfig.test.ts b/__tests__/features/MigrationConfig/createConfig.test.ts index 1a686d13..d199e25f 100644 --- a/__tests__/features/MigrationConfig/createConfig.test.ts +++ b/__tests__/features/MigrationConfig/createConfig.test.ts @@ -1,483 +1,311 @@ import { describe, it, expect } from "vitest"; -import { createDdbConfig } from "../../../src/features/MigrationConfig/createDdbConfig.ts"; -import { createOsConfig } from "../../../src/features/MigrationConfig/createOsConfig.ts"; +import { createConfig } from "../../../src/features/MigrationConfig/createConfig.ts"; const creds = { accessKeyId: "AKIA", secretAccessKey: "secret" }; -describe("createDdbConfig", () => { - it("should return a valid ddb config with storage set", () => { - const config = createDdbConfig({ - source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src" }, - s3: { bucket: "src-bucket" } +const baseSource = { + region: "us-east-1", + credentials: creds, + dynamodb: { tableName: "src-table" }, + s3: { bucket: "src-bucket" } +}; + +const baseTarget = { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "tgt-table" }, + s3: { bucket: "tgt-bucket" } +}; + +describe("createConfig — happy path", () => { + it("returns a config with required fields, no storage field", () => { + const config = createConfig({ source: baseSource, target: baseTarget, pipeline: {} }); + expect(config.source.dynamodb.tableName).toBe("src-table"); + expect(config.target.s3.bucket).toBe("tgt-bucket"); + expect((config as any).storage).toBeUndefined(); + expect((config as any).pipeline?.preset).toBeUndefined(); + }); + + it("accepts optional opensearch on both sides", () => { + const config = createConfig({ + source: { ...baseSource, opensearch: { tableName: "src-os" } }, + target: { + ...baseTarget, + opensearch: { + endpoint: "https://search-x.es.amazonaws.com", + tableName: "tgt-os", + service: "opensearch", + indexPrefix: "" + } }, + pipeline: {} + }); + expect(config.source.opensearch?.tableName).toBe("src-os"); + expect(config.target.opensearch?.endpoint).toBe("https://search-x.es.amazonaws.com"); + }); + + it("accepts opensearch-serverless service", () => { + const config = createConfig({ + source: { ...baseSource, opensearch: { tableName: "src-os" } }, target: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "tgt" }, - s3: { bucket: "tgt-bucket" }, - auditLog: null + ...baseTarget, + opensearch: { + endpoint: "https://xxx.aoss.amazonaws.com", + tableName: "tgt-os", + service: "opensearch-serverless", + indexPrefix: "" + } }, - pipeline: { preset: "v5-to-v6" } + pipeline: {} }); + expect(config.target.opensearch?.service).toBe("opensearch-serverless"); + }); - expect(config.storage).toBe("ddb"); - expect(config.source.dynamodb.tableName).toBe("src"); - expect(config.target.s3.bucket).toBe("tgt-bucket"); - expect(config.pipeline.preset).toBe("v5-to-v6"); + it("accepts optional auditLog", () => { + const config = createConfig({ + source: baseSource, + target: { ...baseTarget, auditLog: { dynamodb: { tableName: "audit-table" } } }, + pipeline: {} + }); + expect(config.target.auditLog?.dynamodb?.tableName).toBe("audit-table"); + }); + + it("accepts nullable auditLog (null = skip)", () => { + const config = createConfig({ + source: baseSource, + target: { ...baseTarget, auditLog: null }, + pipeline: {} + }); + expect(config.target.auditLog).toBeNull(); }); - it("should accept optional segments and modelsDir", () => { - const config = createDdbConfig({ + it("trims whitespace from string fields", () => { + const config = createConfig({ source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src" }, - s3: { bucket: "src-bucket" } - }, - target: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "tgt" }, - s3: { bucket: "tgt-bucket" }, - auditLog: null + ...baseSource, + region: " us-east-1 ", + dynamodb: { tableName: " src " }, + s3: { bucket: " src-b " } }, - pipeline: { preset: "v5-to-v6", segments: 8, modelsDir: "./models" } + target: { ...baseTarget, region: " eu-central-1 " }, + pipeline: {} }); + expect(config.source.region).toBe("us-east-1"); + expect(config.source.dynamodb.tableName).toBe("src"); + expect(config.source.s3.bucket).toBe("src-b"); + expect(config.target.region).toBe("eu-central-1"); + }); - expect(config.pipeline.segments).toBe(8); - expect(config.pipeline.modelsDir).toBe("./models"); + it("accepts optional segments / modelsDir / presetsDir in pipeline", () => { + const config = createConfig({ + source: baseSource, + target: baseTarget, + pipeline: { segments: 8, modelsDir: "./models", presetsDir: "./presets" } + }); + expect(config.pipeline?.segments).toBe(8); + expect(config.pipeline?.modelsDir).toBe("./models"); }); +}); - it("should throw on missing source region", () => { +describe("createConfig — validation errors", () => { + it("throws on missing source region", () => { expect(() => - createDdbConfig({ - source: { - credentials: creds, - dynamodb: { tableName: "src" }, - s3: { bucket: "b" } - } as any, - target: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "tgt" }, - s3: { bucket: "b" }, - auditLog: null - }, - pipeline: { preset: "v5-to-v6" } + createConfig({ + source: { ...baseSource, region: "" } as any, + target: baseTarget, + pipeline: {} }) ).toThrow(); }); - it("should throw on missing credentials", () => { + it("throws on whitespace-only table name", () => { expect(() => - createDdbConfig({ - source: { - region: "us-east-1", - dynamodb: { tableName: "src" }, - s3: { bucket: "b" } - } as any, - target: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "tgt" }, - s3: { bucket: "b" }, - auditLog: null - }, - pipeline: { preset: "v5-to-v6" } + createConfig({ + source: { ...baseSource, dynamodb: { tableName: " " } }, + target: baseTarget, + pipeline: {} }) ).toThrow(); }); - it("should throw on missing preset", () => { + it("throws on missing credentials", () => { expect(() => - createDdbConfig({ - source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src" }, - s3: { bucket: "b" } - }, - target: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "tgt" }, - s3: { bucket: "b" }, - auditLog: null - }, - pipeline: {} as any + createConfig({ + source: { ...baseSource, credentials: undefined as any }, + target: baseTarget, + pipeline: {} }) ).toThrow(); }); -}); -describe("createOsConfig", () => { - it("should return a valid os config with storage set", () => { - const config = createOsConfig({ - source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src-primary" }, - opensearch: { tableName: "src-es" } - }, - target: { - region: "eu-central-1", - credentials: creds, - opensearch: { - endpoint: "https://search-xxx.es.amazonaws.com", - tableName: "tgt-es", - service: "opensearch", - indexPrefix: "" - } - }, - pipeline: { preset: "v5-to-v6-os" } - }); - - expect(config.storage).toBe("os"); - expect(config.source.opensearch.tableName).toBe("src-es"); - expect(config.source.dynamodb.tableName).toBe("src-primary"); - expect(config.target.opensearch.endpoint).toBe("https://search-xxx.es.amazonaws.com"); - }); - - it("should accept opensearch-serverless service", () => { - const config = createOsConfig({ - source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src" }, - opensearch: { tableName: "src-es" } - }, - target: { - region: "us-east-1", - credentials: creds, - opensearch: { - endpoint: "https://xxx.aoss.amazonaws.com", - tableName: "tgt-es", - service: "opensearch-serverless", - indexPrefix: "" - } - }, - pipeline: { preset: "v5-to-v6-os" } - }); - - expect(config.target.opensearch.service).toBe("opensearch-serverless"); - }); - - it("should throw on missing source opensearch", () => { + it("throws when only source.opensearch is set (target must match)", () => { expect(() => - createOsConfig({ - source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src" } - } as any, - target: { - region: "us-east-1", - credentials: creds, - opensearch: { - endpoint: "https://es.example.com", - tableName: "tgt-es", - service: "opensearch", - indexPrefix: "" - } - }, - pipeline: { preset: "v5-to-v6-os" } + createConfig({ + source: { ...baseSource, opensearch: { tableName: "src-os" } }, + target: baseTarget, + pipeline: {} }) - ).toThrow(); + ).toThrow(/both be set or both be absent/); }); - it("should throw on missing target opensearch service", () => { + it("throws when only target.opensearch is set", () => { expect(() => - createOsConfig({ - source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src" }, - opensearch: { tableName: "src-es" } - }, + createConfig({ + source: baseSource, target: { - region: "us-east-1", - credentials: creds, + ...baseTarget, opensearch: { endpoint: "https://es.example.com", - tableName: "tgt-es", + tableName: "tgt-os", + service: "opensearch", indexPrefix: "" - } as any + } }, - pipeline: { preset: "v5-to-v6-os" } + pipeline: {} }) - ).toThrow(); + ).toThrow(/both be set or both be absent/); }); - it("should throw on invalid endpoint URL", () => { + it("throws on same S3 bucket for source and target", () => { expect(() => - createOsConfig({ - source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src" }, - opensearch: { tableName: "src-es" } - }, - target: { - region: "us-east-1", - credentials: creds, - opensearch: { - endpoint: "not-a-url", - tableName: "tgt-es", - service: "opensearch", - indexPrefix: "" - } - }, - pipeline: { preset: "v5-to-v6-os" } + createConfig({ + source: baseSource, + target: { ...baseTarget, s3: { bucket: baseSource.s3.bucket } }, + pipeline: {} }) - ).toThrow(); + ).toThrow(/same as source/); }); - it("throws when target indexPrefix is missing", () => { + it("throws on same region + same DDB table", () => { expect(() => - createOsConfig({ - source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src-primary" }, - opensearch: { tableName: "src-es" } - }, + createConfig({ + source: baseSource, target: { - region: "eu-central-1", - credentials: creds, - opensearch: { - endpoint: "https://search-xxx.es.amazonaws.com", - tableName: "tgt-es", - service: "opensearch" - } as any + ...baseTarget, + region: baseSource.region, + dynamodb: { tableName: baseSource.dynamodb.tableName } }, - pipeline: { preset: "v5-to-v6-os" } + pipeline: {} }) - ).toThrow(); - }); - - it("accepts empty string indexPrefix (no prefix)", () => { - const config = createOsConfig({ - source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src-primary" }, - opensearch: { tableName: "src-es" } - }, - target: { - region: "eu-central-1", - credentials: creds, - opensearch: { - endpoint: "https://search-xxx.es.amazonaws.com", - tableName: "tgt-es", - service: "opensearch", - indexPrefix: "" - } - }, - pipeline: { preset: "v5-to-v6-os" } - }); - expect(config.target.opensearch.indexPrefix).toBe(""); - }); - - it("accepts and trims a non-empty indexPrefix", () => { - const config = createOsConfig({ - source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src-primary" }, - opensearch: { tableName: "src-es" } - }, - target: { - region: "eu-central-1", - credentials: creds, - opensearch: { - endpoint: "https://search-xxx.es.amazonaws.com", - tableName: "tgt-es", - service: "opensearch", - indexPrefix: " my-prefix- " - } - }, - pipeline: { preset: "v5-to-v6-os" } - }); - expect(config.target.opensearch.indexPrefix).toBe("my-prefix-"); + ).toThrow(/matches source/); }); -}); -describe("createDdbConfig — source/target collision guard", () => { - const baseDdbSource = { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src-table" }, - s3: { bucket: "src-bucket" } - }; - const baseDdbTarget = { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "tgt-table" }, - s3: { bucket: "tgt-bucket" }, - auditLog: null - }; - - it("rejects same S3 bucket for source and target", () => { + it("accepts same DDB table across different regions", () => { expect(() => - createDdbConfig({ - source: baseDdbSource, - target: { ...baseDdbTarget, s3: { bucket: baseDdbSource.s3.bucket } }, - pipeline: { preset: "v5-to-v6" } + createConfig({ + source: baseSource, + target: { ...baseTarget, dynamodb: { tableName: baseSource.dynamodb.tableName } }, + pipeline: {} }) - ).toThrow(/same as source/); + ).not.toThrow(); }); - it("rejects same region + same DDB table for source and target", () => { + it("throws on same region + same OS table when opensearch present", () => { expect(() => - createDdbConfig({ - source: baseDdbSource, + createConfig({ + source: { ...baseSource, opensearch: { tableName: "same-os" } }, target: { - ...baseDdbTarget, - region: baseDdbSource.region, - dynamodb: { tableName: baseDdbSource.dynamodb.tableName } + ...baseTarget, + region: baseSource.region, + opensearch: { + endpoint: "https://es.example.com", + tableName: "same-os", + service: "opensearch", + indexPrefix: "" + } }, - pipeline: { preset: "v5-to-v6" } + pipeline: {} }) ).toThrow(/matches source/); }); - it("accepts same DDB table name across different regions", () => { + it("throws on auditLog table matching main target table", () => { expect(() => - createDdbConfig({ - source: baseDdbSource, + createConfig({ + source: baseSource, target: { - ...baseDdbTarget, - // different region, same table name — distinct physical tables - dynamodb: { tableName: baseDdbSource.dynamodb.tableName } + ...baseTarget, + auditLog: { dynamodb: { tableName: baseTarget.dynamodb.tableName } } }, - pipeline: { preset: "v5-to-v6" } + pipeline: {} }) - ).not.toThrow(); + ).toThrow(/must differ/); }); -}); -describe("createDdbConfig — string trimming", () => { - it("trims whitespace around string fields (paste-error tolerance)", () => { - const config = createDdbConfig({ - source: { - region: " us-east-1\t", - credentials: creds, - dynamodb: { tableName: " src-table " }, - s3: { bucket: " src-bucket\n" } - }, - target: { - region: " eu-central-1 ", - credentials: creds, - dynamodb: { tableName: "tgt-table " }, - s3: { bucket: " tgt-bucket " }, - auditLog: null - }, - pipeline: { preset: " v5-to-v6-ddb " } - }); - - expect(config.source.region).toBe("us-east-1"); - expect(config.source.dynamodb.tableName).toBe("src-table"); - expect(config.source.s3.bucket).toBe("src-bucket"); - expect(config.target.region).toBe("eu-central-1"); - expect(config.target.dynamodb.tableName).toBe("tgt-table"); - expect(config.target.s3.bucket).toBe("tgt-bucket"); - expect(config.pipeline.preset).toBe("v5-to-v6-ddb"); - }); - - it("rejects whitespace-only strings (empty after trim)", () => { + it("throws on invalid opensearch endpoint URL", () => { expect(() => - createDdbConfig({ - source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: " " }, - s3: { bucket: "src-bucket" } - }, + createConfig({ + source: { ...baseSource, opensearch: { tableName: "src-os" } }, target: { - region: "eu-central-1", - credentials: creds, - dynamodb: { tableName: "tgt-table" }, - s3: { bucket: "tgt-bucket" }, - auditLog: null + ...baseTarget, + opensearch: { + endpoint: "not-a-url", + tableName: "tgt-os", + service: "opensearch", + indexPrefix: "" + } }, - pipeline: { preset: "v5-to-v6-ddb" } + pipeline: {} }) ).toThrow(); }); - it("collision guard runs against TRIMMED values (trailing-space doesn't mask a same-table mistake)", () => { + it("collision guard runs on trimmed values", () => { expect(() => - createDdbConfig({ - source: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "same-table" }, - s3: { bucket: "src-bucket" } - }, + createConfig({ + source: baseSource, target: { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "same-table " }, - s3: { bucket: "tgt-bucket" }, - auditLog: null + ...baseTarget, + region: baseSource.region, + dynamodb: { tableName: "src-table " } }, - pipeline: { preset: "v5-to-v6-ddb" } + pipeline: {} }) ).toThrow(/matches source/); }); }); -describe("createOsConfig — source/target collision guard", () => { - const baseOsSource = { - region: "us-east-1", - credentials: creds, - dynamodb: { tableName: "src-primary" }, - opensearch: { tableName: "src-es-table" } - }; - const baseOsTarget = { - region: "eu-central-1", - credentials: creds, - opensearch: { - endpoint: "https://search-xxx.example.com", - tableName: "tgt-es-table", - service: "opensearch" as const, - indexPrefix: "" - } - }; +describe("createConfig — tuning.flushEvery", () => { + it("accepts a positive integer", () => { + const config = createConfig({ + source: baseSource, + target: baseTarget, + pipeline: {}, + tuning: { flushEvery: 100 } + }); + expect(config.tuning?.flushEvery).toBe(100); + }); - it("rejects same region + same OS DDB table for source and target", () => { + it("rejects 0 (not positive)", () => { expect(() => - createOsConfig({ - source: baseOsSource, - target: { - ...baseOsTarget, - region: baseOsSource.region, - opensearch: { - ...baseOsTarget.opensearch, - tableName: baseOsSource.opensearch.tableName - } - }, - pipeline: { preset: "v5-to-v6-os" } + createConfig({ + source: baseSource, + target: baseTarget, + pipeline: {}, + tuning: { flushEvery: 0 } }) - ).toThrow(/matches source/); + ).toThrow(); }); - it("accepts same OS table name across different regions", () => { + it("rejects -1 (negative)", () => { expect(() => - createOsConfig({ - source: baseOsSource, - target: { - ...baseOsTarget, - opensearch: { - ...baseOsTarget.opensearch, - tableName: baseOsSource.opensearch.tableName - } - }, - pipeline: { preset: "v5-to-v6-os" } + createConfig({ + source: baseSource, + target: baseTarget, + pipeline: {}, + tuning: { flushEvery: -1 } }) - ).not.toThrow(); + ).toThrow(); + }); + + it("rejects 1.5 (non-integer)", () => { + expect(() => + createConfig({ + source: baseSource, + target: baseTarget, + pipeline: {}, + tuning: { flushEvery: 1.5 } + }) + ).toThrow(); }); }); diff --git a/__tests__/features/OpenSearchClient/enableRefreshHook.test.ts b/__tests__/features/OpenSearchClient/enableRefreshHook.test.ts new file mode 100644 index 00000000..9d6d38e3 --- /dev/null +++ b/__tests__/features/OpenSearchClient/enableRefreshHook.test.ts @@ -0,0 +1,206 @@ +import { describe, it, expect, vi } from "vitest"; +import { join } from "path"; +import { Container } from "@webiny/di"; +import { EnableRefreshHook } from "~/services/OpenSearchClient/hooks/EnableRefreshHook.ts"; +import { AfterTransferHook } from "~/features/TransferLifecycle/abstractions/TransferLifecycle.ts"; +import { TransferLifecycleFeature } from "~/features/TransferLifecycle/feature.ts"; +import { OpenSearchClient } from "~/services/OpenSearchClient/abstractions/OpenSearchClient.ts"; +import { Logger } from "~/tools/Logger/abstractions/Logger.ts"; +import { TransferContext } from "~/features/TransferLifecycle/abstractions/TransferContext.ts"; +import { DirectoryTool } from "~/tools/DirectoryTool/abstractions/DirectoryTool.ts"; +import { FileTool } from "~/tools/FileTool/abstractions/FileTool.ts"; +import type { TouchedIndexes } from "~/features/TouchedIndexes/abstractions/TouchedIndexes.ts"; + +const RUN_ID = "test-run-42"; +const TRANSFER_DIR = join(process.cwd(), ".transfer", RUN_ID); + +interface Harness { + hook: AfterTransferHook.Interface; + putIndexSettings: ReturnType; + warnCalls: string[]; + infoCalls: string[]; + dirFiles: Map; + fileContents: Map; +} + +function makeHarness( + dirFiles: Map = new Map(), + fileContents: Map = new Map() +): Harness { + const putIndexSettings = vi.fn().mockResolvedValue(undefined); + const warnCalls: string[] = []; + const infoCalls: string[] = []; + + const mockOs: OpenSearchClient.Interface = { + indexExists: vi.fn(), + createIndex: vi.fn(), + listIndexes: vi.fn(), + putIndexSettings, + getIndexSettings: vi.fn() + }; + + const mockLogger: Logger.Interface = { + debug: vi.fn(), + info: (_msg: string) => { + infoCalls.push(_msg); + }, + warn: (_msg: string) => { + warnCalls.push(_msg); + }, + error: vi.fn(), + fatal: vi.fn(), + done: vi.fn(), + child: vi.fn() + }; + + const mockDirTool: DirectoryTool.Interface = { + exists: vi.fn(), + create: vi.fn(), + readDir: (path: string) => dirFiles.get(path) ?? null, + readDirOrThrow: vi.fn(), + remove: vi.fn(), + copy: vi.fn(), + copyOrThrow: vi.fn() + }; + + const mockFileTool: FileTool.Interface = { + exists: vi.fn(), + readFile: (path: string) => fileContents.get(path) ?? null, + readFileOrThrow: vi.fn(), + writeFile: vi.fn(), + writeFileOrThrow: vi.fn(), + remove: vi.fn(), + copy: vi.fn(), + copyOrThrow: vi.fn() + }; + + const container = new Container(); + TransferLifecycleFeature.register(container); + container.registerInstance(TransferContext, { runId: RUN_ID }); + container.registerInstance(OpenSearchClient, mockOs); + container.registerInstance(Logger, mockLogger); + container.registerInstance(DirectoryTool, mockDirTool); + container.registerInstance(FileTool, mockFileTool); + container.register(EnableRefreshHook); + + const hook = container.resolve(AfterTransferHook); + return { hook, putIndexSettings, warnCalls, infoCalls, dirFiles, fileContents }; +} + +function indexFile(items: TouchedIndexes.Item[]): string { + return JSON.stringify(items); +} + +describe("EnableRefreshHook", () => { + it("does nothing when the .transfer/ directory does not exist", async () => { + const { hook, putIndexSettings } = makeHarness(); + // dirFiles is empty → readDir returns null + await hook.execute(); + expect(putIndexSettings).not.toHaveBeenCalled(); + }); + + it("does nothing when the directory exists but has no index files", async () => { + const dirFiles = new Map([[TRANSFER_DIR, ["segment-0.log", "segment-0-unmatched.log"]]]); + const { hook, putIndexSettings } = makeHarness(dirFiles); + await hook.execute(); + expect(putIndexSettings).not.toHaveBeenCalled(); + }); + + it("restores refresh_interval for each index in the file", async () => { + const items: TouchedIndexes.Item[] = [ + { indexName: "tenant-cms-entries", originalRefresh: "1s" }, + { indexName: "tenant-cms-models", originalRefresh: "5s" } + ]; + const filePath = join(TRANSFER_DIR, "segment-0-indexes.json"); + const dirFiles = new Map([[TRANSFER_DIR, ["segment-0-indexes.json"]]]); + const fileContents = new Map([[filePath, indexFile(items)]]); + + const { hook, putIndexSettings } = makeHarness(dirFiles, fileContents); + await hook.execute(); + + expect(putIndexSettings).toHaveBeenCalledTimes(2); + expect(putIndexSettings).toHaveBeenCalledWith("tenant-cms-entries", { + index: { refresh_interval: "1s" } + }); + expect(putIndexSettings).toHaveBeenCalledWith("tenant-cms-models", { + index: { refresh_interval: "5s" } + }); + }); + + it("applies first-writer-wins when the same index appears in multiple segment files", async () => { + const seg0Items: TouchedIndexes.Item[] = [ + { indexName: "shared-index", originalRefresh: "1s" } + ]; + const seg1Items: TouchedIndexes.Item[] = [ + { indexName: "shared-index", originalRefresh: "30s" } + ]; + const dirFiles = new Map([ + [TRANSFER_DIR, ["segment-0-indexes.json", "segment-1-indexes.json"]] + ]); + const fileContents = new Map([ + [join(TRANSFER_DIR, "segment-0-indexes.json"), indexFile(seg0Items)], + [join(TRANSFER_DIR, "segment-1-indexes.json"), indexFile(seg1Items)] + ]); + + const { hook, putIndexSettings } = makeHarness(dirFiles, fileContents); + await hook.execute(); + + expect(putIndexSettings).toHaveBeenCalledOnce(); + expect(putIndexSettings).toHaveBeenCalledWith("shared-index", { + index: { refresh_interval: "1s" } + }); + }); + + it("logs a warning and continues when putIndexSettings throws", async () => { + const items: TouchedIndexes.Item[] = [ + { indexName: "idx-a", originalRefresh: "1s" }, + { indexName: "idx-b", originalRefresh: "5s" } + ]; + const filePath = join(TRANSFER_DIR, "segment-0-indexes.json"); + const dirFiles = new Map([[TRANSFER_DIR, ["segment-0-indexes.json"]]]); + const fileContents = new Map([[filePath, indexFile(items)]]); + + const { hook, putIndexSettings, warnCalls } = makeHarness(dirFiles, fileContents); + putIndexSettings.mockRejectedValueOnce(new Error("OS unavailable")); + + await hook.execute(); + + // Second index is still restored despite the first failing + expect(putIndexSettings).toHaveBeenCalledTimes(2); + expect(warnCalls.some(m => m.includes("idx-a"))).toBe(true); + }); + + it("warns and skips a file whose content is not a JSON array", async () => { + const filePath = join(TRANSFER_DIR, "segment-0-indexes.json"); + const dirFiles = new Map([[TRANSFER_DIR, ["segment-0-indexes.json"]]]); + const fileContents = new Map([[filePath, JSON.stringify({ notAnArray: true })]]); + + const { hook, putIndexSettings, warnCalls } = makeHarness(dirFiles, fileContents); + await hook.execute(); + + expect(putIndexSettings).not.toHaveBeenCalled(); + expect(warnCalls.some(m => m.includes("segment-0-indexes.json"))).toBe(true); + }); + + it("warns and skips a file that readFile cannot read (returns null)", async () => { + const dirFiles = new Map([[TRANSFER_DIR, ["segment-0-indexes.json"]]]); + // fileContents is empty → readFile returns null + const { hook, putIndexSettings, warnCalls } = makeHarness(dirFiles); + await hook.execute(); + + expect(putIndexSettings).not.toHaveBeenCalled(); + expect(warnCalls.some(m => m.includes("segment-0-indexes.json"))).toBe(true); + }); + + it("warns and skips a file that contains invalid JSON", async () => { + const filePath = join(TRANSFER_DIR, "segment-0-indexes.json"); + const dirFiles = new Map([[TRANSFER_DIR, ["segment-0-indexes.json"]]]); + const fileContents = new Map([[filePath, "{ broken json"]]); + + const { hook, putIndexSettings, warnCalls } = makeHarness(dirFiles, fileContents); + await hook.execute(); + + expect(putIndexSettings).not.toHaveBeenCalled(); + expect(warnCalls.some(m => m.includes("segment-0-indexes.json"))).toBe(true); + }); +}); diff --git a/__tests__/features/OsProcessor/OsIndexPrefixHook.test.ts b/__tests__/features/OsProcessor/OsIndexPrefixHook.test.ts index afa81f1b..133ebb8c 100644 --- a/__tests__/features/OsProcessor/OsIndexPrefixHook.test.ts +++ b/__tests__/features/OsProcessor/OsIndexPrefixHook.test.ts @@ -30,4 +30,12 @@ describe("OsIndexPrefixHook", () => { await hook.execute(); expect(process.env.OPENSEARCH_INDEX_PREFIX).toBe(""); }); + + it("does not set OPENSEARCH_INDEX_PREFIX when OpenSearch is not configured", async () => { + delete process.env.OPENSEARCH_INDEX_PREFIX; + const container = createOsContainer({ noOpenSearch: true }); + const hook = container.resolve(BeforeTransferHook); + await hook.execute(); + expect(process.env.OPENSEARCH_INDEX_PREFIX).toBeUndefined(); + }); }); diff --git a/__tests__/features/OsProcessor/OsProcessor.test.ts b/__tests__/features/OsProcessor/OsProcessor.test.ts index 6110091b..fee0796c 100644 --- a/__tests__/features/OsProcessor/OsProcessor.test.ts +++ b/__tests__/features/OsProcessor/OsProcessor.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { Logger } from "~/tools/Logger/abstractions/Logger.ts"; import { mkdtemp, readFile, readdir } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -13,6 +13,7 @@ import { OpenSearchClient } from "~/services/OpenSearchClient/abstractions/OpenS import { CompressionHandler } from "@webiny/utils/exports/api.js"; import type { OsScanner } from "~/features/OsScanner/index.ts"; import type { BaseTransformContext } from "~/features/TransformContext/abstractions/BaseTransformContext.ts"; +import { OsProcessor } from "~/features/OsProcessor/index.ts"; import { MockOpenSearchClient } from "../../services/OpenSearchClient/MockOpenSearchClient.ts"; interface OsProcessorSlice { @@ -229,4 +230,102 @@ describe("OsProcessor", () => { await expect(readdir(transferDir)).rejects.toThrow(/ENOENT/); }); }); + + describe("checkAccess", () => { + it("returns ok when listIndexes succeeds", async () => { + const container = createOsContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === OsProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries).toHaveLength(1); + expect(entries[0]).toEqual({ + label: "OpenSearch cluster: https://es.example.com", + status: "ok" + }); + }); + + it("returns denied when listIndexes throws HTTP 403", async () => { + const container = createOsContainer(); + const osClient = container.resolve(OpenSearchClient) as MockOpenSearchClient; + vi.spyOn(osClient, "listIndexes").mockRejectedValue( + Object.assign(new Error("Forbidden"), { statusCode: 403 }) + ); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === OsProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "OpenSearch cluster: https://es.example.com", + status: "denied" + }); + }); + + it("returns denied when listIndexes throws HTTP 401", async () => { + const container = createOsContainer(); + const osClient = container.resolve(OpenSearchClient) as MockOpenSearchClient; + vi.spyOn(osClient, "listIndexes").mockRejectedValue( + Object.assign(new Error("Unauthorized"), { statusCode: 401 }) + ); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === OsProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "OpenSearch cluster: https://es.example.com", + status: "denied" + }); + }); + + it("returns missing when listIndexes throws HTTP 404", async () => { + const container = createOsContainer(); + const osClient = container.resolve(OpenSearchClient) as MockOpenSearchClient; + vi.spyOn(osClient, "listIndexes").mockRejectedValue( + Object.assign(new Error("Not found"), { statusCode: 404 }) + ); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === OsProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "OpenSearch cluster: https://es.example.com", + status: "missing" + }); + }); + + it("returns unknown when listIndexes throws a non-auth error", async () => { + const container = createOsContainer(); + const osClient = container.resolve(OpenSearchClient) as MockOpenSearchClient; + vi.spyOn(osClient, "listIndexes").mockRejectedValue(new Error("connection refused")); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === OsProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "OpenSearch cluster: https://es.example.com", + status: "unknown" + }); + }); + + it("returns empty array when OpenSearch is not configured", async () => { + const container = createOsContainer({ noOpenSearch: true }); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === OsProcessor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries).toHaveLength(0); + }); + }); }); diff --git a/__tests__/features/OsScanner/OsScanner.test.ts b/__tests__/features/OsScanner/OsScanner.test.ts index 86358e18..716fb5bb 100644 --- a/__tests__/features/OsScanner/OsScanner.test.ts +++ b/__tests__/features/OsScanner/OsScanner.test.ts @@ -159,10 +159,10 @@ describe("OsScanner", () => { await import("~/features/OsRecordDecompressor/index.ts"); const { OsScannerFeature: ScannerFeature } = await import("~/features/OsScanner/index.ts"); - // Construct a DDB-mode config and inject it into a fresh container that still - // registers OsScanner as the Scanner. When scan() is called, its guard should fire. + // Construct a DDB-only config (no opensearch) and inject it into a fresh container + // that still registers OsScanner as the Scanner. When scan() is called, its guard + // should fire because config.source.opensearch is absent. const ddbConfig = { - storage: "ddb" as const, source: { region: "us-east-1", credentials: { accessKeyId: "x", secretAccessKey: "y" }, @@ -176,7 +176,7 @@ describe("OsScanner", () => { s3: { bucket: "ddb-target-bucket" }, auditLog: null }, - pipeline: { preset: "v5-to-v6" } + pipeline: {} }; const container = new Container(); @@ -201,6 +201,6 @@ describe("OsScanner", () => { for await (const _ of scanner.scan({ segment: 0, total: 1 })) { // Should never iterate } - }).rejects.toThrow(/OS storage mode/i); + }).rejects.toThrow(/config\.source\.opensearch is not configured/i); }); }); diff --git a/__tests__/features/PipelineRunner/PipelineRunner.test.ts b/__tests__/features/PipelineRunner/PipelineRunner.test.ts index 46ba2c72..53014c64 100644 --- a/__tests__/features/PipelineRunner/PipelineRunner.test.ts +++ b/__tests__/features/PipelineRunner/PipelineRunner.test.ts @@ -15,9 +15,11 @@ import { } from "~/features/PipelineBuilderFactory/index.ts"; import { SnapshotWriter } from "~/features/SnapshotWriter/index.ts"; import { BaseTransformContextFactory } from "~/features/TransformContext/abstractions/BaseTransformContext.ts"; +import { MigrationConfig } from "~/features/MigrationConfig/abstractions/MigrationConfig.ts"; import { Commands } from "~/domain/transform/commands/Commands.ts"; import { PutRecord } from "~/domain/transform/commands/PutRecord.ts"; import { Processor, Hook, createFilter } from "~/domain/pipeline/index.ts"; +import { AccessCheck } from "~/domain/pipeline/abstractions/Processor.ts"; import type { Pipeline } from "~/domain/pipeline/index.ts"; import { Scanner } from "~/domain/pipeline/abstractions/Scanner.ts"; import type { BaseTransformContext } from "~/features/TransformContext/abstractions/BaseTransformContext.ts"; @@ -104,7 +106,7 @@ class FakeBaseContextFactory implements BaseTransformContextFactory.Interface { } } -function makeContainer(options: { runId?: string } = {}): { +function makeContainer(options: { runId?: string; flushEvery?: number } = {}): { container: Container; logger: TestLogger; } { @@ -120,6 +122,9 @@ function makeContainer(options: { runId?: string } = {}): { }); container.registerInstance(DroppedRecordLog, new MockDroppedRecordLog()); container.registerInstance(TransferredRecordLog, new MockTransferredRecordLog()); + container.registerInstance(MigrationConfig, { + tuning: options.flushEvery !== undefined ? { flushEvery: options.flushEvery } : undefined + } as unknown as MigrationConfig.Interface); container.register(FakeScannerImpl).inSingletonScope(); container.register(FakeProcessorImpl).inSingletonScope(); container.register(FakeHookAImpl).inSingletonScope(); @@ -645,6 +650,10 @@ class SecondaryFakeProcessor implements Processor.Interface< return { label: () => "secondary" }; } + public async checkAccess(): Promise { + return []; + } + public async execute(): Promise { // No-op — this processor doesn't claim any command key. } @@ -855,3 +864,121 @@ describe("PipelineRunner.run() — blackhole pipelines", () => { expect(processor.executed[0]?.size()).toBe(1); }); }); + +describe("PipelineRunner — periodic flush (flushEvery)", () => { + it("flushes mid-shard every flushEvery records", async () => { + const { container } = makeContainer({ flushEvery: 2 }); + const scanner = container.resolve(Scanner) as FakeScanner; + const processor = container.resolve(Processor) as FakeProcessor; + scanner.records = [ + { id: "r1", type: "foo" }, + { id: "r2", type: "foo" }, + { id: "r3", type: "foo" }, + { id: "r4", type: "foo" }, + { id: "r5", type: "foo" } + ]; + + const runner = container.resolve(PipelineRunner); + runner.register(buildPipeline(container, "flush-mid-shard")); + await runner.run(); + + // flushEvery=2, 5 records: flush at 2, flush at 4, final flush at 5 + expect(processor.executed).toHaveLength(3); + expect(processor.executed[0]?.size()).toBe(2); + expect(processor.executed[1]?.size()).toBe(2); + expect(processor.executed[2]?.size()).toBe(1); + }); + + it("flushes exactly N/flushEvery times when count is divisible", async () => { + const { container } = makeContainer({ flushEvery: 2 }); + const scanner = container.resolve(Scanner) as FakeScanner; + const processor = container.resolve(Processor) as FakeProcessor; + scanner.records = [ + { id: "r1", type: "foo" }, + { id: "r2", type: "foo" }, + { id: "r3", type: "foo" }, + { id: "r4", type: "foo" } + ]; + + const runner = container.resolve(PipelineRunner); + runner.register(buildPipeline(container, "flush-divisible")); + await runner.run(); + + // flushEvery=2, 4 records: flush at 2, flush at 4, no remainder + expect(processor.executed).toHaveLength(2); + expect(processor.executed[0]?.size()).toBe(2); + expect(processor.executed[1]?.size()).toBe(2); + }); + + it("no record loss across flush boundaries", async () => { + const { container } = makeContainer({ flushEvery: 2 }); + const scanner = container.resolve(Scanner) as FakeScanner; + const processor = container.resolve(Processor) as FakeProcessor; + scanner.records = [ + { id: "r1", type: "foo" }, + { id: "r2", type: "foo" }, + { id: "r3", type: "foo" } + ]; + + const runner = container.resolve(PipelineRunner); + runner.register(buildPipeline(container, "flush-no-loss")); + await runner.run(); + + const totalCommands = processor.executed.reduce((sum, c) => sum + c.size(), 0); + expect(totalCommands).toBe(3); + }); + + it("afterShard fires exactly once regardless of flush count", async () => { + const { container } = makeContainer({ flushEvery: 2 }); + const scanner = container.resolve(Scanner) as FakeScanner; + const processor = container.resolve(Processor) as FakeProcessor; + scanner.records = [ + { id: "r1", type: "foo" }, + { id: "r2", type: "foo" }, + { id: "r3", type: "foo" }, + { id: "r4", type: "foo" }, + { id: "r5", type: "foo" } + ]; + + const runner = container.resolve(PipelineRunner); + runner.register(buildPipeline(container, "flush-aftershard")); + await runner.run(); + + expect(processor.afterShardCalls).toHaveLength(1); + expect(processor.afterShardCalls[0]).toEqual({ segment: 0, totalSegments: 1 }); + }); + + it("without flushEvery set, uses a single shard-end flush (default 500 > record count)", async () => { + const { container } = makeContainer(); // no flushEvery → default 500 + const scanner = container.resolve(Scanner) as FakeScanner; + const processor = container.resolve(Processor) as FakeProcessor; + scanner.records = [ + { id: "r1", type: "foo" }, + { id: "r2", type: "foo" } + ]; + + const runner = container.resolve(PipelineRunner); + runner.register(buildPipeline(container, "flush-default")); + await runner.run(); + + // 2 records < 500 default → single execute call at shard end + expect(processor.executed).toHaveLength(1); + expect(processor.executed[0]?.size()).toBe(2); + }); + + it("calls execute() once when the shard yields zero records", async () => { + const { container } = makeContainer({ flushEvery: 2 }); + const scanner = container.resolve(Scanner) as FakeScanner; + const processor = container.resolve(Processor) as FakeProcessor; + scanner.records = []; + + const runner = container.resolve(PipelineRunner); + runner.register(buildPipeline(container, "flush-empty-shard")); + await runner.run(); + + // periodicFlushCount === 0 path: no mid-shard flush occurred, so final flush + // still fires once to honour the "execute at least once per shard" contract + expect(processor.executed).toHaveLength(1); + expect(processor.executed[0]?.size()).toBe(0); + }); +}); diff --git a/__tests__/features/PresetLifecycle/hookComposites.test.ts b/__tests__/features/PresetLifecycle/hookComposites.test.ts new file mode 100644 index 00000000..5b97d446 --- /dev/null +++ b/__tests__/features/PresetLifecycle/hookComposites.test.ts @@ -0,0 +1,82 @@ +import { describe, it, expect } from "vitest"; +import { Container } from "@webiny/di"; +import { BeforeLoadPresetHookComposite } from "~/features/PresetLifecycle/BeforeLoadPresetHookComposite.ts"; +import { AfterLoadPresetHookComposite } from "~/features/PresetLifecycle/AfterLoadPresetHookComposite.ts"; +import { + BeforeLoadPresetHook, + AfterLoadPresetHook +} from "~/features/PresetLifecycle/abstractions/PresetLifecycle.ts"; +import type { MigrationConfig } from "~/features/MigrationConfig/abstractions/MigrationConfig.ts"; +import type { MigrationPreset } from "~/domain/transform/Preset.ts"; + +const STUB_CONFIG = {} as MigrationConfig.Interface; +const STUB_PRESET = {} as MigrationPreset; + +function createContainer(): Container { + const container = new Container(); + container.registerComposite(BeforeLoadPresetHookComposite); + container.registerComposite(AfterLoadPresetHookComposite); + return container; +} + +describe("BeforeLoadPresetHookComposite", () => { + it("forwards config to all registered hooks", async () => { + const container = createContainer(); + const received: MigrationConfig.Interface[] = []; + container.registerInstance(BeforeLoadPresetHook, { + execute: async cfg => { + received.push(cfg); + } + }); + container.registerInstance(BeforeLoadPresetHook, { + execute: async cfg => { + received.push(cfg); + } + }); + + await container.resolve(BeforeLoadPresetHook).execute(STUB_CONFIG); + expect(received).toEqual([STUB_CONFIG, STUB_CONFIG]); + }); + + it("resolves without error when no hooks are registered", async () => { + const container = createContainer(); + await expect( + container.resolve(BeforeLoadPresetHook).execute(STUB_CONFIG) + ).resolves.toBeUndefined(); + }); +}); + +describe("AfterLoadPresetHookComposite", () => { + it("forwards config and preset to all registered hooks in order", async () => { + const container = createContainer(); + const order: string[] = []; + const argsA: [MigrationConfig.Interface, MigrationPreset][] = []; + const argsB: [MigrationConfig.Interface, MigrationPreset][] = []; + + container.registerInstance(AfterLoadPresetHook, { + execute: async (cfg, preset) => { + order.push("a"); + argsA.push([cfg, preset]); + } + }); + container.registerInstance(AfterLoadPresetHook, { + execute: async (cfg, preset) => { + order.push("b"); + argsB.push([cfg, preset]); + } + }); + + await container.resolve(AfterLoadPresetHook).execute(STUB_CONFIG, STUB_PRESET); + + expect(order).toEqual(["a", "b"]); + expect(argsA[0]).toEqual([STUB_CONFIG, STUB_PRESET]); + expect(argsB[0]).toEqual([STUB_CONFIG, STUB_PRESET]); + }); + + it("resolves without error when no hooks are registered", async () => { + const container = createContainer(); + await expect( + container.resolve(AfterLoadPresetHook).execute(STUB_CONFIG, STUB_PRESET) + ).resolves.toBeUndefined(); + }); +}); diff --git a/__tests__/features/S3Processor/S3Processor.test.ts b/__tests__/features/S3Processor/S3Processor.test.ts index 628646fa..b40041c6 100644 --- a/__tests__/features/S3Processor/S3Processor.test.ts +++ b/__tests__/features/S3Processor/S3Processor.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; import { Logger } from "~/tools/Logger/abstractions/Logger.ts"; import { createDdbContainer } from "../../containers/index.ts"; import { MockS3Client } from "../../services/S3Client/MockS3Client.ts"; @@ -10,6 +10,11 @@ import { Commands } from "~/domain/transform/commands/Commands.ts"; import { PutRecord } from "~/domain/transform/commands/PutRecord.ts"; import type { BaseTransformContext } from "~/features/TransformContext/abstractions/BaseTransformContext.ts"; import { CompressionHandler } from "@webiny/utils/exports/api.js"; +import { S3 } from "@webiny/aws-sdk/client-s3/index.js"; + +vi.mock("@webiny/aws-sdk/client-s3/index.js", () => ({ + S3: vi.fn() +})); interface BaseStub { base: BaseTransformContext.Interface; @@ -46,6 +51,10 @@ function makeBase(record: TRecord): BaseStub { } describe("S3Processor", () => { + afterEach(() => { + vi.resetAllMocks(); + }); + describe("extendContext", () => { it("returns copyFile that pushes S3Copy commands using configured source/target buckets", () => { const container = createDdbContainer(); @@ -155,4 +164,153 @@ describe("S3Processor", () => { expect(processor.afterShard).toBeUndefined(); }); }); + + describe("checkAccess", () => { + let mockHeadBucket: ReturnType; + + beforeEach(() => { + mockHeadBucket = vi.fn(); + vi.mocked(S3).mockImplementation(function (this: { + headBucket: typeof mockHeadBucket; + destroy: ReturnType; + }) { + this.headBucket = mockHeadBucket; + this.destroy = vi.fn(); + } as unknown as typeof S3); + }); + + it("returns ok entries for source and target buckets when HeadBucket succeeds", async () => { + mockHeadBucket.mockResolvedValue({}); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === S3Processor) as unknown as Processor.Interface< + any, + any + >; + + const entries = await processor.checkAccess(); + + expect(entries).toHaveLength(2); + expect(entries[0]).toEqual({ label: "S3 source bucket: source-bucket", status: "ok" }); + expect(entries[1]).toEqual({ label: "S3 target bucket: target-bucket", status: "ok" }); + }); + + it("returns denied when HeadBucket throws AccessDenied on source", async () => { + mockHeadBucket + .mockRejectedValueOnce( + Object.assign(new Error("Access denied"), { name: "AccessDenied" }) + ) + .mockResolvedValue({}); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === S3Processor) as unknown as Processor.Interface< + any, + any + >; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "S3 source bucket: source-bucket", + status: "denied" + }); + expect(entries[1]).toEqual({ label: "S3 target bucket: target-bucket", status: "ok" }); + }); + + it("returns denied when HeadBucket returns HTTP 403", async () => { + mockHeadBucket.mockRejectedValue( + Object.assign(new Error("Forbidden"), { $metadata: { httpStatusCode: 403 } }) + ); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === S3Processor) as unknown as Processor.Interface< + any, + any + >; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "S3 source bucket: source-bucket", + status: "denied" + }); + expect(entries[1]).toEqual({ + label: "S3 target bucket: target-bucket", + status: "denied" + }); + }); + + it("returns missing when HeadBucket throws NoSuchBucket", async () => { + mockHeadBucket.mockRejectedValue( + Object.assign(new Error("Bucket not found"), { name: "NoSuchBucket" }) + ); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === S3Processor) as unknown as Processor.Interface< + any, + any + >; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "S3 source bucket: source-bucket", + status: "missing" + }); + expect(entries[1]).toEqual({ + label: "S3 target bucket: target-bucket", + status: "missing" + }); + }); + + it("returns missing when HeadBucket returns HTTP 404", async () => { + mockHeadBucket.mockRejectedValue( + Object.assign(new Error("Not found"), { $metadata: { httpStatusCode: 404 } }) + ); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === S3Processor) as unknown as Processor.Interface< + any, + any + >; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "S3 source bucket: source-bucket", + status: "missing" + }); + expect(entries[1]).toEqual({ + label: "S3 target bucket: target-bucket", + status: "missing" + }); + }); + + it("returns unknown for other errors", async () => { + mockHeadBucket.mockRejectedValue(new Error("connection refused")); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === S3Processor) as unknown as Processor.Interface< + any, + any + >; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "S3 source bucket: source-bucket", + status: "unknown" + }); + expect(entries[1]).toEqual({ + label: "S3 target bucket: target-bucket", + status: "unknown" + }); + }); + }); }); diff --git a/__tests__/features/SnapshotWriter/SnapshotWriter.test.ts b/__tests__/features/SnapshotWriter/SnapshotWriter.test.ts index 15007b7c..4b6f3530 100644 --- a/__tests__/features/SnapshotWriter/SnapshotWriter.test.ts +++ b/__tests__/features/SnapshotWriter/SnapshotWriter.test.ts @@ -60,7 +60,6 @@ function buildContainer(options: BuildOptions = {}): { const logger = new CapturingLogger(); container.registerInstance(Logger, logger); const config: MigrationConfig.Interface = { - storage: "ddb", source: { region: "us-east-1", credentials: { accessKeyId: "x", secretAccessKey: "y" }, @@ -74,7 +73,7 @@ function buildContainer(options: BuildOptions = {}): { s3: { bucket: "t-b" }, auditLog: null }, - pipeline: { preset: "noop" }, + pipeline: { segments: 1 }, debug: options.snapshot !== undefined ? { snapshot: options.snapshot } : undefined }; container.registerInstance(MigrationConfig, config); diff --git a/__tests__/features/TransferLifecycle/hookComposites.test.ts b/__tests__/features/TransferLifecycle/hookComposites.test.ts new file mode 100644 index 00000000..0b8c7575 --- /dev/null +++ b/__tests__/features/TransferLifecycle/hookComposites.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect, vi } from "vitest"; +import { Container } from "@webiny/di"; +import { TransferLifecycleFeature } from "~/features/TransferLifecycle/feature.ts"; +import { + BeforeTransferHook, + AfterTransferHook +} from "~/features/TransferLifecycle/abstractions/TransferLifecycle.ts"; + +function createContainer(): Container { + const container = new Container(); + TransferLifecycleFeature.register(container); + return container; +} + +describe("BeforeTransferHookComposite", () => { + it("calls all registered hooks", async () => { + const container = createContainer(); + const calls: number[] = []; + container.registerInstance(BeforeTransferHook, { + execute: async () => { + calls.push(1); + } + }); + container.registerInstance(BeforeTransferHook, { + execute: async () => { + calls.push(2); + } + }); + + await container.resolve(BeforeTransferHook).execute(); + expect(calls).toEqual([1, 2]); + }); + + it("calls hooks in registration order", async () => { + const container = createContainer(); + const order: string[] = []; + container.registerInstance(BeforeTransferHook, { + execute: async () => { + order.push("first"); + } + }); + container.registerInstance(BeforeTransferHook, { + execute: async () => { + order.push("second"); + } + }); + container.registerInstance(BeforeTransferHook, { + execute: async () => { + order.push("third"); + } + }); + + await container.resolve(BeforeTransferHook).execute(); + expect(order).toEqual(["first", "second", "third"]); + }); + + it("resolves without error when no hooks are registered", async () => { + const container = createContainer(); + await expect(container.resolve(BeforeTransferHook).execute()).resolves.toBeUndefined(); + }); +}); + +describe("AfterTransferHookComposite", () => { + it("calls all registered hooks", async () => { + const container = createContainer(); + const fn1 = vi.fn(); + const fn2 = vi.fn(); + container.registerInstance(AfterTransferHook, { execute: fn1 }); + container.registerInstance(AfterTransferHook, { execute: fn2 }); + + await container.resolve(AfterTransferHook).execute(); + expect(fn1).toHaveBeenCalledOnce(); + expect(fn2).toHaveBeenCalledOnce(); + }); + + it("resolves without error when no hooks are registered", async () => { + const container = createContainer(); + await expect(container.resolve(AfterTransferHook).execute()).resolves.toBeUndefined(); + }); +}); diff --git a/__tests__/fixtures/wizard/ddb.config.ts b/__tests__/fixtures/wizard/ddb.config.ts index c77e6156..7e62ab01 100644 --- a/__tests__/fixtures/wizard/ddb.config.ts +++ b/__tests__/fixtures/wizard/ddb.config.ts @@ -1 +1,15 @@ -export default { storage: "ddb" as const }; +export default { + source: { + region: "eu-central-1", + credentials: { accessKeyId: "AKIA", secretAccessKey: "secret" }, + dynamodb: { tableName: "src-table" }, + s3: { bucket: "src-bucket" } + }, + target: { + region: "us-east-1", + credentials: { accessKeyId: "AKIA", secretAccessKey: "secret" }, + dynamodb: { tableName: "tgt-table" }, + s3: { bucket: "tgt-bucket" } + }, + pipeline: {} +}; diff --git a/__tests__/fixtures/wizard/os.config.ts b/__tests__/fixtures/wizard/os.config.ts deleted file mode 100644 index 29c75f6c..00000000 --- a/__tests__/fixtures/wizard/os.config.ts +++ /dev/null @@ -1 +0,0 @@ -export default { storage: "os" as const }; diff --git a/__tests__/integration/integrationContainer.ts b/__tests__/integration/integrationContainer.ts index 049d2726..ae1cf529 100644 --- a/__tests__/integration/integrationContainer.ts +++ b/__tests__/integration/integrationContainer.ts @@ -71,7 +71,6 @@ export interface DdbIntegrationContainerOptions { */ export function createDdbIntegrationContainer(options: DdbIntegrationContainerOptions): Container { const config: MigrationConfig.Interface = { - storage: "ddb", source: { region: "us-east-1", credentials: FAKE_CREDS, @@ -90,7 +89,6 @@ export function createDdbIntegrationContainer(options: DdbIntegrationContainerOp auditLog: null }, pipeline: { - preset: "integration", segments: options.segments ?? 1, modelsDir: options.modelsDir }, diff --git a/__tests__/integration/pipeline.realData.test.ts b/__tests__/integration/pipeline.realData.test.ts index 6376e4e0..730fb4a5 100644 --- a/__tests__/integration/pipeline.realData.test.ts +++ b/__tests__/integration/pipeline.realData.test.ts @@ -19,8 +19,8 @@ async function loadFixture(path: string): Promise { return JSON.parse(raw) as BaseRecord[]; } -async function createDdbTable(doc: DynamoDBDocument, tableName: string): Promise { - await doc.send( +async function createDdbTable(client: DynamoDBClient, tableName: string): Promise { + await client.send( new CreateTableCommand({ TableName: tableName, BillingMode: "PAY_PER_REQUEST", @@ -73,12 +73,13 @@ function indexByPkSk(records: BaseRecord[]): Map { describe("pipeline — real-world data transfer against dynalite", () => { let instance: DynaliteInstance; + let client: DynamoDBClient; let doc: DynamoDBDocument; let fixture: BaseRecord[]; beforeAll(async () => { instance = await startDynalite(); - const client = new DynamoDBClient({ + client = new DynamoDBClient({ endpoint: instance.endpoint, region: "us-east-1", credentials: FAKE_CREDS @@ -94,8 +95,8 @@ describe("pipeline — real-world data transfer against dynalite", () => { it("roundtrips every record in __tests__/data/small-one.json byte-exact via the pipeline", async () => { const source = "real-small-src"; const target = "real-small-tgt"; - await createDdbTable(doc, source); - await createDdbTable(doc, target); + await createDdbTable(client, source); + await createDdbTable(client, target); await seedRecords(doc, source, fixture); // Sanity check: dynalite actually stored what we seeded. diff --git a/__tests__/services/S3Client/MockS3Client.ts b/__tests__/services/S3Client/MockS3Client.ts index 4d6bda2d..7c82a07f 100644 --- a/__tests__/services/S3Client/MockS3Client.ts +++ b/__tests__/services/S3Client/MockS3Client.ts @@ -26,7 +26,7 @@ export class MockS3Client implements SourceS3Client.Interface { return stored; } - // Test helpers + // Test helper — seed objects into the in-memory store. public putObject(bucket: string, key: string, data: Buffer): void { this.objects.set(`${bucket}/${key}`, data); } diff --git a/__tests__/transformers/cms/fieldUtils.test.ts b/__tests__/transformers/cms/fieldUtils.test.ts new file mode 100644 index 00000000..e6b15631 --- /dev/null +++ b/__tests__/transformers/cms/fieldUtils.test.ts @@ -0,0 +1,33 @@ +import { describe, it, expect } from "vitest"; +import { getCorrectStorageId, isStorageIdCorrupt } from "~/transformers/cms/fieldUtils.ts"; +import type { ModelField } from "~/transformers/cms/modelTypes.ts"; + +function field(type: string, id: string, storageId: string): ModelField { + return { id, fieldId: id, storageId, type }; +} + +describe("getCorrectStorageId", () => { + it("returns type@id", () => { + expect(getCorrectStorageId(field("text", "title", "text@title"))).toBe("text@title"); + }); + + it("uses type prefix, not the declared storageId", () => { + expect(getCorrectStorageId(field("dynamicZone", "hero", "text@hero"))).toBe( + "dynamicZone@hero" + ); + }); +}); + +describe("isStorageIdCorrupt", () => { + it("returns false when storageId matches type@id", () => { + expect(isStorageIdCorrupt(field("text", "title", "text@title"))).toBe(false); + }); + + it("returns true when storageId uses wrong type prefix", () => { + expect(isStorageIdCorrupt(field("dynamicZone", "hero", "text@hero"))).toBe(true); + }); + + it("returns true when storageId uses the plain id without a type prefix", () => { + expect(isStorageIdCorrupt(field("object", "address", "address"))).toBe(true); + }); +}); diff --git a/__tests__/transformers/cms/fieldVisitor.test.ts b/__tests__/transformers/cms/fieldVisitor.test.ts new file mode 100644 index 00000000..23890dcc --- /dev/null +++ b/__tests__/transformers/cms/fieldVisitor.test.ts @@ -0,0 +1,178 @@ +import { describe, it, expect, vi } from "vitest"; +import { visitFields } from "~/transformers/cms/fieldVisitor.ts"; +import type { ModelField } from "~/transformers/cms/modelTypes.ts"; + +function textField(id: string): ModelField { + return { id, fieldId: id, storageId: `text@${id}`, type: "text" }; +} + +function objectField(id: string, nestedFields: ModelField[], multipleValues = false): ModelField { + return { + id, + fieldId: id, + storageId: `object@${id}`, + type: "object", + multipleValues, + settings: { fields: nestedFields } + }; +} + +function dynamicZoneField( + id: string, + templates: { id: string; fields: ModelField[] }[] +): ModelField { + return { + id, + fieldId: id, + storageId: `dynamicZone@${id}`, + type: "dynamicZone", + settings: { + templates: templates.map(t => ({ + id: t.id, + name: t.id, + fields: t.fields + })) + } + }; +} + +describe("visitFields", () => { + it("invokes callback for each top-level field with a value", async () => { + const calls: string[] = []; + const values = { "text@title": "hello", "text@body": "world" }; + await visitFields(values, [textField("title"), textField("body")], (_v, field) => { + calls.push(field.id); + }); + expect(calls).toEqual(["title", "body"]); + }); + + it("skips fields whose storageId is absent from values", async () => { + const calls: string[] = []; + await visitFields({}, [textField("title")], (_v, field) => { + calls.push(field.id); + }); + expect(calls).toHaveLength(0); + }); + + it("recurses into a single nested object", async () => { + const calls: string[] = []; + const values = { + "object@address": { + "text@street": "Main St" + } + }; + const fields = [objectField("address", [textField("street")])]; + await visitFields(values, fields, (_v, field) => { + calls.push(field.id); + }); + expect(calls).toContain("address"); + expect(calls).toContain("street"); + }); + + it("recurses into each item of an object array (multipleValues=true)", async () => { + const calls: string[] = []; + const values = { + "object@tags": [{ "text@label": "a" }, { "text@label": "b" }] + }; + const fields = [objectField("tags", [textField("label")], true)]; + await visitFields(values, fields, (_v, field) => { + calls.push(field.id); + }); + expect(calls.filter(c => c === "label")).toHaveLength(2); + }); + + it("skips array items that are not objects", async () => { + const callback = vi.fn(); + const values = { "object@tags": [null, "string", 42] }; + const fields = [objectField("tags", [textField("label")], true)]; + await visitFields(values, fields, callback); + // only the outer "tags" field itself; none of the array items recurse + expect(callback).toHaveBeenCalledTimes(1); + expect(callback.mock.calls[0][1].id).toBe("tags"); + }); + + it("does not recurse into an object field without settings.fields", async () => { + const calls: string[] = []; + const values = { "object@meta": { "text@x": "y" } }; + const bareField: ModelField = { + id: "meta", + fieldId: "meta", + storageId: "object@meta", + type: "object" + // no settings + }; + await visitFields(values, [bareField], (_v, field) => { + calls.push(field.id); + }); + expect(calls).toEqual(["meta"]); + }); + + it("recurses into a dynamicZone item whose templateId matches", async () => { + const calls: string[] = []; + const values = { + "dynamicZone@hero": [{ _templateId: "banner", "text@headline": "Hello" }] + }; + const fields = [ + dynamicZoneField("hero", [{ id: "banner", fields: [textField("headline")] }]) + ]; + await visitFields(values, fields, (_v, field) => { + calls.push(field.id); + }); + expect(calls).toContain("hero"); + expect(calls).toContain("headline"); + }); + + it("does not recurse into a dynamicZone item with an unknown templateId", async () => { + const calls: string[] = []; + const values = { + "dynamicZone@hero": [{ _templateId: "unknown", "text@headline": "Hello" }] + }; + const fields = [ + dynamicZoneField("hero", [{ id: "banner", fields: [textField("headline")] }]) + ]; + await visitFields(values, fields, (_v, field) => { + calls.push(field.id); + }); + expect(calls).toEqual(["hero"]); + }); + + it("handles a scalar (non-array) dynamicZone value", async () => { + const calls: string[] = []; + const values = { + "dynamicZone@hero": { _templateId: "banner", "text@headline": "Hello" } + }; + const fields = [ + dynamicZoneField("hero", [{ id: "banner", fields: [textField("headline")] }]) + ]; + await visitFields(values, fields, (_v, field) => { + calls.push(field.id); + }); + expect(calls).toContain("hero"); + expect(calls).toContain("headline"); + }); + + it("does not recurse into a dynamicZone field without templates", async () => { + const calls: string[] = []; + const values = { "dynamicZone@zone": [{ _templateId: "t1" }] }; + const bareZone: ModelField = { + id: "zone", + fieldId: "zone", + storageId: "dynamicZone@zone", + type: "dynamicZone" + // no settings + }; + await visitFields(values, [bareZone], (_v, field) => { + calls.push(field.id); + }); + expect(calls).toEqual(["zone"]); + }); + + it("skips null and non-object dynamicZone items", async () => { + const callback = vi.fn(); + const values = { "dynamicZone@zone": [null, "oops"] }; + const fields = [dynamicZoneField("zone", [{ id: "t1", fields: [textField("x")] }])]; + await visitFields(values, fields, callback); + expect(callback).toHaveBeenCalledTimes(1); + expect(callback.mock.calls[0][1].id).toBe("zone"); + }); +}); diff --git a/__tests__/transformers/file-manager/copyFileToTarget.test.ts b/__tests__/transformers/file-manager/copyFileToTarget.test.ts new file mode 100644 index 00000000..887ac6b9 --- /dev/null +++ b/__tests__/transformers/file-manager/copyFileToTarget.test.ts @@ -0,0 +1,64 @@ +import { describe, it, expect } from "vitest"; +import { copyFileToTarget } from "~/transformers/file-manager/copyFileToTarget.ts"; +import { makeFakeBaseContext } from "../fakeContext.ts"; +import type { DdbTransformContext } from "~/features/TransformContext/abstractions/contextAliases.ts"; + +function makeCtx(record: Record): { + ctx: DdbTransformContext.Interface; + copyFileCalls: [string, string][]; +} { + const copyFileCalls: [string, string][] = []; + const base = makeFakeBaseContext(record); + const ctx = base as unknown as DdbTransformContext.Interface & { + copyFile(src: string, dst: string): void; + }; + ctx.copyFile = (src, dst) => { + copyFileCalls.push([src, dst]); + }; + return { ctx, copyFileCalls }; +} + +describe("copyFileToTarget", () => { + it("emits a same-key S3 copy for a raw v5 record (values at root)", async () => { + const { ctx, copyFileCalls } = makeCtx({ + PK: "T#root#FM#F#abc", + SK: "A", + TYPE: "fm.file", + values: { "text@key": "uploads/abc.jpg" } + }); + await copyFileToTarget(ctx); + expect(copyFileCalls).toEqual([["uploads/abc.jpg", "uploads/abc.jpg"]]); + }); + + it("emits a same-key S3 copy for a post-wrapInData record (values under data)", async () => { + const { ctx, copyFileCalls } = makeCtx({ + PK: "T#root#FM#F#abc", + SK: "A", + TYPE: "fm.file", + data: { values: { "text@key": "uploads/abc.jpg" } } + }); + await copyFileToTarget(ctx); + expect(copyFileCalls).toEqual([["uploads/abc.jpg", "uploads/abc.jpg"]]); + }); + + it("does not emit any copy when text@key is absent", async () => { + const { ctx, copyFileCalls } = makeCtx({ + PK: "T#root#FM#F#abc", + SK: "A", + TYPE: "fm.file", + data: { values: { "text@type": "image/png" } } + }); + await copyFileToTarget(ctx); + expect(copyFileCalls).toHaveLength(0); + }); + + it("does not emit any copy when neither values nor data.values is present", async () => { + const { ctx, copyFileCalls } = makeCtx({ + PK: "T#root#FM#F#abc", + SK: "A", + TYPE: "fm.file" + }); + await copyFileToTarget(ctx); + expect(copyFileCalls).toHaveLength(0); + }); +}); diff --git a/__tests__/utils/fromEnv.test.ts b/__tests__/utils/fromEnv.test.ts index fbc7170d..7519ad66 100644 --- a/__tests__/utils/fromEnv.test.ts +++ b/__tests__/utils/fromEnv.test.ts @@ -48,6 +48,20 @@ describe("fromEnv", () => { process.env[TEST_VAR] = ""; expect(() => fromEnv(TEST_VAR)).toThrow(/__TEST_FROM_ENV_VAR__/); }); + + it("returns null when defaultValue is null and the env var is absent", () => { + expect(fromEnv(TEST_VAR, null)).toBeNull(); + }); + + it("returns null when defaultValue is null and the env var is empty", () => { + process.env[TEST_VAR] = ""; + expect(fromEnv(TEST_VAR, null)).toBeNull(); + }); + + it("returns the env var value when set, even with null default", () => { + process.env[TEST_VAR] = "present"; + expect(fromEnv(TEST_VAR, null)).toBe("present"); + }); }); describe("numberFromEnv", () => { diff --git a/docs/superpowers/plans/2026-05-10-unified-config.md b/docs/superpowers/plans/2026-05-10-unified-config.md new file mode 100644 index 00000000..e8c423cf --- /dev/null +++ b/docs/superpowers/plans/2026-05-10-unified-config.md @@ -0,0 +1,2017 @@ +# Unified Config — One Config Per Project + +> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:subagent-driven-development` (recommended) or `superpowers:executing-plans` to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace `createDdbConfig` + `createOsConfig` with a single `createConfig`, remove the `storage` discriminator and `preset` from the config schema, and route all preset selection through `TransferWizard`. + +**Architecture:** Single Zod schema (`unified.schema.ts`) with required DDB+S3 fields and optional opensearch (both-or-neither cross-field rule). Bootstrap always registers all processors; only OpenSearch client is conditional on `config.target.opensearch != null`. `TransferWizard` gains a new preset-selection step and returns `{ configPath, preset }` to the run handler. Worker processes receive the preset name via a new `--preset` CLI argument. + +**Tech Stack:** Zod v3, `@inquirer/prompts`, `@webiny/di`, Vitest, Node.js 22, TypeScript path alias `~/*` = `src/` + +--- + +## File Map + +**Create:** +- `src/features/MigrationConfig/schemas/unified.schema.ts` — single Zod schema for all configs +- `src/features/MigrationConfig/createConfig.ts` — `createConfig(input)` builder +- `src/commands/run/wizard/presetDiscovery.ts` — `listAvailablePresets(presetsDir?)` for the wizard +- `templates/internal-project/config.ts` — new unified project template + +**Modify:** +- `src/features/MigrationConfig/schemas/shared.schema.ts` — remove `preset` from `pipelineSettingsSchema` +- `src/features/MigrationConfig/validation.ts` — single `MigrationConfiguration` type (no discriminated union) +- `src/features/MigrationConfig/loadConfig.ts` — drop storage guard, remove preset path resolution +- `src/features/MigrationConfig/index.ts` — update exports +- `src/bootstrap.ts` — always register S3 + all processors; OS features conditional on `config.target.opensearch` +- `src/commands/run/handler.ts` — accept `presetName`, pass `--preset` to workers, fix `logConfig` +- `src/commands/run/register.ts` — handle `WizardResult | null` from wizard +- `src/commands/run/wizard/TransferWizard.ts` — add preset-selection step; return `WizardResult | null` +- `src/commands/run/wizard/configDiscovery.ts` — simplify: look for `config.ts` only +- `src/commands/run/wizard/types.ts` — add `WizardResult` interface +- `src/commands/processSegment/register.ts` — add `--preset` option (required) +- `src/commands/processSegment/handler.ts` — use `argv.preset` instead of `config.pipeline.preset` +- `src/features/OsScanner/OsScanner.ts` — `storage !== "os"` → `!config.source.opensearch` +- `src/features/OsProcessor/OsProcessor.ts` — `storage !== "os"` → `!config.target.opensearch` +- `src/features/DdbProcessor/DdbProcessor.ts` — remove `storage !== "ddb"` guard +- `src/features/S3Processor/S3Processor.ts` — remove `storage !== "ddb"` guard +- `src/features/DdbScanner/DdbScanner.ts` — remove `storage !== "ddb"` guard +- `src/features/AuditLogProcessor/AuditLogProcessor.ts` — remove `storage === "ddb"` checks +- `src/index.ts` — export `createConfig`; remove `createDdbConfig`, `createOsConfig`, old type exports +- `projects/v5-to-v6/config.ts` — finalize with `createConfig` (sketch already exists) +- `__tests__/containers/ddb.ts` — remove `storage`, remove `preset` +- `__tests__/containers/os.ts` — remove `storage`, remove `preset`, add required `dynamodb`/`s3` to target +- `__tests__/integration/integrationContainer.ts` — remove `storage`, remove `preset` +- `__tests__/bootstrap.test.ts` — rewrite +- `__tests__/features/MigrationConfig/createConfig.test.ts` — rewrite +- `__tests__/features/MigrationConfig/MigrationConfig.test.ts` — rewrite (loadConfig tests) +- `__tests__/commands/run/wizard/TransferWizard.test.ts` — update for preset step +- `__tests__/commands/run/wizard/configDiscovery.test.ts` — rewrite for `config.ts`-only discovery +- `__tests__/commands/processSegment.test.ts` — add `preset` to args, fix mock config +- `__tests__/fixtures/wizard/ddb.config.ts` — replace content (becomes a unified config fixture) + +**Delete:** +- `src/features/MigrationConfig/createDdbConfig.ts` +- `src/features/MigrationConfig/createOsConfig.ts` +- `src/features/MigrationConfig/schemas/ddb.schema.ts` +- `src/features/MigrationConfig/schemas/os.schema.ts` +- `templates/internal-project/ddb.transfer.config.ts` +- `templates/internal-project/os.transfer.config.ts` +- `__tests__/fixtures/wizard/os.config.ts` + +--- + +## Task 1: Unified schema + `createConfig` + test containers + +This is the foundation. After this task `MigrationConfig.Interface` no longer has `storage` or `pipeline.preset`. All downstream compilation errors are expected until later tasks fix them. + +**Files:** +- Create: `src/features/MigrationConfig/schemas/unified.schema.ts` +- Create: `src/features/MigrationConfig/createConfig.ts` +- Modify: `src/features/MigrationConfig/schemas/shared.schema.ts` +- Modify: `src/features/MigrationConfig/validation.ts` +- Modify: `src/features/MigrationConfig/index.ts` +- Modify: `__tests__/containers/ddb.ts` +- Modify: `__tests__/containers/os.ts` +- Modify: `__tests__/integration/integrationContainer.ts` +- Rewrite: `__tests__/features/MigrationConfig/createConfig.test.ts` +- Delete: `src/features/MigrationConfig/schemas/ddb.schema.ts` +- Delete: `src/features/MigrationConfig/schemas/os.schema.ts` +- Delete: `src/features/MigrationConfig/createDdbConfig.ts` +- Delete: `src/features/MigrationConfig/createOsConfig.ts` + +- [ ] **Step 1: Write the failing tests for `createConfig`** + +Replace `__tests__/features/MigrationConfig/createConfig.test.ts` entirely: + +```typescript +import { describe, it, expect } from "vitest"; +import { createConfig } from "../../../src/features/MigrationConfig/createConfig.ts"; + +const creds = { accessKeyId: "AKIA", secretAccessKey: "secret" }; + +const baseSource = { + region: "us-east-1", + credentials: creds, + dynamodb: { tableName: "src-table" }, + s3: { bucket: "src-bucket" } +}; + +const baseTarget = { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "tgt-table" }, + s3: { bucket: "tgt-bucket" } +}; + +describe("createConfig — happy path", () => { + it("returns a config with required fields, no storage field", () => { + const config = createConfig({ source: baseSource, target: baseTarget, pipeline: {} }); + expect(config.source.dynamodb.tableName).toBe("src-table"); + expect(config.target.s3.bucket).toBe("tgt-bucket"); + expect((config as any).storage).toBeUndefined(); + expect((config as any).pipeline?.preset).toBeUndefined(); + }); + + it("accepts optional opensearch on both sides", () => { + const config = createConfig({ + source: { ...baseSource, opensearch: { tableName: "src-os" } }, + target: { + ...baseTarget, + opensearch: { + endpoint: "https://search-x.es.amazonaws.com", + tableName: "tgt-os", + service: "opensearch", + indexPrefix: "" + } + }, + pipeline: {} + }); + expect(config.source.opensearch?.tableName).toBe("src-os"); + expect(config.target.opensearch?.endpoint).toBe("https://search-x.es.amazonaws.com"); + }); + + it("accepts opensearch-serverless service", () => { + const config = createConfig({ + source: { ...baseSource, opensearch: { tableName: "src-os" } }, + target: { + ...baseTarget, + opensearch: { + endpoint: "https://xxx.aoss.amazonaws.com", + tableName: "tgt-os", + service: "opensearch-serverless", + indexPrefix: "" + } + }, + pipeline: {} + }); + expect(config.target.opensearch?.service).toBe("opensearch-serverless"); + }); + + it("accepts optional auditLog", () => { + const config = createConfig({ + source: baseSource, + target: { ...baseTarget, auditLog: { dynamodb: { tableName: "audit-table" } } }, + pipeline: {} + }); + expect(config.target.auditLog?.dynamodb?.tableName).toBe("audit-table"); + }); + + it("accepts nullable auditLog (null = skip)", () => { + const config = createConfig({ + source: baseSource, + target: { ...baseTarget, auditLog: null }, + pipeline: {} + }); + expect(config.target.auditLog).toBeNull(); + }); + + it("trims whitespace from string fields", () => { + const config = createConfig({ + source: { ...baseSource, region: " us-east-1 ", dynamodb: { tableName: " src " }, s3: { bucket: " src-b " } }, + target: { ...baseTarget, region: " eu-central-1 " }, + pipeline: {} + }); + expect(config.source.region).toBe("us-east-1"); + expect(config.source.dynamodb.tableName).toBe("src"); + expect(config.source.s3.bucket).toBe("src-b"); + expect(config.target.region).toBe("eu-central-1"); + }); + + it("accepts optional segments / modelsDir / presetsDir in pipeline", () => { + const config = createConfig({ + source: baseSource, + target: baseTarget, + pipeline: { segments: 8, modelsDir: "./models", presetsDir: "./presets" } + }); + expect(config.pipeline?.segments).toBe(8); + expect(config.pipeline?.modelsDir).toBe("./models"); + }); +}); + +describe("createConfig — validation errors", () => { + it("throws on missing source region", () => { + expect(() => + createConfig({ source: { ...baseSource, region: "" } as any, target: baseTarget, pipeline: {} }) + ).toThrow(); + }); + + it("throws on whitespace-only table name", () => { + expect(() => + createConfig({ + source: { ...baseSource, dynamodb: { tableName: " " } }, + target: baseTarget, + pipeline: {} + }) + ).toThrow(); + }); + + it("throws on missing credentials", () => { + expect(() => + createConfig({ source: { ...baseSource, credentials: undefined as any }, target: baseTarget, pipeline: {} }) + ).toThrow(); + }); + + it("throws when only source.opensearch is set (target must match)", () => { + expect(() => + createConfig({ + source: { ...baseSource, opensearch: { tableName: "src-os" } }, + target: baseTarget, + pipeline: {} + }) + ).toThrow(/both be set or both be absent/); + }); + + it("throws when only target.opensearch is set", () => { + expect(() => + createConfig({ + source: baseSource, + target: { + ...baseTarget, + opensearch: { + endpoint: "https://es.example.com", + tableName: "tgt-os", + service: "opensearch", + indexPrefix: "" + } + }, + pipeline: {} + }) + ).toThrow(/both be set or both be absent/); + }); + + it("throws on same S3 bucket for source and target", () => { + expect(() => + createConfig({ + source: baseSource, + target: { ...baseTarget, s3: { bucket: baseSource.s3.bucket } }, + pipeline: {} + }) + ).toThrow(/same as source/); + }); + + it("throws on same region + same DDB table", () => { + expect(() => + createConfig({ + source: baseSource, + target: { ...baseTarget, region: baseSource.region, dynamodb: { tableName: baseSource.dynamodb.tableName } }, + pipeline: {} + }) + ).toThrow(/matches source/); + }); + + it("accepts same DDB table across different regions", () => { + expect(() => + createConfig({ + source: baseSource, + target: { ...baseTarget, dynamodb: { tableName: baseSource.dynamodb.tableName } }, + pipeline: {} + }) + ).not.toThrow(); + }); + + it("throws on same region + same OS table when opensearch present", () => { + expect(() => + createConfig({ + source: { ...baseSource, opensearch: { tableName: "same-os" } }, + target: { + ...baseTarget, + region: baseSource.region, + opensearch: { + endpoint: "https://es.example.com", + tableName: "same-os", + service: "opensearch", + indexPrefix: "" + } + }, + pipeline: {} + }) + ).toThrow(/matches source/); + }); + + it("throws on auditLog table matching main target table", () => { + expect(() => + createConfig({ + source: baseSource, + target: { + ...baseTarget, + auditLog: { dynamodb: { tableName: baseTarget.dynamodb.tableName } } + }, + pipeline: {} + }) + ).toThrow(/must differ/); + }); + + it("throws on invalid opensearch endpoint URL", () => { + expect(() => + createConfig({ + source: { ...baseSource, opensearch: { tableName: "src-os" } }, + target: { + ...baseTarget, + opensearch: { + endpoint: "not-a-url", + tableName: "tgt-os", + service: "opensearch", + indexPrefix: "" + } + }, + pipeline: {} + }) + ).toThrow(); + }); + + it("collision guard runs on trimmed values", () => { + expect(() => + createConfig({ + source: baseSource, + target: { + ...baseTarget, + region: baseSource.region, + dynamodb: { tableName: "src-table " } + }, + pipeline: {} + }) + ).toThrow(/matches source/); + }); +}); +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test __tests__/features/MigrationConfig/createConfig.test.ts 2>&1 | tail -20 +``` + +Expected: FAIL — `createConfig` does not exist yet. + +- [ ] **Step 3: Create `src/features/MigrationConfig/schemas/unified.schema.ts`** + +```typescript +import { z } from "zod"; +import { + credentialsOrProviderSchema, + debugSettingsSchema, + pipelineSettingsSchema, + trimmedString, + tuningSchema +} from "./shared.schema.ts"; + +const opensearchSourceSchema = z.object({ + tableName: trimmedString() +}); + +const opensearchTargetSchema = z.object({ + endpoint: trimmedString().url(), + tableName: trimmedString(), + service: z.enum(["opensearch", "opensearch-serverless"]), + indexPrefix: z.string().trim() +}); + +const sourceSchema = z.object({ + region: trimmedString(), + credentials: credentialsOrProviderSchema, + dynamodb: z.object({ tableName: trimmedString() }), + s3: z.object({ bucket: trimmedString() }), + opensearch: opensearchSourceSchema.nullable().optional() +}); + +const targetSchema = z.object({ + region: trimmedString(), + credentials: credentialsOrProviderSchema, + dynamodb: z.object({ tableName: trimmedString() }), + s3: z.object({ bucket: trimmedString() }), + opensearch: opensearchTargetSchema.nullable().optional(), + auditLog: z + .object({ + dynamodb: z.object({ tableName: trimmedString().nullable() }) + }) + .nullable() + .optional() +}); + +export const unifiedTransferInputSchema = z + .object({ + source: sourceSchema, + target: targetSchema, + pipeline: pipelineSettingsSchema, + tuning: tuningSchema, + debug: debugSettingsSchema + }) + .superRefine((data, ctx) => { + if (data.source.s3.bucket === data.target.s3.bucket) { + ctx.addIssue({ + code: "custom", + path: ["target", "s3", "bucket"], + message: `Target S3 bucket "${data.target.s3.bucket}" is the same as source — would overwrite source files. Use a different bucket.` + }); + } + + if ( + data.source.region === data.target.region && + data.source.dynamodb.tableName === data.target.dynamodb.tableName + ) { + ctx.addIssue({ + code: "custom", + path: ["target", "dynamodb", "tableName"], + message: `Target DynamoDB table "${data.target.dynamodb.tableName}" in region "${data.target.region}" matches source. If these are different AWS accounts, rename one or change the target region to make the intent explicit.` + }); + } + + if ( + data.target.auditLog?.dynamodb?.tableName != null && + data.target.auditLog.dynamodb.tableName === data.target.dynamodb.tableName + ) { + ctx.addIssue({ + code: "custom", + path: ["target", "auditLog", "dynamodb", "tableName"], + message: `Audit log DynamoDB table "${data.target.auditLog.dynamodb.tableName}" must differ from the main target table.` + }); + } + + const hasSourceOs = data.source.opensearch != null; + const hasTargetOs = data.target.opensearch != null; + if (hasSourceOs !== hasTargetOs) { + ctx.addIssue({ + code: "custom", + path: hasSourceOs ? ["target", "opensearch"] : ["source", "opensearch"], + message: "source.opensearch and target.opensearch must both be set or both be absent." + }); + } + + if ( + hasSourceOs && + hasTargetOs && + data.source.region === data.target.region && + data.source.opensearch!.tableName === data.target.opensearch!.tableName + ) { + ctx.addIssue({ + code: "custom", + path: ["target", "opensearch", "tableName"], + message: `Target OpenSearch DDB table "${data.target.opensearch!.tableName}" in region "${data.target.region}" matches source. If these are different AWS accounts, rename one or change the target region to make the intent explicit.` + }); + } + }); + +export type UnifiedConfigInput = z.infer; +``` + +- [ ] **Step 4: Update `src/features/MigrationConfig/schemas/shared.schema.ts` — remove `preset` from `pipelineSettingsSchema`** + +Change: +```typescript +export const pipelineSettingsSchema = z.object({ + preset: trimmedString(), + segments: z.number().int().positive().optional(), + modelsDir: trimmedString().optional(), + presetsDir: trimmedString().optional() +}); +``` +To: +```typescript +export const pipelineSettingsSchema = z.object({ + segments: z.number().int().positive().optional(), + modelsDir: trimmedString().optional(), + presetsDir: trimmedString().optional() +}); +``` + +- [ ] **Step 5: Rewrite `src/features/MigrationConfig/validation.ts`** + +```typescript +import { z } from "zod"; +import { unifiedTransferInputSchema } from "./schemas/unified.schema.ts"; + +export const migrationConfigSchema = unifiedTransferInputSchema; +export type MigrationConfiguration = z.infer; +``` + +- [ ] **Step 6: Create `src/features/MigrationConfig/createConfig.ts`** + +```typescript +import { unifiedTransferInputSchema, type UnifiedConfigInput } from "./schemas/unified.schema.ts"; +import type { MigrationConfiguration } from "./validation.ts"; + +export function createConfig(input: UnifiedConfigInput): MigrationConfiguration { + return unifiedTransferInputSchema.parse(input); +} +``` + +- [ ] **Step 7: Update `src/features/MigrationConfig/index.ts`** + +Read the current file and replace all exports. The new file must export `createConfig` and remove `createDdbConfig`, `createOsConfig`, `DdbMigrationConfiguration`, `OsMigrationConfiguration`. Keep `MigrationConfig` abstraction, `MigrationConfigFeature`, `loadConfig`, `migrationConfigSchema`, `MigrationConfiguration`. + +The file currently imports from `createDdbConfig.ts` and `createOsConfig.ts`. Replace with `createConfig.ts`: + +```typescript +export { createConfig } from "./createConfig.ts"; +export { loadConfig } from "./loadConfig.ts"; +export { MigrationConfigFeature } from "./feature.ts"; +export { MigrationConfig } from "./abstractions/MigrationConfig.ts"; +export { migrationConfigSchema } from "./validation.ts"; +export type { MigrationConfiguration } from "./validation.ts"; +``` + +(Read the current `index.ts` first to ensure you keep any other exports that should remain.) + +- [ ] **Step 8: Delete old schema files and builders** + +```bash +rm /Users/brunozoric/work/webiny/webiny-v5-to-v6/src/features/MigrationConfig/schemas/ddb.schema.ts +rm /Users/brunozoric/work/webiny/webiny-v5-to-v6/src/features/MigrationConfig/schemas/os.schema.ts +rm /Users/brunozoric/work/webiny/webiny-v5-to-v6/src/features/MigrationConfig/createDdbConfig.ts +rm /Users/brunozoric/work/webiny/webiny-v5-to-v6/src/features/MigrationConfig/createOsConfig.ts +``` + +- [ ] **Step 9: Update `__tests__/containers/ddb.ts` — remove `storage` and `preset`** + +In the `config` object inside `createDdbContainer`, change: +```typescript +const config: MigrationConfig.Interface = { + storage: "ddb", + source: { ... }, + target: { ... }, + pipeline: { + preset: "v5-to-v6", + modelsDir: options.modelsDir, + presetsDir: options.presetsDir, + ... + } +}; +``` +To: +```typescript +const config: MigrationConfig.Interface = { + source: { + region: "us-east-1", + credentials: DEFAULT_CREDS, + dynamodb: { tableName: "source-table" }, + s3: { bucket: "source-bucket" } + }, + target: { + region: "eu-central-1", + credentials: DEFAULT_CREDS, + dynamodb: { tableName: "target-table" }, + s3: { bucket: "target-bucket" }, + auditLog: null + }, + pipeline: { + modelsDir: options.modelsDir, + presetsDir: options.presetsDir, + ...(options.pipelineOverride?.segments !== undefined + ? { segments: options.pipelineOverride.segments } + : {}) + } +}; +``` + +- [ ] **Step 10: Update `__tests__/containers/os.ts` — remove `storage`, `preset`; add required `dynamodb`+`s3` to target** + +```typescript +const config: MigrationConfig.Interface = { + source: { + region: "us-east-1", + credentials: DEFAULT_CREDS, + dynamodb: { tableName: "source-primary" }, + s3: { bucket: "source-bucket" }, + opensearch: { tableName: "source-os" } + }, + target: { + region: "eu-central-1", + credentials: DEFAULT_CREDS, + dynamodb: { tableName: "target-table" }, + s3: { bucket: "target-bucket" }, + opensearch: { + endpoint: "https://es.example.com", + tableName: "target-os", + service: "opensearch" as const, + indexPrefix: options.indexPrefix ?? "" + } + }, + pipeline: { + modelsDir: options.modelsDir, + presetsDir: options.presetsDir, + ...(options.pipelineOverride?.segments !== undefined + ? { segments: options.pipelineOverride.segments } + : {}) + } +}; +``` + +- [ ] **Step 11: Update `__tests__/integration/integrationContainer.ts` — remove `storage` and `preset`** + +Find lines with `storage: "ddb"` and `preset: "integration"` in the config object and remove them. The pipeline section should not have a `preset` field. + +- [ ] **Step 12: Run the target tests to verify they pass** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test __tests__/features/MigrationConfig/createConfig.test.ts 2>&1 | tail -20 +``` + +Expected: all tests PASS. + +- [ ] **Step 13: Commit** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && git add -p && git commit -m "$(cat <<'EOF' +feat: unified config schema — createConfig replaces createDdbConfig/createOsConfig + +Co-Authored-By: Claude Sonnet 4.6 +EOF +)" +``` + +--- + +## Task 2: Update `loadConfig` + +**Files:** +- Modify: `src/features/MigrationConfig/loadConfig.ts` +- Rewrite: `__tests__/features/MigrationConfig/MigrationConfig.test.ts` + +- [ ] **Step 1: Write failing tests** + +Replace `__tests__/features/MigrationConfig/MigrationConfig.test.ts`: + +```typescript +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { Container } from "@webiny/di"; +import { writeFileSync, mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + MigrationConfig, + MigrationConfigFeature, + loadConfig +} from "../../../src/features/MigrationConfig/index.ts"; + +describe("loadConfig", () => { + let tmpDir: string; + + beforeEach(() => { tmpDir = mkdtempSync(join(tmpdir(), "mc-test-")); }); + afterEach(() => { rmSync(tmpDir, { recursive: true, force: true }); }); + + const creds = { accessKeyId: "AKIA", secretAccessKey: "secret" }; + + function writeConfig(config: object): string { + const p = join(tmpDir, "config.ts"); + writeFileSync(p, `export default ${JSON.stringify(config, null, 2)};`); + return p; + } + + it("loads a valid unified config", async () => { + const p = writeConfig({ + source: { region: "eu-central-1", credentials: creds, dynamodb: { tableName: "src" }, s3: { bucket: "src-b" } }, + target: { region: "eu-central-1", credentials: creds, dynamodb: { tableName: "tgt" }, s3: { bucket: "tgt-b" } }, + pipeline: {} + }); + const config = await loadConfig(p); + expect(config.source.dynamodb.tableName).toBe("src"); + expect((config as any).storage).toBeUndefined(); + }); + + it("loads a config with opensearch fields", async () => { + const p = writeConfig({ + source: { + region: "eu-central-1", credentials: creds, + dynamodb: { tableName: "src" }, s3: { bucket: "src-b" }, + opensearch: { tableName: "src-os" } + }, + target: { + region: "eu-central-1", credentials: creds, + dynamodb: { tableName: "tgt" }, s3: { bucket: "tgt-b" }, + opensearch: { endpoint: "https://es.example.com", tableName: "tgt-os", service: "opensearch", indexPrefix: "" } + }, + pipeline: {} + }); + const config = await loadConfig(p); + expect(config.source.opensearch?.tableName).toBe("src-os"); + }); + + it("rejects invalid config", async () => { + const p = writeConfig({ invalid: true }); + await expect(loadConfig(p)).rejects.toThrow(); + }); + + it("rejects config missing required fields", async () => { + const p = writeConfig({ source: { region: "us-east-1" } }); + await expect(loadConfig(p)).rejects.toThrow(); + }); + + it("rejects file with no default export", async () => { + const p = join(tmpDir, "config.ts"); + writeFileSync(p, "export const x = 1;"); + await expect(loadConfig(p)).rejects.toThrow(/default export/); + }); + + it("resolves presetsDir relative to config file directory", async () => { + const p = writeConfig({ + source: { region: "eu-central-1", credentials: creds, dynamodb: { tableName: "src" }, s3: { bucket: "src-b" } }, + target: { region: "eu-central-1", credentials: creds, dynamodb: { tableName: "tgt" }, s3: { bucket: "tgt-b" } }, + pipeline: { presetsDir: "./custom-presets" } + }); + const config = await loadConfig(p); + expect(config.pipeline?.presetsDir).toBe(join(tmpDir, "custom-presets")); + }); + + it("resolves modelsDir relative to config file directory", async () => { + const p = writeConfig({ + source: { region: "eu-central-1", credentials: creds, dynamodb: { tableName: "src" }, s3: { bucket: "src-b" } }, + target: { region: "eu-central-1", credentials: creds, dynamodb: { tableName: "tgt" }, s3: { bucket: "tgt-b" } }, + pipeline: { modelsDir: "./models" } + }); + const config = await loadConfig(p); + expect(config.pipeline?.modelsDir).toBe(join(tmpDir, "models")); + }); +}); + +describe("MigrationConfig DI registration", () => { + it("registers and resolves the config", async () => { + const creds = { accessKeyId: "AKIA", secretAccessKey: "secret" }; + const config = { + source: { region: "eu-central-1", credentials: creds, dynamodb: { tableName: "src" }, s3: { bucket: "src-b" } }, + target: { region: "eu-central-1", credentials: creds, dynamodb: { tableName: "tgt" }, s3: { bucket: "tgt-b" } }, + pipeline: {} + }; + const { MigrationConfiguration: _, ...rest } = await import("../../../src/features/MigrationConfig/validation.ts"); + const { migrationConfigSchema } = rest; + const parsed = migrationConfigSchema.parse(config); + const container = new Container(); + MigrationConfigFeature.register(container, { config: parsed }); + const resolved = container.resolve(MigrationConfig); + expect(resolved.source.dynamodb.tableName).toBe("src"); + const second = container.resolve(MigrationConfig); + expect(resolved).toBe(second); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test __tests__/features/MigrationConfig/MigrationConfig.test.ts 2>&1 | tail -20 +``` + +Expected: FAIL — `loadConfig` still checks `config.storage`. + +- [ ] **Step 3: Rewrite `src/features/MigrationConfig/loadConfig.ts`** + +```typescript +import { pathToFileURL } from "node:url"; +import { dirname, resolve } from "node:path"; +import { MigrationConfig } from "./abstractions/MigrationConfig.ts"; +import { migrationConfigSchema } from "./validation.ts"; + +export async function loadConfig(configPath: string): Promise { + const absolutePath = resolve(process.cwd(), configPath); + const fileUrl = pathToFileURL(absolutePath).href; + + try { + const module = await import(fileUrl); + const raw = module.default; + + if (!raw) { + throw new Error( + `Config file ${configPath} must have a default export. ` + + `Use createConfig() to create your config.` + ); + } + + const parsed = migrationConfigSchema.safeParse(raw); + if (!parsed.success) { + throw new Error( + `Invalid config in ${configPath}:\n${parsed.error.message}` + ); + } + + const config = parsed.data; + const configDir = dirname(absolutePath); + const pipeline = config.pipeline ?? {}; + + return { + ...config, + pipeline: { + ...pipeline, + ...(pipeline.modelsDir + ? { modelsDir: resolve(configDir, pipeline.modelsDir) } + : {}), + ...(pipeline.presetsDir + ? { presetsDir: resolve(configDir, pipeline.presetsDir) } + : {}) + } + }; + } catch (error) { + if (error instanceof Error) { + throw new Error(`Failed to load config from ${configPath}: ${error.message}`); + } + throw error; + } +} +``` + +- [ ] **Step 4: Run test and verify it passes** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test __tests__/features/MigrationConfig/MigrationConfig.test.ts 2>&1 | tail -20 +``` + +Expected: all PASS. + +- [ ] **Step 5: Commit** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && git add src/features/MigrationConfig/loadConfig.ts __tests__/features/MigrationConfig/MigrationConfig.test.ts && git commit -m "$(cat <<'EOF' +feat: loadConfig uses unified schema, drops storage guard + +Co-Authored-By: Claude Sonnet 4.6 +EOF +)" +``` + +--- + +## Task 3: Bootstrap — always register all features + +**Files:** +- Modify: `src/bootstrap.ts` +- Rewrite: `__tests__/bootstrap.test.ts` + +- [ ] **Step 1: Write failing tests** + +Replace `__tests__/bootstrap.test.ts`: + +```typescript +import { describe, it, expect } from "vitest"; +import { bootstrap } from "../src/bootstrap.ts"; +import { MigrationConfig } from "../src/features/MigrationConfig/index.ts"; +import { SourceDynamoDbClient, TargetDynamoDbClient } from "../src/services/DynamoDbClient/index.ts"; +import { Logger } from "../src/tools/Logger/index.ts"; +import { Cache } from "../src/tools/Cache/index.ts"; +import { ModelProvider } from "../src/features/ModelProvider/index.ts"; +import { TenantLocales } from "../src/features/TenantLocales/index.ts"; +import { SourceS3Client, TargetS3Client } from "../src/services/S3Client/index.ts"; +import { PresetLoader } from "../src/features/PresetLoader/index.ts"; +import { WorkerSpawner } from "../src/features/WorkerSpawner/index.ts"; +import { DirectoryTool } from "../src/tools/DirectoryTool/index.ts"; +import { FileTool } from "../src/tools/FileTool/index.ts"; +import { OpenSearchClient } from "../src/services/OpenSearchClient/index.ts"; + +const creds = { accessKeyId: "test", secretAccessKey: "test" }; + +const ddbOnlyConfig: MigrationConfig.Interface = { + source: { + region: "us-east-1", + credentials: creds, + dynamodb: { tableName: "source-table" }, + s3: { bucket: "source-bucket" } + }, + target: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "target-table" }, + s3: { bucket: "target-bucket" }, + auditLog: null + }, + pipeline: {} +}; + +const fullConfig: MigrationConfig.Interface = { + source: { + region: "us-east-1", + credentials: creds, + dynamodb: { tableName: "source-primary" }, + s3: { bucket: "source-bucket" }, + opensearch: { tableName: "source-os" } + }, + target: { + region: "eu-central-1", + credentials: creds, + dynamodb: { tableName: "target-table" }, + s3: { bucket: "target-bucket" }, + opensearch: { + endpoint: "https://es.example.com", + tableName: "target-os", + service: "opensearch" as const, + indexPrefix: "" + } + }, + pipeline: {} +}; + +describe("bootstrap — DDB-only config", () => { + it("resolves all core features", () => { + const container = bootstrap({ config: ddbOnlyConfig }); + expect(container.resolve(MigrationConfig)).toBeDefined(); + expect(container.resolve(Logger)).toBeDefined(); + expect(container.resolve(Cache)).toBeDefined(); + expect(container.resolve(DirectoryTool)).toBeDefined(); + expect(container.resolve(FileTool)).toBeDefined(); + expect(container.resolve(SourceDynamoDbClient)).toBeDefined(); + expect(container.resolve(TargetDynamoDbClient)).toBeDefined(); + expect(container.resolve(SourceS3Client)).toBeDefined(); + expect(container.resolve(TargetS3Client)).toBeDefined(); + expect(container.resolve(ModelProvider)).toBeDefined(); + expect(container.resolve(TenantLocales)).toBeDefined(); + expect(container.resolve(PresetLoader)).toBeDefined(); + expect(container.resolve(WorkerSpawner)).toBeDefined(); + }); + + it("does NOT register OpenSearchClient when opensearch is absent", () => { + const container = bootstrap({ config: ddbOnlyConfig }); + expect(() => container.resolve(OpenSearchClient)).toThrow(); + }); +}); + +describe("bootstrap — full config (DDB + OS)", () => { + it("resolves OpenSearchClient when target.opensearch is set", () => { + const container = bootstrap({ config: fullConfig }); + expect(container.resolve(OpenSearchClient)).toBeDefined(); + }); + + it("also resolves S3 clients in full config", () => { + const container = bootstrap({ config: fullConfig }); + expect(container.resolve(SourceS3Client)).toBeDefined(); + expect(container.resolve(TargetS3Client)).toBeDefined(); + }); +}); + +describe("bootstrap — singleton behavior", () => { + it("returns same instance on multiple resolves", () => { + const container = bootstrap({ config: ddbOnlyConfig }); + expect(container.resolve(Logger)).toBe(container.resolve(Logger)); + expect(container.resolve(Cache)).toBe(container.resolve(Cache)); + expect(container.resolve(SourceDynamoDbClient)).toBe(container.resolve(SourceDynamoDbClient)); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test __tests__/bootstrap.test.ts 2>&1 | tail -20 +``` + +Expected: FAIL — config objects have no `storage` field yet, and bootstrap still branches on it. + +- [ ] **Step 3: Rewrite `src/bootstrap.ts`** + +Replace the `if (config.storage === "ddb")` / `if (config.storage === "os")` blocks. Full replacement of the service registration and feature registration sections: + +```typescript +// Services — always register DDB +container.registerInstance(DynamoDbClientConfig, { + source: { + region: config.source.region, + credentials: config.source.credentials + }, + target: { + region: config.target.region, + credentials: config.target.credentials + }, + tuning: config.tuning?.ddb +}); +DynamoDbClientFeature.register(container); + +// Services — always register S3 +container.registerInstance(S3ClientConfig, { + source: { + region: config.source.region, + credentials: config.source.credentials + }, + target: { + region: config.target.region, + credentials: config.target.credentials + }, + tuning: config.tuning?.s3 +}); +S3ClientFeature.register(container); + +// Services — OS only when configured +if (config.target.opensearch != null) { + container.registerInstance(OpenSearchClientConfig, { + endpoint: config.target.opensearch.endpoint, + region: config.target.region, + service: config.target.opensearch.service, + credentials: config.target.credentials, + maxRetries: config.tuning?.os?.maxRetries + }); + OpenSearchClientFeature.register(container); +} + +// Features — always register all processors/scanners +TransferLifecycleFeature.register(container); +PresetLifecycleFeature.register(container); +PresetLoaderFeature.register(container); +WorkerSpawnerFeature.register(container); +ModelProviderFeature.register(container); +TenantLocalesFeature.register(container); +TransformContextFeature.register(container); +PipelineBuilderFactoryFeature.register(container); +SnapshotWriterFeature.register(container); +DroppedRecordLogFeature.register(container); +TransferredRecordLogFeature.register(container); +PipelineRunnerFeature.register(container); +DdbExecutorFeature.register(container); +S3ProcessorFeature.register(container); +DdbScannerFeature.register(container); +DdbProcessorFeature.register(container); +AuditLogProcessorFeature.register(container); +TouchedIndexesFeature.register(container); +OsRecordDecompressorFeature.register(container); +OsScannerFeature.register(container); +OsProcessorFeature.register(container); +``` + +- [ ] **Step 4: Run test and verify it passes** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test __tests__/bootstrap.test.ts 2>&1 | tail -20 +``` + +Expected: all PASS. + +- [ ] **Step 5: Commit** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && git add src/bootstrap.ts __tests__/bootstrap.test.ts && git commit -m "$(cat <<'EOF' +feat: bootstrap registers all processors always; OS features conditional on config + +Co-Authored-By: Claude Sonnet 4.6 +EOF +)" +``` + +--- + +## Task 4: Remove/replace `config.storage` guards in processors and scanners + +**Files:** +- Modify: `src/features/DdbScanner/DdbScanner.ts` +- Modify: `src/features/DdbProcessor/DdbProcessor.ts` +- Modify: `src/features/S3Processor/S3Processor.ts` +- Modify: `src/features/OsScanner/OsScanner.ts` +- Modify: `src/features/OsProcessor/OsProcessor.ts` +- Modify: `src/features/AuditLogProcessor/AuditLogProcessor.ts` + +Read each file before editing. The changes are: + +- [ ] **Step 1: `DdbScanner.ts` — remove `config.storage` guard** + +Open the file and find: `if (this.config.storage !== "ddb") { throw ... }` — delete this block entirely. DDB is always available. + +- [ ] **Step 2: `DdbProcessor.ts` — remove `config.storage` guard** + +Find: `if (this.config.storage !== "ddb") { throw ... }` — delete this block. + +- [ ] **Step 3: `S3Processor.ts` — remove `config.storage` guard** + +Find: `if (this.config.storage !== "ddb") { throw ... }` — delete this block. + +- [ ] **Step 4: `OsScanner.ts` — replace storage guard with opensearch null-check** + +Change: +```typescript +if (this.config.storage !== "os") { + throw new Error("OsScanner: source is not in OS storage mode; check config.storage"); +} +``` +To: +```typescript +if (!this.config.source.opensearch) { + throw new Error("OsScanner: config.source.opensearch is not configured."); +} +``` + +- [ ] **Step 5: `OsProcessor.ts` — replace storage guard** + +Change: +```typescript +if (this.config.storage !== "os") { + throw new Error(...) +} +``` +To: +```typescript +if (!this.config.target.opensearch) { + throw new Error("OsProcessor: config.target.opensearch is not configured."); +} +``` + +- [ ] **Step 6: `AuditLogProcessor.ts` — remove `storage === "ddb"` checks** + +Find in `extendContext`: +```typescript +const tableName = + this.config.storage === "ddb" + ? (this.config.target.auditLog?.dynamodb?.tableName ?? null) + : null; +``` +Change to: +```typescript +const tableName = this.config.target.auditLog?.dynamodb?.tableName ?? null; +``` + +Find the second occurrence (likely in an `isEnabled` or similar guard): +```typescript +this.config.storage === "ddb" +``` +Remove this condition (or the whole check if it guards a method return). The audit log is enabled only when `tableName != null`, which is already handled by the existing null-check logic. + +- [ ] **Step 7: Run affected tests** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test __tests__/features/DdbScanner __tests__/features/DdbProcessor __tests__/features/S3Processor __tests__/features/OsScanner __tests__/features/OsProcessor __tests__/features/AuditLogProcessor 2>&1 | tail -30 +``` + +Expected: all PASS (they use test containers from Task 1). + +- [ ] **Step 8: Commit** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && git add src/features/DdbScanner/DdbScanner.ts src/features/DdbProcessor/DdbProcessor.ts src/features/S3Processor/S3Processor.ts src/features/OsScanner/OsScanner.ts src/features/OsProcessor/OsProcessor.ts src/features/AuditLogProcessor/AuditLogProcessor.ts && git commit -m "$(cat <<'EOF' +feat: replace config.storage guards with opensearch null-checks in processors + +Co-Authored-By: Claude Sonnet 4.6 +EOF +)" +``` + +--- + +## Task 5: Add `--preset` to `processSegment` + +Workers currently read `config.pipeline.preset` which no longer exists. They must receive the preset name as a CLI argument. + +**Files:** +- Modify: `src/commands/processSegment/register.ts` +- Modify: `src/commands/processSegment/handler.ts` +- Modify: `__tests__/commands/processSegment.test.ts` + +- [ ] **Step 1: Add `--preset` option to `src/commands/processSegment/register.ts`** + +Add to the yargs options: +```typescript +.option("preset", { + type: "string", + demandOption: true, + description: "Preset name or path to use for this segment" +}) +``` +And pass it to the handler: +```typescript +async argv => { + await handler({ ...argv, preset: argv.preset, logLevel: argv["log-level"] as string | undefined }); +} +``` + +- [ ] **Step 2: Update `ProcessSegmentArgs` and handler in `src/commands/processSegment/handler.ts`** + +Add `preset: string` to `ProcessSegmentArgs`: +```typescript +export interface ProcessSegmentArgs { + runId: string; + segment: number; + total: number; + config: string; + preset: string; + logLevel?: string; +} +``` + +Change line 42 from: +```typescript +const preset = await presetLoader.load(config.pipeline.preset); +``` +To: +```typescript +const preset = await presetLoader.load(argv.preset); +``` + +- [ ] **Step 3: Update `__tests__/commands/processSegment.test.ts`** + +Find the `loadConfig` mock: +```typescript +vi.mock("~/features/MigrationConfig/loadConfig.ts", () => ({ + loadConfig: vi.fn(async (_path: string) => ({ storage: "ddb", pipeline: { preset: "x" } })) +})); +``` +Change to: +```typescript +vi.mock("~/features/MigrationConfig/loadConfig.ts", () => ({ + loadConfig: vi.fn(async (_path: string) => ({ + source: { region: "us-east-1", credentials: { accessKeyId: "t", secretAccessKey: "t" }, dynamodb: { tableName: "src" }, s3: { bucket: "src-b" } }, + target: { region: "us-east-1", credentials: { accessKeyId: "t", secretAccessKey: "t" }, dynamodb: { tableName: "tgt" }, s3: { bucket: "tgt-b" } }, + pipeline: {} + })) +})); +``` + +Find all `handler({ runId: ..., segment: ..., total: ..., config: ... })` calls in the test and add `preset: "test-preset"` to each. + +- [ ] **Step 4: Run tests** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test __tests__/commands/processSegment.test.ts 2>&1 | tail -20 +``` + +Expected: all PASS. + +- [ ] **Step 5: Commit** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && git add src/commands/processSegment/register.ts src/commands/processSegment/handler.ts __tests__/commands/processSegment.test.ts && git commit -m "$(cat <<'EOF' +feat: processSegment receives preset via --preset CLI argument + +Co-Authored-By: Claude Sonnet 4.6 +EOF +)" +``` + +--- + +## Task 6: Preset discovery + simplified config discovery + +**Files:** +- Create: `src/commands/run/wizard/presetDiscovery.ts` +- Modify: `src/commands/run/wizard/configDiscovery.ts` +- Rewrite: `__tests__/commands/run/wizard/configDiscovery.test.ts` +- Create: `__tests__/commands/run/wizard/presetDiscovery.test.ts` +- Modify: `__tests__/fixtures/wizard/ddb.config.ts` → replace with unified config fixture +- Delete: `__tests__/fixtures/wizard/os.config.ts` + +- [ ] **Step 1: Write failing tests for `presetDiscovery`** + +Create `__tests__/commands/run/wizard/presetDiscovery.test.ts`: + +```typescript +import { describe, it, expect } from "vitest"; +import { join } from "node:path"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { listAvailablePresets } from "../../../../src/commands/run/wizard/presetDiscovery.ts"; + +describe("listAvailablePresets", () => { + it("returns built-in preset names (at minimum v5-to-v6-ddb and v5-to-v6-os)", () => { + const presets = listAvailablePresets(); + expect(presets).toContain("v5-to-v6-ddb"); + expect(presets).toContain("v5-to-v6-os"); + }); + + it("includes user presets from presetsDir when provided", () => { + const tmp = mkdtempSync(join(tmpdir(), "presetdiscovery-")); + try { + writeFileSync(join(tmp, "my-preset.ts"), "export default {}"); + writeFileSync(join(tmp, "another.ts"), "export default {}"); + const presets = listAvailablePresets(tmp); + expect(presets).toContain("my-preset"); + expect(presets).toContain("another"); + } finally { + rmSync(tmp, { recursive: true }); + } + }); + + it("deduplicates when user preset name matches a built-in", () => { + const tmp = mkdtempSync(join(tmpdir(), "presetdiscovery-dup-")); + try { + writeFileSync(join(tmp, "v5-to-v6-ddb.ts"), "export default {}"); + const presets = listAvailablePresets(tmp); + const count = presets.filter(p => p === "v5-to-v6-ddb").length; + expect(count).toBe(1); + } finally { + rmSync(tmp, { recursive: true }); + } + }); + + it("returns empty list when presetsDir does not exist", () => { + const presets = listAvailablePresets("/nonexistent/path/xyz"); + // Built-ins still present; user dir gracefully ignored + expect(Array.isArray(presets)).toBe(true); + }); + + it("returns sorted list", () => { + const presets = listAvailablePresets(); + const sorted = [...presets].sort(); + expect(presets).toEqual(sorted); + }); +}); +``` + +- [ ] **Step 2: Write failing tests for simplified `configDiscovery`** + +Replace `__tests__/commands/run/wizard/configDiscovery.test.ts`: + +```typescript +import { describe, it, expect } from "vitest"; +import { join } from "node:path"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { discoverConfig } from "../../../../src/commands/run/wizard/configDiscovery.ts"; + +describe("discoverConfig", () => { + it("returns the resolved path to config.ts when it exists", async () => { + const tmp = mkdtempSync(join(tmpdir(), "configdiscovery-")); + try { + const configPath = join(tmp, "config.ts"); + writeFileSync(configPath, "export default {};"); + const result = await discoverConfig(tmp); + expect(result).toBe(configPath); + } finally { + rmSync(tmp, { recursive: true }); + } + }); + + it("returns null when config.ts does not exist", async () => { + const tmp = mkdtempSync(join(tmpdir(), "configdiscovery-empty-")); + try { + const result = await discoverConfig(tmp); + expect(result).toBeNull(); + } finally { + rmSync(tmp, { recursive: true }); + } + }); + + it("returns null for nonexistent directory", async () => { + const result = await discoverConfig("/nonexistent/path/xyz"); + expect(result).toBeNull(); + }); +}); +``` + +- [ ] **Step 3: Run tests to verify they fail** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test __tests__/commands/run/wizard/presetDiscovery.test.ts __tests__/commands/run/wizard/configDiscovery.test.ts 2>&1 | tail -20 +``` + +Expected: FAIL — neither file exists yet. + +- [ ] **Step 4: Create `src/commands/run/wizard/presetDiscovery.ts`** + +```typescript +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { existsSync, readdirSync } from "node:fs"; + +const BUILTIN_PRESETS_DIR = join( + dirname(fileURLToPath(import.meta.url)), + "../../../presets" +); + +const PRESET_EXTENSIONS: ReadonlySet = new Set([".ts", ".js"]); + +function stripExtension(filename: string): string | null { + for (const ext of PRESET_EXTENSIONS) { + if (filename.endsWith(ext)) { + return filename.slice(0, -ext.length); + } + } + return null; +} + +function scanDir(dir: string): string[] { + if (!existsSync(dir)) { + return []; + } + try { + return readdirSync(dir) + .map(stripExtension) + .filter((name): name is string => name !== null); + } catch { + return []; + } +} + +export function listAvailablePresets(presetsDir?: string): string[] { + const builtIns = scanDir(BUILTIN_PRESETS_DIR); + const userPresets = presetsDir ? scanDir(presetsDir) : []; + const all = new Set([...builtIns, ...userPresets]); + return [...all].sort(); +} +``` + +- [ ] **Step 5: Rewrite `src/commands/run/wizard/configDiscovery.ts`** + +```typescript +import { access } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +export async function discoverConfig(projectDir: string): Promise { + const configPath = resolve(join(projectDir, "config.ts")); + try { + await access(configPath); + return configPath; + } catch { + return null; + } +} +``` + +- [ ] **Step 6: Update fixture — replace `__tests__/fixtures/wizard/ddb.config.ts` with a unified config stub** + +```typescript +export default { + source: { + region: "eu-central-1", + credentials: { accessKeyId: "AKIA", secretAccessKey: "secret" }, + dynamodb: { tableName: "src-table" }, + s3: { bucket: "src-bucket" } + }, + target: { + region: "us-east-1", + credentials: { accessKeyId: "AKIA", secretAccessKey: "secret" }, + dynamodb: { tableName: "tgt-table" }, + s3: { bucket: "tgt-bucket" } + }, + pipeline: {} +}; +``` + +- [ ] **Step 7: Delete `__tests__/fixtures/wizard/os.config.ts`** + +```bash +rm /Users/brunozoric/work/webiny/webiny-v5-to-v6/__tests__/fixtures/wizard/os.config.ts +``` + +- [ ] **Step 8: Run tests and verify they pass** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test __tests__/commands/run/wizard/presetDiscovery.test.ts __tests__/commands/run/wizard/configDiscovery.test.ts 2>&1 | tail -20 +``` + +Expected: all PASS. + +- [ ] **Step 9: Commit** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && git add src/commands/run/wizard/presetDiscovery.ts src/commands/run/wizard/configDiscovery.ts __tests__/commands/run/wizard/presetDiscovery.test.ts __tests__/commands/run/wizard/configDiscovery.test.ts __tests__/fixtures/wizard/ddb.config.ts && git rm __tests__/fixtures/wizard/os.config.ts && git commit -m "$(cat <<'EOF' +feat: preset discovery + simplified configDiscovery (config.ts only) + +Co-Authored-By: Claude Sonnet 4.6 +EOF +)" +``` + +--- + +## Task 7: TransferWizard — add preset selection step + +**Files:** +- Modify: `src/commands/run/wizard/types.ts` +- Modify: `src/commands/run/wizard/TransferWizard.ts` +- Modify: `src/commands/run/register.ts` +- Modify: `__tests__/commands/run/wizard/TransferWizard.test.ts` + +- [ ] **Step 1: Add `WizardResult` to `src/commands/run/wizard/types.ts`** + +Read the current file. Add at the end: + +```typescript +export interface WizardResult { + configPath: string; + preset: string; +} +``` + +- [ ] **Step 2: Write failing wizard tests** + +Replace `__tests__/commands/run/wizard/TransferWizard.test.ts`: + +```typescript +import { describe, it, expect, vi, beforeEach } from "vitest"; +import type { Stats } from "node:fs"; +import { TransferWizard } from "../../../../src/commands/run/wizard/TransferWizard.ts"; +import type { RawOutputValues } from "../../../../src/commands/run/wizard/types.ts"; + +vi.mock("../../../../src/commands/run/wizard/projectDiscovery.ts"); +vi.mock("../../../../src/commands/run/wizard/configDiscovery.ts"); +vi.mock("../../../../src/commands/run/wizard/presetDiscovery.ts"); +vi.mock("../../../../src/commands/run/wizard/envWriter.ts"); +vi.mock("../../../../src/commands/run/wizard/sources/WebinyOutputSource.ts"); +vi.mock("../../../../src/commands/run/wizard/sources/PulumiStateSource.ts"); +vi.mock("@inquirer/prompts"); +vi.mock("node:fs/promises"); +vi.mock("node:fs", () => ({ existsSync: vi.fn(() => false) })); +vi.mock("../../../../src/commands/initProject/scaffoldProject.ts", () => ({ + scaffoldProject: vi.fn().mockResolvedValue(undefined) +})); + +import { discoverProjects } from "../../../../src/commands/run/wizard/projectDiscovery.ts"; +import { discoverConfig } from "../../../../src/commands/run/wizard/configDiscovery.ts"; +import { listAvailablePresets } from "../../../../src/commands/run/wizard/presetDiscovery.ts"; +import { writeEnv } from "../../../../src/commands/run/wizard/envWriter.ts"; +import { extractFromWebinyOutput } from "../../../../src/commands/run/wizard/sources/WebinyOutputSource.ts"; +import { extractFromPulumiState } from "../../../../src/commands/run/wizard/sources/PulumiStateSource.ts"; +import { input, select } from "@inquirer/prompts"; +import { stat, access } from "node:fs/promises"; +import { scaffoldProject } from "../../../../src/commands/initProject/scaffoldProject.ts"; + +const mockDiscoverProjects = vi.mocked(discoverProjects); +const mockDiscoverConfig = vi.mocked(discoverConfig); +const mockListAvailablePresets = vi.mocked(listAvailablePresets); +const mockWriteEnv = vi.mocked(writeEnv); +const mockExtractFromWebinyOutput = vi.mocked(extractFromWebinyOutput); +const mockExtractFromPulumiState = vi.mocked(extractFromPulumiState); +const mockInput = vi.mocked(input); +const mockSelect = vi.mocked(select); +const mockStat = vi.mocked(stat); +const mockAccess = vi.mocked(access); +const mockScaffoldProject = vi.mocked(scaffoldProject); + +const noFile = (): never => { + throw Object.assign(new Error("ENOENT"), { code: "ENOENT" }); +}; + +const SOURCE_VALS: RawOutputValues = { + region: "eu-central-1", + primaryDynamodbTableName: "wby-source-primary", + fileManagerBucketId: "wby-source-bucket", + osTableName: "", + osEndpoint: "" +}; + +const TARGET_VALS: RawOutputValues = { + region: "us-east-1", + primaryDynamodbTableName: "wby-target-primary", + fileManagerBucketId: "wby-target-bucket", + osTableName: "", + osEndpoint: "" +}; + +beforeEach(() => { + vi.resetAllMocks(); + mockWriteEnv.mockResolvedValue(undefined); + mockScaffoldProject.mockResolvedValue(undefined); +}); + +describe("TransferWizard", () => { + it("env-setup path: writes .env and returns null (no preset selection yet)", async () => { + mockDiscoverProjects.mockResolvedValue(["my-project"]); + mockSelect.mockResolvedValue("my-project"); + mockStat.mockImplementation(async (p: unknown) => { + const path = String(p); + if (path.endsWith("source.webiny.json") || path.endsWith("target.webiny.json")) { + return { size: 100 } as unknown as Stats; + } + return noFile(); + }); + mockAccess.mockRejectedValue(Object.assign(new Error("ENOENT"), { code: "ENOENT" })); + mockExtractFromWebinyOutput + .mockResolvedValueOnce(SOURCE_VALS) + .mockResolvedValueOnce(TARGET_VALS); + mockInput.mockResolvedValue("4"); + + const result = await new TransferWizard(process.cwd()).run(); + + expect(result).toBeNull(); + expect(mockWriteEnv).toHaveBeenCalledOnce(); + }); + + it("re-run path: .env exists, no JSON → finds config.ts, prompts for preset, returns WizardResult", async () => { + const CONFIG_PATH = "/projects/my-project/config.ts"; + mockDiscoverProjects.mockResolvedValue(["my-project"]); + mockSelect + .mockResolvedValueOnce("my-project") + .mockResolvedValueOnce("v5-to-v6-ddb"); + mockStat.mockImplementation(async (p: unknown) => { + if (String(p).endsWith(".env")) { + return { size: 100 } as unknown as Stats; + } + return noFile(); + }); + mockDiscoverConfig.mockResolvedValue(CONFIG_PATH); + mockListAvailablePresets.mockReturnValue(["v5-to-v6-ddb", "v5-to-v6-os"]); + + const result = await new TransferWizard(process.cwd()).run(); + + expect(result).toEqual({ configPath: CONFIG_PATH, preset: "v5-to-v6-ddb" }); + expect(mockWriteEnv).not.toHaveBeenCalled(); + }); + + it("re-run path: exits with error when no config.ts found in project", async () => { + mockDiscoverProjects.mockResolvedValue(["my-project"]); + mockSelect.mockResolvedValue("my-project"); + mockStat.mockImplementation(async (p: unknown) => { + if (String(p).endsWith(".env")) { + return { size: 100 } as unknown as Stats; + } + return noFile(); + }); + mockDiscoverConfig.mockResolvedValue(null); + + // Should call process.exit(1) or throw + const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => { throw new Error("exit"); }); + await expect(new TransferWizard(process.cwd()).run()).rejects.toThrow("exit"); + exitSpy.mockRestore(); + }); + + it("writes .env with correct values from webiny output", async () => { + mockDiscoverProjects.mockResolvedValue(["my-project"]); + mockSelect.mockResolvedValue("my-project"); + mockStat.mockImplementation(async (p: unknown) => { + const path = String(p); + if (path.endsWith("source.webiny.json") || path.endsWith("target.webiny.json")) { + return { size: 100 } as unknown as Stats; + } + return noFile(); + }); + mockAccess.mockRejectedValue(Object.assign(new Error("ENOENT"), { code: "ENOENT" })); + mockExtractFromWebinyOutput + .mockResolvedValueOnce(SOURCE_VALS) + .mockResolvedValueOnce(TARGET_VALS); + mockInput.mockResolvedValue("4"); + + await new TransferWizard(process.cwd()).run(); + + expect(mockWriteEnv).toHaveBeenCalledOnce(); + const [, envValues] = mockWriteEnv.mock.calls[0]; + expect(envValues.sourceRegion).toBe("eu-central-1"); + expect(envValues.targetRegion).toBe("us-east-1"); + expect(envValues.segments).toBe(4); + }); + + it("throws when same-side files disagree on osTableName", async () => { + mockDiscoverProjects.mockResolvedValue(["my-project"]); + mockSelect.mockResolvedValue("my-project"); + mockStat.mockImplementation(async (p: unknown) => { + const path = String(p); + if (path.endsWith("source.webiny.json") || path.endsWith("source.pulumi.json")) { + return { size: 100 } as unknown as Stats; + } + return noFile(); + }); + mockExtractFromWebinyOutput.mockResolvedValue({ ...SOURCE_VALS, osTableName: "wby-es-webiny" }); + mockExtractFromPulumiState.mockResolvedValue({ ...SOURCE_VALS, osTableName: "wby-es-pulumi" }); + + await expect(new TransferWizard(process.cwd()).run()).rejects.toThrow(/osTableName/); + }); +}); +``` + +- [ ] **Step 3: Run test to verify it fails** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test __tests__/commands/run/wizard/TransferWizard.test.ts 2>&1 | tail -20 +``` + +Expected: FAIL. + +- [ ] **Step 4: Update `src/commands/run/wizard/TransferWizard.ts`** + +Key changes: +1. Import `discoverConfig` instead of `discoverConfigs` (removed). +2. Import `listAvailablePresets` from `./presetDiscovery.ts`. +3. Change return type of `run()` to `Promise`. +4. Update `runConfigSelection` to use `discoverConfig` and add a preset selection step. + +The new `run()` return type: `Promise` — null means env was just written. + +The `runConfigSelection` method becomes `runPresetSelection(projectName)`: + +```typescript +private async runPresetSelection(projectName: string): Promise { + const projectDir = resolve(join(this.cwd, "projects", projectName)); + const configPath = await discoverConfig(projectDir); + + if (!configPath) { + console.error( + `\nNo config.ts found in projects/${projectName}/.\n` + + `Run "yarn transfer init-project ${projectName}" to scaffold one.\n` + ); + process.exit(1); + } + + // Dynamically import config to read presetsDir + let presetsDir: string | undefined; + try { + const mod = await import(pathToFileURL(configPath).href); + presetsDir = mod.default?.pipeline?.presetsDir; + } catch { + // ignore — presets from built-ins only + } + + const presets = listAvailablePresets(presetsDir); + + if (presets.length === 0) { + console.error("\nNo presets available. Check your presetsDir configuration.\n"); + process.exit(1); + } + + const preset = await select({ + message: "Which preset do you want to run?", + choices: presets.map(p => ({ value: p, name: p })) + }); + + return { configPath, preset }; +} +``` + +The main `run()` method should call `this.runPresetSelection(projectName)` instead of the old `this.runConfigSelection(projectName)`. + +Also update the import at the top of the file to add: +- `import { discoverConfig } from "./configDiscovery.ts";` +- `import { listAvailablePresets } from "./presetDiscovery.ts";` +- `import type { WizardResult } from "./types.ts";` +- `import { pathToFileURL } from "node:url";` + +Remove import of `discoverConfigs`. + +- [ ] **Step 5: Update `src/commands/run/register.ts`** + +Change the wizard result handling from: +```typescript +const configPath = await wizard.run(); +if (configPath === null) { + process.exit(0); +} +await handler(configPath, argv.segments, argv["log-level"] as string | undefined); +``` +To: +```typescript +const result = await wizard.run(); +if (result === null) { + process.exit(0); +} +await handler(result.configPath, result.preset, argv.segments, argv["log-level"] as string | undefined); +``` + +Also remove the `if (argv.config)` shortcut path (it previously bypassed the wizard; now everything goes through the wizard). Keep `--config` as an option for potential future use but route it through the wizard's preset selection only (or just remove the branch — the user said "everything will go through interactive CLI"). + +Actually, remove the `if (argv.config)` shortcut entirely. The `--config` flag no longer bypasses the wizard. + +- [ ] **Step 6: Run wizard tests** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test __tests__/commands/run/wizard/TransferWizard.test.ts 2>&1 | tail -30 +``` + +Expected: all PASS. + +- [ ] **Step 7: Commit** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && git add src/commands/run/wizard/types.ts src/commands/run/wizard/TransferWizard.ts src/commands/run/register.ts __tests__/commands/run/wizard/TransferWizard.test.ts && git commit -m "$(cat <<'EOF' +feat: TransferWizard adds preset selection step, returns WizardResult + +Co-Authored-By: Claude Sonnet 4.6 +EOF +)" +``` + +--- + +## Task 8: Update `run/handler.ts` + +**Files:** +- Modify: `src/commands/run/handler.ts` + +- [ ] **Step 1: Update `handler` signature and body** + +Change: +```typescript +export async function handler( + configPath: string, + segmentsFilter?: number[], + logLevel?: string +): Promise +``` +To: +```typescript +export async function handler( + configPath: string, + presetName: string, + segmentsFilter?: number[], + logLevel?: string +): Promise +``` + +Replace: +```typescript +await presetLoader.load(config.pipeline.preset); +``` +With: +```typescript +await presetLoader.load(presetName); +``` + +Update `logConfig` to remove the `config.storage` switch. Replace the storage-conditional log lines: +```typescript +if (config.storage === "ddb") { + logger.info(` Source Region: ${config.source.region}`); + ... +} else { + ... +} +``` +With unified logging: +```typescript +logger.info(` Preset: ${presetName}`); +logger.info(` Source Region: ${config.source.region}`); +logger.info(` Source DDB Table: ${config.source.dynamodb.tableName}`); +logger.info(` Source S3 Bucket: ${config.source.s3.bucket}`); +if (config.source.opensearch) { + logger.info(` Source OS Table: ${config.source.opensearch.tableName}`); +} +logger.info(` Target Region: ${config.target.region}`); +logger.info(` Target DDB Table: ${config.target.dynamodb.tableName}`); +logger.info(` Target S3 Bucket: ${config.target.s3.bucket}`); +if (config.target.opensearch) { + logger.info(` Target OS Table: ${config.target.opensearch.tableName}`); + logger.info(` OS Endpoint: ${config.target.opensearch.endpoint}`); +} +``` + +Also remove `logger.info(\` Storage: ${config.storage}\`);` and `logger.info(\` Preset: ${config.pipeline.preset}\`);` lines. + +Update `LogConfigParams` interface — add `presetName: string`, remove any storage references: +```typescript +interface LogConfigParams { + logger: Logger.Interface; + config: MigrationConfig.Interface; + runId: string; + segments: number; + segmentsToRun: number[]; + logLevel?: string; + presetName: string; +} +``` + +Update `spawnWorker` to pass `--preset`: + +Change signature: +```typescript +async function spawnWorker( + segment: number, + total: number, + runId: string, + configPath: string, + presetName: string, + logLevel?: string +): Promise +``` + +Add to args array: +```typescript +"--preset", presetName, +``` + +Update the `spawnWorker` call site to pass `presetName`. + +- [ ] **Step 2: Run type-check** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn ts-check 2>&1 | grep -v "node_modules" | head -40 +``` + +Expected: 0 errors (or the same 5 pre-existing errors on `bruno/refactor/user-presets` — see AGENTS.md §7 "presetsDir feature"). + +- [ ] **Step 3: Commit** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && git add src/commands/run/handler.ts && git commit -m "$(cat <<'EOF' +feat: run/handler accepts presetName param, passes --preset to workers + +Co-Authored-By: Claude Sonnet 4.6 +EOF +)" +``` + +--- + +## Task 9: Public API, templates, project config, final verification + +**Files:** +- Modify: `src/index.ts` +- Create: `templates/internal-project/config.ts` +- Delete: `templates/internal-project/ddb.transfer.config.ts` +- Delete: `templates/internal-project/os.transfer.config.ts` +- Modify: `projects/v5-to-v6/config.ts` + +- [ ] **Step 1: Update `src/index.ts` — export `createConfig`, remove old exports** + +Read `src/index.ts`. Remove the exports for `createDdbConfig`, `createOsConfig`, `DdbMigrationConfiguration`, `OsMigrationConfiguration`. Add export for `createConfig`. Keep all other exports unchanged. + +Replace: +```typescript +export { createDdbConfig } from "~/features/MigrationConfig/createDdbConfig.ts"; +export { createOsConfig } from "~/features/MigrationConfig/createOsConfig.ts"; +``` +With: +```typescript +export { createConfig } from "~/features/MigrationConfig/createConfig.ts"; +``` + +Also remove type exports for `DdbMigrationConfiguration` and `OsMigrationConfiguration` if present. + +- [ ] **Step 2: Create `templates/internal-project/config.ts`** + +```typescript +import { createConfig, fromAwsProfile, fromEnv, loadEnv, numberFromEnv } from "@webiny/data-transfer"; + +// Loads .env from the same directory as this file. `.env*` is gitignored. +loadEnv(import.meta.url); + +const DEFAULT_REGION = "eu-central-1"; +const DEFAULT_PROFILE = "default"; + +export default createConfig({ + source: { + region: fromEnv("SOURCE_REGION", DEFAULT_REGION), + credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", DEFAULT_PROFILE) }), + dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, + s3: { bucket: fromEnv("SOURCE_S3_BUCKET") }, + // Remove or set to null if your environment has no OpenSearch: + opensearch: { tableName: fromEnv("SOURCE_OS_TABLE") } + }, + target: { + region: fromEnv("TARGET_REGION", DEFAULT_REGION), + credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", DEFAULT_PROFILE) }), + dynamodb: { tableName: fromEnv("TARGET_DDB_TABLE") }, + s3: { bucket: fromEnv("TARGET_S3_BUCKET") }, + // Audit log table. Set tableName to null or omit the block to skip: + auditLog: { dynamodb: { tableName: fromEnv("TARGET_AUDIT_LOGS_TABLE") } }, + // Remove or set to null if your target has no OpenSearch: + opensearch: { + endpoint: fromEnv("TARGET_OS_ENDPOINT"), + tableName: fromEnv("TARGET_OS_TABLE"), + service: "opensearch", + indexPrefix: fromEnv("TARGET_OS_INDEX_PREFIX", "") + } + }, + pipeline: { + segments: numberFromEnv("SEGMENTS", 4), + modelsDir: fromEnv("MODELS_DIR", "./models"), + presetsDir: "./presets" + } +}); +``` + +- [ ] **Step 3: Delete old templates** + +```bash +rm /Users/brunozoric/work/webiny/webiny-v5-to-v6/templates/internal-project/ddb.transfer.config.ts +rm /Users/brunozoric/work/webiny/webiny-v5-to-v6/templates/internal-project/os.transfer.config.ts +``` + +- [ ] **Step 4: Finalize `projects/v5-to-v6/config.ts`** + +The file already has the correct shape from an earlier sketch. Verify it matches the `createConfig` signature exactly (especially the import path — it should use `~/index.ts` since it's inside the repo). If the imports use `~/index.ts` already, it should be fine. Make any corrections needed. + +- [ ] **Step 5: Run format, type-check, full test suite** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn format:fix && yarn ts-check 2>&1 | grep -v "node_modules" | head -40 +``` + +Expected: 0 TypeScript errors. + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && yarn test 2>&1 | tail -40 +``` + +Expected: all tests PASS, coverage thresholds met (lines ≥77%, functions ≥80%, branches ≥70%, statements ≥77%). + +If any tests fail, investigate and fix before committing. Common failure areas: +- Tests that mock `loadConfig` and return `{ storage: "ddb", pipeline: { preset: "x" } }` → update mocks +- Tests using `config.storage` or `config.pipeline.preset` in assertions → remove those assertions +- Wizard tests that mock `discoverConfigs` (old function) → update to mock `discoverConfig` (new) + +- [ ] **Step 6: Commit** + +```bash +cd /Users/brunozoric/work/webiny/webiny-v5-to-v6 && git add src/index.ts templates/internal-project/config.ts projects/v5-to-v6/config.ts && git rm templates/internal-project/ddb.transfer.config.ts templates/internal-project/os.transfer.config.ts && git commit -m "$(cat <<'EOF' +feat: public API exports createConfig; unified config template; project config finalized + +Co-Authored-By: Claude Sonnet 4.6 +EOF +)" +``` + +--- + +## Self-review + +Spec coverage check: + +| Requirement | Task | +|---|---| +| `createConfig` replaces `createDdbConfig` + `createOsConfig` | Tasks 1, 9 | +| `storage` discriminator removed | Task 1 | +| `pipeline.preset` removed from schema | Task 1 | +| Required: `source.dynamodb`, `source.s3`, `target.dynamodb`, `target.s3` | Task 1 | +| Optional: `source.opensearch`, `target.opensearch` (both-or-neither) | Task 1 | +| `loadConfig` validates with new schema | Task 2 | +| Bootstrap registers all processors always | Task 3 | +| OS client conditional on `target.opensearch != null` | Task 3 | +| Scanner/processor `storage` guards removed/replaced | Task 4 | +| Worker receives preset via `--preset` | Task 5 | +| `presetDiscovery.ts` lists built-ins + user presets | Task 6 | +| `configDiscovery.ts` simplified to `config.ts` only | Task 6 | +| `TransferWizard` adds preset selection step | Task 7 | +| `TransferWizard.run()` returns `WizardResult \| null` | Task 7 | +| `run/register.ts` uses `WizardResult` | Task 7 | +| `run/handler.ts` accepts `presetName` | Task 8 | +| `--preset` passed to worker spawn | Task 8 | +| `src/index.ts` exports `createConfig` | Task 9 | +| Unified template created | Task 9 | +| All tests green, coverage thresholds met | Task 9 | + +All requirements covered. No gaps found. diff --git a/docs/superpowers/plans/2026-05-11-access-checker.md b/docs/superpowers/plans/2026-05-11-access-checker.md new file mode 100644 index 00000000..537ae56e --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-access-checker.md @@ -0,0 +1,1205 @@ +# Access Checker Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Run a per-processor preflight access check before any transfer segment starts, collecting structured results (ok / denied / unknown) per resource and aborting the transfer on any `denied` entry. + +**Architecture:** Add a required `checkAccess(): Promise` method to `Processor.Interface`. Each processor probes its own resources (DynamoDB `DescribeTable`, S3 `HeadBucket`, OpenSearch `listIndexes`) and returns labelled entries. A new `AccessChecker` DI service fans out to all registered processors via `runner.getProcessors()`, flattens the results, and the run handler prints the table and exits on any denial. + +**Tech Stack:** AWS SDK v3 (`@webiny/aws-sdk/client-dynamodb`, `@webiny/aws-sdk/client-s3`), vitest `vi.mock`, `@webiny/di` container, existing `isAccessDeniedError` helper (added to `src/base/isRetryableAwsError.ts`). + +--- + +### Task 1: AccessCheck types + required checkAccess() + stubs + +**Files:** +- Modify: `src/domain/pipeline/abstractions/Processor.ts` +- Modify: `src/features/DdbProcessor/DdbProcessor.ts` +- Modify: `src/features/AuditLogProcessor/AuditLogProcessor.ts` +- Modify: `src/features/S3Processor/S3Processor.ts` +- Modify: `src/features/OsProcessor/OsProcessor.ts` +- Modify: `__tests__/domain/pipeline/Processor.test.ts` + +- [ ] **Step 1: Add `AccessCheck` namespace and required `checkAccess()` to `Processor.ts`** + +In `src/domain/pipeline/abstractions/Processor.ts`, add the `AccessCheck` namespace before the `IProcessor` interface and add `checkAccess()` as a required method: + +```ts +export namespace AccessCheck { + export type Status = "ok" | "denied" | "unknown"; + + export interface Entry { + label: string; + status: Status; + } + + export type Report = Entry[]; +} + +interface IProcessor< + TBaseContext extends BaseTransformContext.Interface = + BaseTransformContext.Interface, + TSlice = Record +> { + extendContext?(base: TBaseContext): TSlice; + onEnd?(ctx: TBaseContext & TSlice): void | Promise; + + /** + * Pre-transfer access check. Called before any segment worker is spawned. + * Returns one entry per probed resource (table, bucket, cluster endpoint). + * "denied" entries abort the transfer; "unknown" entries warn and proceed. + */ + checkAccess(): Promise; + + getGuardWarning?(): Promise; + execute(commands: Commands): Promise; + afterShard?(ctx: IAfterShardContext): void | Promise; +} +``` + +- [ ] **Step 2: Add stub `checkAccess()` to `DdbProcessor.ts`** + +Inside `DdbProcessorImpl`, add after `onEnd`: + +```ts +public async checkAccess(): Promise { + return []; +} +``` + +Also add to the import in `DdbProcessor.ts`: +```ts +import { Processor, AccessCheck } from "~/domain/pipeline/abstractions/Processor.ts"; +``` + +Wait — `AccessCheck` is exported from `Processor.ts`, so update the import line in `DdbProcessor.ts` from: +```ts +import { Processor } from "~/domain/pipeline/abstractions/Processor.ts"; +``` +to: +```ts +import { AccessCheck, Processor } from "~/domain/pipeline/abstractions/Processor.ts"; +``` + +- [ ] **Step 3: Add stub `checkAccess()` to `AuditLogProcessor.ts`** + +Inside `AuditLogProcessorImpl`, add after `onEnd`: + +```ts +public async checkAccess(): Promise { + return []; +} +``` + +Update the import at the top of `AuditLogProcessor.ts`: +```ts +import { AccessCheck, Processor } from "~/domain/pipeline/abstractions/Processor.ts"; +``` + +- [ ] **Step 4: Add stub `checkAccess()` to `S3Processor.ts`** + +Inside `S3ProcessorImpl`, add after `getGuardWarning`: + +```ts +public async checkAccess(): Promise { + return []; +} +``` + +Update the import at the top of `S3Processor.ts`: +```ts +import { AccessCheck, Processor } from "~/domain/pipeline/abstractions/Processor.ts"; +``` + +- [ ] **Step 5: Add stub `checkAccess()` to `OsProcessor.ts`** + +Inside `OsProcessorImpl`, add after `onEnd`: + +```ts +public async checkAccess(): Promise { + return []; +} +``` + +Update the import at the top of `OsProcessor.ts`: +```ts +import { AccessCheck, Processor } from "~/domain/pipeline/abstractions/Processor.ts"; +``` + +- [ ] **Step 6: Add `checkAccess()` to `FakeProcessor` in `__tests__/domain/pipeline/Processor.test.ts`** + +Inside `FakeProcessor`, add after `afterShard`: + +```ts +public async checkAccess(): Promise { + return []; +} +``` + +Add `AccessCheck` to the import at the top: +```ts +import { Processor, AccessCheck } from "~/domain/pipeline/index.ts"; +``` + +(Verify `AccessCheck` is re-exported from `~/domain/pipeline/index.ts`. If not, import directly from `~/domain/pipeline/abstractions/Processor.ts`.) + +- [ ] **Step 7: Run the full test suite** + +```bash +yarn test +``` + +Expected: all tests pass. If any test fails, it is because a processor instance somewhere implements `Processor.Interface` without `checkAccess()` — find it with `grep -rn "implements Processor.Interface" src __tests__` and add the stub. + +- [ ] **Step 8: Commit** + +```bash +git add src/domain/pipeline/abstractions/Processor.ts \ + src/features/DdbProcessor/DdbProcessor.ts \ + src/features/AuditLogProcessor/AuditLogProcessor.ts \ + src/features/S3Processor/S3Processor.ts \ + src/features/OsProcessor/OsProcessor.ts \ + __tests__/domain/pipeline/Processor.test.ts +git commit -m "feat: add required checkAccess() to Processor.Interface with stub implementations" +``` + +--- + +### Task 2: isAccessDeniedError helper + DdbProcessor.checkAccess() + +**Files:** +- Modify: `src/base/isRetryableAwsError.ts` +- Modify: `src/base/index.ts` +- Modify: `src/features/DdbProcessor/DdbProcessor.ts` +- Modify: `__tests__/features/DdbProcessor/DdbProcessor.test.ts` + +- [ ] **Step 1: Add `isAccessDeniedError` to `src/base/isRetryableAwsError.ts`** + +Add after the `isTokenBucketExhausted` function: + +```ts +/** + * Returns true when the error indicates an IAM / credentials access denial. + * Covers DynamoDB AccessDeniedException, S3 AccessDenied, and HTTP 403. + */ +export function isAccessDeniedError(error: unknown): boolean { + if (!error || typeof error !== "object") { + return false; + } + const candidate = error as AwsErrorLike; + const name = candidate.name ?? candidate.code; + if (typeof name === "string" && TERMINAL_ERROR_NAMES.has(name)) { + return true; + } + return candidate.$metadata?.httpStatusCode === 403; +} +``` + +- [ ] **Step 2: Export `isAccessDeniedError` from `src/base/index.ts`** + +Update the existing export block: + +```ts +export { + isRetryableAwsError, + isThrottlingError, + isAccessDeniedError, +} from "./isRetryableAwsError.ts"; +``` + +- [ ] **Step 3: Write failing tests for `DdbProcessor.checkAccess()`** + +Add to `__tests__/features/DdbProcessor/DdbProcessor.test.ts`: + +```ts +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { DynamoDB } from "@webiny/aws-sdk/client-dynamodb/index.js"; + +vi.mock("@webiny/aws-sdk/client-dynamodb/index.js", () => ({ + DynamoDB: vi.fn() +})); +``` + +Then add a new `describe("checkAccess", ...)` block at the end of the existing `describe("DdbProcessor", ...)`: + +```ts +describe("checkAccess", () => { + let mockDescribeTable: ReturnType; + + beforeEach(() => { + mockDescribeTable = vi.fn(); + vi.mocked(DynamoDB).mockImplementation( + () => ({ describeTable: mockDescribeTable }) as unknown as DynamoDB + ); + }); + + it("returns ok entries for source and target tables when DescribeTable succeeds", async () => { + mockDescribeTable.mockResolvedValue({}); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor.name === "DdbProcessorImpl")!; + + const entries = await processor.checkAccess(); + + expect(entries).toHaveLength(2); + expect(entries[0]).toEqual({ label: "DynamoDB source table: source-table", status: "ok" }); + expect(entries[1]).toEqual({ label: "DynamoDB target table: target-table", status: "ok" }); + }); + + it("returns denied when DescribeTable throws AccessDeniedException on source", async () => { + mockDescribeTable + .mockRejectedValueOnce( + Object.assign(new Error("Access denied"), { name: "AccessDeniedException" }) + ) + .mockResolvedValue({}); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor.name === "DdbProcessorImpl")!; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "DynamoDB source table: source-table", + status: "denied" + }); + expect(entries[1]).toEqual({ + label: "DynamoDB target table: target-table", + status: "ok" + }); + }); + + it("returns unknown when DescribeTable throws a non-access error", async () => { + mockDescribeTable.mockRejectedValue( + Object.assign(new Error("connection timeout"), { name: "ETIMEDOUT" }) + ); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor.name === "DdbProcessorImpl")!; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "DynamoDB source table: source-table", + status: "unknown" + }); + expect(entries[1]).toEqual({ + label: "DynamoDB target table: target-table", + status: "unknown" + }); + }); +}); +``` + +- [ ] **Step 4: Run the new tests to verify they fail** + +```bash +yarn test __tests__/features/DdbProcessor/DdbProcessor.test.ts +``` + +Expected: the three new `checkAccess` tests FAIL (the stub returns `[]`, so `entries` has length 0). + +- [ ] **Step 5: Implement `DdbProcessor.checkAccess()`** + +Replace the stub in `DdbProcessorImpl` with: + +```ts +import { DynamoDB } from "@webiny/aws-sdk/client-dynamodb/index.js"; +import { AccessCheck, Processor } from "~/domain/pipeline/abstractions/Processor.ts"; +import { isAccessDeniedError } from "~/base/index.ts"; +``` + +(Add these to the imports at the top of `DdbProcessor.ts`.) + +Replace the stub method body: + +```ts +public async checkAccess(): Promise { + const [sourceEntry, targetEntry] = await Promise.all([ + this.describeTable( + this.config.source.credentials, + this.config.source.region, + this.config.source.dynamodb.tableName, + "source" + ), + this.describeTable( + this.config.target.credentials, + this.config.target.region, + this.config.target.dynamodb.tableName, + "target" + ) + ]); + return [sourceEntry, targetEntry]; +} + +private async describeTable( + credentials: MigrationConfig.Interface["source"]["credentials"], + region: string, + tableName: string, + side: string +): Promise { + const label = `DynamoDB ${side} table: ${tableName}`; + try { + const client = new DynamoDB({ region, credentials: credentials as never }); + await client.describeTable({ TableName: tableName }); + return { label, status: "ok" }; + } catch (error) { + if (isAccessDeniedError(error)) { + return { label, status: "denied" }; + } + return { label, status: "unknown" }; + } +} +``` + +- [ ] **Step 6: Run the tests to verify they pass** + +```bash +yarn test __tests__/features/DdbProcessor/DdbProcessor.test.ts +``` + +Expected: all tests PASS. + +- [ ] **Step 7: Run full suite to check for regressions** + +```bash +yarn test +``` + +Expected: all tests PASS. + +- [ ] **Step 8: Commit** + +```bash +git add src/base/isRetryableAwsError.ts src/base/index.ts src/features/DdbProcessor/DdbProcessor.ts __tests__/features/DdbProcessor/DdbProcessor.test.ts +git commit -m "feat: implement DdbProcessor.checkAccess() via DescribeTable" +``` + +--- + +### Task 3: AuditLogProcessor.checkAccess() + +**Files:** +- Modify: `src/features/AuditLogProcessor/AuditLogProcessor.ts` +- Modify: `__tests__/features/AuditLogProcessor/AuditLogProcessor.test.ts` + +- [ ] **Step 1: Write failing tests for `AuditLogProcessor.checkAccess()`** + +Add to `__tests__/features/AuditLogProcessor/AuditLogProcessor.test.ts`: + +```ts +import { vi, beforeEach } from "vitest"; +import { DynamoDB } from "@webiny/aws-sdk/client-dynamodb/index.js"; + +vi.mock("@webiny/aws-sdk/client-dynamodb/index.js", () => ({ + DynamoDB: vi.fn() +})); +``` + +Add a `describe("checkAccess", ...)` block: + +```ts +describe("checkAccess", () => { + let mockDescribeTable: ReturnType; + + beforeEach(() => { + mockDescribeTable = vi.fn(); + vi.mocked(DynamoDB).mockImplementation( + () => ({ describeTable: mockDescribeTable }) as unknown as DynamoDB + ); + }); + + it("returns empty array when audit log is not configured", async () => { + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor.name === "AuditLogProcessorImpl")!; + + const entries = await processor.checkAccess(); + + expect(entries).toHaveLength(0); + }); + + it("returns ok when DescribeTable succeeds for the audit log table", async () => { + mockDescribeTable.mockResolvedValue({}); + const container = createDdbContainer({ + auditLogTable: "audit-log-table" + }); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor.name === "AuditLogProcessorImpl")!; + + const entries = await processor.checkAccess(); + + expect(entries).toHaveLength(1); + expect(entries[0]).toEqual({ + label: "DynamoDB audit log table: audit-log-table", + status: "ok" + }); + }); + + it("returns denied when DescribeTable throws AccessDeniedException", async () => { + mockDescribeTable.mockRejectedValue( + Object.assign(new Error("Access denied"), { name: "AccessDeniedException" }) + ); + const container = createDdbContainer({ + auditLogTable: "audit-log-table" + }); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor.name === "AuditLogProcessorImpl")!; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "DynamoDB audit log table: audit-log-table", + status: "denied" + }); + }); + + it("returns unknown for non-access errors", async () => { + mockDescribeTable.mockRejectedValue(new Error("ResourceNotFoundException")); + const container = createDdbContainer({ + auditLogTable: "audit-log-table" + }); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor.name === "AuditLogProcessorImpl")!; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "DynamoDB audit log table: audit-log-table", + status: "unknown" + }); + }); +}); +``` + +Note: `createDdbContainer` in `__tests__/containers/ddb.ts` needs to accept an `auditLogTable` option. Check `DdbContainerOptions` — if it doesn't have `auditLogTable`, add it: + +In `__tests__/containers/ddb.ts`, add to `DdbContainerOptions`: +```ts +auditLogTable?: string; +``` + +And in the config object inside `createDdbContainer`: +```ts +target: { + // ... existing fields ... + auditLog: options.auditLogTable + ? { dynamodb: { tableName: options.auditLogTable } } + : null +} +``` + +- [ ] **Step 2: Run the new tests to verify they fail** + +```bash +yarn test __tests__/features/AuditLogProcessor/AuditLogProcessor.test.ts +``` + +Expected: the new `checkAccess` tests FAIL. + +- [ ] **Step 3: Implement `AuditLogProcessor.checkAccess()`** + +Add imports to the top of `AuditLogProcessor.ts`: + +```ts +import { DynamoDB } from "@webiny/aws-sdk/client-dynamodb/index.js"; +import { isAccessDeniedError } from "~/base/index.ts"; +``` + +Replace the stub method body in `AuditLogProcessorImpl`: + +```ts +public async checkAccess(): Promise { + const tableName = this.config.target.auditLog?.dynamodb?.tableName ?? null; + if (!tableName) { + return []; + } + const label = `DynamoDB audit log table: ${tableName}`; + try { + const client = new DynamoDB({ + region: this.config.target.region, + credentials: this.config.target.credentials as never + }); + await client.describeTable({ TableName: tableName }); + return [{ label, status: "ok" }]; + } catch (error) { + if (isAccessDeniedError(error)) { + return [{ label, status: "denied" }]; + } + return [{ label, status: "unknown" }]; + } +} +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +```bash +yarn test __tests__/features/AuditLogProcessor/AuditLogProcessor.test.ts +``` + +Expected: all tests PASS. + +- [ ] **Step 5: Run full suite** + +```bash +yarn test +``` + +Expected: all tests PASS. + +- [ ] **Step 6: Commit** + +```bash +git add src/features/AuditLogProcessor/AuditLogProcessor.ts __tests__/features/AuditLogProcessor/AuditLogProcessor.test.ts __tests__/containers/ddb.ts +git commit -m "feat: implement AuditLogProcessor.checkAccess() via DescribeTable" +``` + +--- + +### Task 4: S3Processor.checkAccess() + +**Files:** +- Modify: `src/features/S3Processor/S3Processor.ts` +- Modify: `__tests__/features/S3Processor/S3Processor.test.ts` + +- [ ] **Step 1: Write failing tests for `S3Processor.checkAccess()`** + +The test file already imports `vi` and `beforeEach`. Add the mock at the top of the file (it is hoisted by vitest, but write it near other imports): + +```ts +import { S3 } from "@webiny/aws-sdk/client-s3/index.js"; + +vi.mock("@webiny/aws-sdk/client-s3/index.js", () => ({ + S3: vi.fn() +})); +``` + +Add a `describe("checkAccess", ...)` block inside the outer `describe("S3Processor", ...)`: + +```ts +describe("checkAccess", () => { + let mockHeadBucket: ReturnType; + + beforeEach(() => { + mockHeadBucket = vi.fn(); + vi.mocked(S3).mockImplementation( + () => ({ headBucket: mockHeadBucket }) as unknown as S3 + ); + }); + + it("returns ok entries for source and target buckets when HeadBucket succeeds", async () => { + mockHeadBucket.mockResolvedValue({}); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === S3Processor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries).toHaveLength(2); + expect(entries[0]).toEqual({ label: "S3 source bucket: source-bucket", status: "ok" }); + expect(entries[1]).toEqual({ label: "S3 target bucket: target-bucket", status: "ok" }); + }); + + it("returns denied when HeadBucket throws AccessDenied on source", async () => { + mockHeadBucket + .mockRejectedValueOnce( + Object.assign(new Error("Access denied"), { name: "AccessDenied" }) + ) + .mockResolvedValue({}); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === S3Processor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ label: "S3 source bucket: source-bucket", status: "denied" }); + expect(entries[1]).toEqual({ label: "S3 target bucket: target-bucket", status: "ok" }); + }); + + it("returns denied when HeadBucket returns HTTP 403", async () => { + mockHeadBucket.mockRejectedValue( + Object.assign(new Error("Forbidden"), { + $metadata: { httpStatusCode: 403 } + }) + ); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === S3Processor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ label: "S3 source bucket: source-bucket", status: "denied" }); + expect(entries[1]).toEqual({ label: "S3 target bucket: target-bucket", status: "denied" }); + }); + + it("returns unknown for non-access errors", async () => { + mockHeadBucket.mockRejectedValue(new Error("NoSuchBucket")); + const container = createDdbContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor === S3Processor) as unknown as Processor.Interface; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "S3 source bucket: source-bucket", + status: "unknown" + }); + }); +}); +``` + +- [ ] **Step 2: Run the new tests to verify they fail** + +```bash +yarn test __tests__/features/S3Processor/S3Processor.test.ts +``` + +Expected: the new `checkAccess` tests FAIL (stub returns `[]`). + +- [ ] **Step 3: Implement `S3Processor.checkAccess()`** + +Add to the imports at the top of `S3Processor.ts`: + +```ts +import { S3 } from "@webiny/aws-sdk/client-s3/index.js"; +import { isAccessDeniedError } from "~/base/index.ts"; +``` + +Replace the stub method body in `S3ProcessorImpl`: + +```ts +public async checkAccess(): Promise { + const [sourceEntry, targetEntry] = await Promise.all([ + this.headBucket( + this.config.source.credentials, + this.config.source.region, + this.config.source.s3.bucket, + "source" + ), + this.headBucket( + this.config.target.credentials, + this.config.target.region, + this.config.target.s3.bucket, + "target" + ) + ]); + return [sourceEntry, targetEntry]; +} + +private async headBucket( + credentials: MigrationConfig.Interface["source"]["credentials"], + region: string, + bucket: string, + side: string +): Promise { + const label = `S3 ${side} bucket: ${bucket}`; + try { + const client = new S3({ region, credentials: credentials as never }); + await client.headBucket({ Bucket: bucket }); + return { label, status: "ok" }; + } catch (error) { + if (isAccessDeniedError(error)) { + return { label, status: "denied" }; + } + return { label, status: "unknown" }; + } +} +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +```bash +yarn test __tests__/features/S3Processor/S3Processor.test.ts +``` + +Expected: all tests PASS. + +- [ ] **Step 5: Run full suite** + +```bash +yarn test +``` + +Expected: all tests PASS. + +- [ ] **Step 6: Commit** + +```bash +git add src/features/S3Processor/S3Processor.ts __tests__/features/S3Processor/S3Processor.test.ts +git commit -m "feat: implement S3Processor.checkAccess() via HeadBucket" +``` + +--- + +### Task 5: OsProcessor.checkAccess() + +**Files:** +- Modify: `src/features/OsProcessor/OsProcessor.ts` +- Modify: `__tests__/features/OsProcessor/OsProcessor.test.ts` + +- [ ] **Step 1: Write failing tests for `OsProcessor.checkAccess()`** + +Add to `__tests__/features/OsProcessor/OsProcessor.test.ts` (check if `vi` is already imported — add it if not): + +```ts +import { vi } from "vitest"; +``` + +Add a `describe("checkAccess", ...)` block: + +```ts +describe("checkAccess", () => { + it("returns ok when listIndexes succeeds", async () => { + const container = createOsContainer(); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor.name === "OsProcessorImpl")!; + + const entries = await processor.checkAccess(); + + expect(entries).toHaveLength(1); + expect(entries[0]).toEqual({ + label: "OpenSearch cluster: https://es.example.com", + status: "ok" + }); + }); + + it("returns denied when listIndexes throws a 403 error", async () => { + const container = createOsContainer(); + const osClient = container.resolve(OpenSearchClient) as MockOpenSearchClient; + vi.spyOn(osClient, "listIndexes").mockRejectedValue( + Object.assign(new Error("Forbidden"), { statusCode: 403 }) + ); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor.name === "OsProcessorImpl")!; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "OpenSearch cluster: https://es.example.com", + status: "denied" + }); + }); + + it("returns unknown when listIndexes throws a non-access error", async () => { + const container = createOsContainer(); + const osClient = container.resolve(OpenSearchClient) as MockOpenSearchClient; + vi.spyOn(osClient, "listIndexes").mockRejectedValue(new Error("connection refused")); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor.name === "OsProcessorImpl")!; + + const entries = await processor.checkAccess(); + + expect(entries[0]).toEqual({ + label: "OpenSearch cluster: https://es.example.com", + status: "unknown" + }); + }); + + it("returns empty array when OpenSearch is not configured", async () => { + const container = createOsContainer({ noOpenSearch: true }); + const processor = container + .resolveAll(Processor) + .find(p => p.constructor.name === "OsProcessorImpl")!; + + const entries = await processor.checkAccess(); + + expect(entries).toHaveLength(0); + }); +}); +``` + +Note: The `noOpenSearch: true` option needs to be added to `createOsContainer` and `OsContainerOptions`. Add it to `__tests__/containers/os.ts`: + +```ts +export interface OsContainerOptions { + // ... existing fields ... + noOpenSearch?: boolean; +} +``` + +And in the config object: +```ts +target: { + // ... existing fields ... + opensearch: options.noOpenSearch + ? undefined + : { + endpoint: "https://es.example.com", + tableName: "target-os", + service: "opensearch" as const, + indexPrefix: options.indexPrefix ?? "" + } +} +``` + +- [ ] **Step 2: Run the new tests to verify they fail** + +```bash +yarn test __tests__/features/OsProcessor/OsProcessor.test.ts +``` + +Expected: the new `checkAccess` tests FAIL. + +- [ ] **Step 3: Implement `OsProcessor.checkAccess()`** + +Replace the stub method body in `OsProcessorImpl`: + +```ts +public async checkAccess(): Promise { + if (!this.config.target.opensearch) { + return []; + } + const endpoint = this.config.target.opensearch.endpoint; + const label = `OpenSearch cluster: ${endpoint}`; + try { + await this.osClient.listIndexes(); + return [{ label, status: "ok" }]; + } catch (error) { + const e = error as { statusCode?: number }; + if (e.statusCode === 401 || e.statusCode === 403) { + return [{ label, status: "denied" }]; + } + return [{ label, status: "unknown" }]; + } +} +``` + +Note: `this.osClient` uses the lazy getter already defined in `OsProcessorImpl`. No new imports needed. + +- [ ] **Step 4: Run the tests to verify they pass** + +```bash +yarn test __tests__/features/OsProcessor/OsProcessor.test.ts +``` + +Expected: all tests PASS. + +- [ ] **Step 5: Run full suite** + +```bash +yarn test +``` + +Expected: all tests PASS. + +- [ ] **Step 6: Commit** + +```bash +git add src/features/OsProcessor/OsProcessor.ts __tests__/features/OsProcessor/OsProcessor.test.ts __tests__/containers/os.ts +git commit -m "feat: implement OsProcessor.checkAccess() via listIndexes" +``` + +--- + +### Task 6: AccessChecker abstraction, implementation, and tests + +**Files:** +- Create: `src/features/AccessChecker/abstractions/AccessChecker.ts` +- Create: `src/features/AccessChecker/AccessChecker.ts` +- Create: `src/features/AccessChecker/feature.ts` +- Create: `src/features/AccessChecker/index.ts` +- Create: `__tests__/features/AccessChecker/AccessChecker.test.ts` + +- [ ] **Step 1: Write the failing test first** + +Create `__tests__/features/AccessChecker/AccessChecker.test.ts`: + +```ts +import { describe, expect, it, vi } from "vitest"; +import { Container } from "@webiny/di"; +import { ContainerToken } from "~/base/index.ts"; +import { PipelineRunner } from "~/features/PipelineRunner/index.ts"; +import { AccessChecker, AccessCheckerFeature } from "~/features/AccessChecker/index.ts"; + +function makeRunner( + processors: Array<{ checkAccess(): Promise<{ label: string; status: string }[]>; execute(): Promise }> +): PipelineRunner.Interface { + return { + register: vi.fn(), + run: vi.fn(), + getProcessors: vi.fn().mockReturnValue(processors), + getShardStats: vi.fn().mockReturnValue(null) + } as unknown as PipelineRunner.Interface; +} + +describe("AccessChecker", () => { + it("returns a flat report from all processor checkAccess results", async () => { + const p1 = { + checkAccess: vi.fn().mockResolvedValue([{ label: "DynamoDB source", status: "ok" }]), + execute: vi.fn() + }; + const p2 = { + checkAccess: vi.fn().mockResolvedValue([ + { label: "S3 source bucket: sb", status: "ok" }, + { label: "S3 target bucket: tb", status: "denied" } + ]), + execute: vi.fn() + }; + + const container = new Container(); + container.registerInstance(ContainerToken, container); + container.registerInstance(PipelineRunner, makeRunner([p1, p2])); + AccessCheckerFeature.register(container); + + const checker = container.resolve(AccessChecker); + const report = await checker.run(); + + expect(report).toHaveLength(3); + expect(report[0]).toEqual({ label: "DynamoDB source", status: "ok" }); + expect(report[1]).toEqual({ label: "S3 source bucket: sb", status: "ok" }); + expect(report[2]).toEqual({ label: "S3 target bucket: tb", status: "denied" }); + }); + + it("returns empty report when no processors are registered", async () => { + const container = new Container(); + container.registerInstance(ContainerToken, container); + container.registerInstance(PipelineRunner, makeRunner([])); + AccessCheckerFeature.register(container); + + const checker = container.resolve(AccessChecker); + const report = await checker.run(); + + expect(report).toHaveLength(0); + }); + + it("returns empty report when all processors return empty arrays", async () => { + const p = { + checkAccess: vi.fn().mockResolvedValue([]), + execute: vi.fn() + }; + const container = new Container(); + container.registerInstance(ContainerToken, container); + container.registerInstance(PipelineRunner, makeRunner([p])); + AccessCheckerFeature.register(container); + + const checker = container.resolve(AccessChecker); + const report = await checker.run(); + + expect(report).toHaveLength(0); + }); +}); +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +yarn test __tests__/features/AccessChecker/AccessChecker.test.ts +``` + +Expected: FAIL — `~/features/AccessChecker/index.ts` does not exist. + +- [ ] **Step 3: Create the AccessChecker abstraction** + +Create `src/features/AccessChecker/abstractions/AccessChecker.ts`: + +```ts +import { createAbstraction } from "~/base/index.ts"; +import type { AccessCheck } from "~/domain/pipeline/abstractions/Processor.ts"; + +interface IAccessChecker { + run(): Promise; +} + +export const AccessChecker = createAbstraction("Core/AccessChecker"); + +export namespace AccessChecker { + export type Interface = IAccessChecker; + export type Report = AccessCheck.Report; + export type Entry = AccessCheck.Entry; +} +``` + +- [ ] **Step 4: Create the AccessChecker implementation** + +Create `src/features/AccessChecker/AccessChecker.ts`: + +```ts +import { AccessChecker as AccessCheckerAbstraction } from "./abstractions/AccessChecker.ts"; +import { PipelineRunner } from "~/features/PipelineRunner/index.ts"; +import type { AccessCheck } from "~/domain/pipeline/abstractions/Processor.ts"; + +class AccessCheckerImpl implements AccessCheckerAbstraction.Interface { + public constructor(private readonly runner: PipelineRunner.Interface) {} + + public async run(): Promise { + const processors = this.runner.getProcessors(); + const nested = await Promise.all(processors.map(p => p.checkAccess())); + return nested.flat(); + } +} + +export const AccessChecker = AccessCheckerAbstraction.createImplementation({ + implementation: AccessCheckerImpl, + dependencies: [PipelineRunner] +}); +``` + +- [ ] **Step 5: Create the feature registration** + +Create `src/features/AccessChecker/feature.ts`: + +```ts +import { createFeature } from "~/base/index.ts"; +import { AccessChecker } from "./AccessChecker.ts"; + +export const AccessCheckerFeature = createFeature({ + name: "Core/AccessCheckerFeature", + register(container) { + container.register(AccessChecker).inSingletonScope(); + } +}); +``` + +- [ ] **Step 6: Create the index** + +Create `src/features/AccessChecker/index.ts`: + +```ts +export { AccessChecker } from "./abstractions/AccessChecker.ts"; +export { AccessCheckerFeature } from "./feature.ts"; +``` + +- [ ] **Step 7: Run the tests to verify they pass** + +```bash +yarn test __tests__/features/AccessChecker/AccessChecker.test.ts +``` + +Expected: all tests PASS. + +- [ ] **Step 8: Run full suite** + +```bash +yarn test +``` + +Expected: all tests PASS. + +- [ ] **Step 9: Commit** + +```bash +git add src/features/AccessChecker/ __tests__/features/AccessChecker/ +git commit -m "feat: add AccessChecker aggregator service" +``` + +--- + +### Task 7: handler.ts + bootstrap.ts + test container wiring + +**Files:** +- Modify: `src/commands/run/handler.ts` +- Modify: `src/bootstrap.ts` +- Modify: `__tests__/containers/ddb.ts` +- Modify: `__tests__/containers/os.ts` + +- [ ] **Step 1: Register `AccessCheckerFeature` in `src/bootstrap.ts`** + +Add the import after the existing processor feature imports: + +```ts +import { AccessCheckerFeature } from "~/features/AccessChecker/index.ts"; +``` + +Add the registration at the end of the feature registrations block (after `OsProcessorFeature.register(container)`): + +```ts +AccessCheckerFeature.register(container); +``` + +- [ ] **Step 2: Register `AccessCheckerFeature` in `__tests__/containers/ddb.ts`** + +Add the import: + +```ts +import { AccessCheckerFeature } from "../../src/features/AccessChecker/index.ts"; +``` + +Add the registration after `AuditLogProcessorFeature.register(container)`: + +```ts +AccessCheckerFeature.register(container); +``` + +- [ ] **Step 3: Register `AccessCheckerFeature` in `__tests__/containers/os.ts`** + +Add the import: + +```ts +import { AccessCheckerFeature } from "../../src/features/AccessChecker/index.ts"; +``` + +Add the registration after `OsProcessorFeature.register(container)`: + +```ts +AccessCheckerFeature.register(container); +``` + +- [ ] **Step 4: Integrate `AccessChecker` in `src/commands/run/handler.ts`** + +Add the import near the existing feature imports: + +```ts +import { AccessChecker } from "~/features/AccessChecker/index.ts"; +``` + +Add the access check block immediately after `preset.configure(...)` (before the existing `guardWarnings` block, around line 95): + +```ts +const accessChecker = container.resolve(AccessChecker); +const accessReport = await accessChecker.run(); + +if (accessReport.length > 0) { + logger.info("Pre-transfer access check:"); + for (const entry of accessReport) { + if (entry.status === "ok") { + logger.info(` ok ${entry.label}`); + } else if (entry.status === "denied") { + logger.error(` DENIED ${entry.label}`); + } else { + logger.warn(` unknown ${entry.label}`); + } + } +} + +const denied = accessReport.filter(e => e.status === "denied"); +if (denied.length > 0) { + logger.fatal("Access check failed — aborting transfer."); + process.exit(1); +} +``` + +- [ ] **Step 5: Run the full test suite** + +```bash +yarn test +``` + +Expected: all tests PASS. + +- [ ] **Step 6: Run type-check** + +```bash +yarn ts-check +``` + +Expected: 0 new errors (the 5 pre-existing ts-check errors on main are unrelated to this feature). + +- [ ] **Step 7: Commit** + +```bash +git add src/commands/run/handler.ts src/bootstrap.ts __tests__/containers/ddb.ts __tests__/containers/os.ts +git commit -m "feat: wire AccessChecker into run handler; abort on denied access entries" +``` diff --git a/docs/superpowers/plans/2026-05-11-flush-every.md b/docs/superpowers/plans/2026-05-11-flush-every.md new file mode 100644 index 00000000..fec091c5 --- /dev/null +++ b/docs/superpowers/plans/2026-05-11-flush-every.md @@ -0,0 +1,555 @@ +# Periodic Shard Flush (flushEvery) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Bound per-shard memory usage by flushing `processor.execute()` every N records instead of once at shard end. + +**Architecture:** Add `flushEvery: number` to `tuningSchema`; inject `MigrationConfig` into `PipelineRunner`; replace the single shard-end `execute` drain with a periodic mid-shard flush triggered by a record counter, plus a final flush for the remainder. `afterShard` is unchanged — still fires once per shard after all flushes. + +**Tech Stack:** Zod (schema), Vitest (tests), existing DI container wiring. + +--- + +## Task 1: Add `flushEvery` to `tuningSchema` + +**Files:** +- Modify: `src/features/MigrationConfig/schemas/shared.schema.ts` +- Test: `__tests__/features/MigrationConfig/createConfig.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Append a new `describe` block to `__tests__/features/MigrationConfig/createConfig.test.ts`: + +```typescript +describe("createConfig — tuning.flushEvery", () => { + it("accepts a positive integer", () => { + const config = createConfig({ + source: baseSource, + target: baseTarget, + pipeline: {}, + tuning: { flushEvery: 100 } + }); + expect(config.tuning?.flushEvery).toBe(100); + }); + + it("rejects 0 (not positive)", () => { + expect(() => + createConfig({ + source: baseSource, + target: baseTarget, + pipeline: {}, + tuning: { flushEvery: 0 } + }) + ).toThrow(); + }); + + it("rejects -1 (negative)", () => { + expect(() => + createConfig({ + source: baseSource, + target: baseTarget, + pipeline: {}, + tuning: { flushEvery: -1 } + }) + ).toThrow(); + }); + + it("rejects 1.5 (non-integer)", () => { + expect(() => + createConfig({ + source: baseSource, + target: baseTarget, + pipeline: {}, + tuning: { flushEvery: 1.5 } + }) + ).toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +```bash +yarn test __tests__/features/MigrationConfig/createConfig.test.ts +``` + +Expected: the 4 new tests fail (unknown field / no validation yet). + +- [ ] **Step 3: Add `flushEvery` to `tuningSchema`** + +In `src/features/MigrationConfig/schemas/shared.schema.ts`, add `flushEvery` as the first field of `tuningSchema`: + +```typescript +export const tuningSchema = z + .object({ + flushEvery: z.number().int().positive().optional(), + ddb: z + .object({ + maxRetries: z.number().int().nonnegative().optional(), + initialBackoffMs: z.number().int().nonnegative().optional(), + requestTimeoutMs: z.number().int().positive().optional() + }) + .optional(), + s3: z + .object({ + concurrency: z.number().int().positive().optional(), + maxRetries: z.number().int().nonnegative().optional(), + initialBackoffMs: z.number().int().nonnegative().optional(), + requestTimeoutMs: z.number().int().positive().optional() + }) + .optional(), + os: z + .object({ + maxRetries: z.number().int().nonnegative().optional(), + retryScheduleMs: z.array(z.number().int().nonnegative()).optional(), + gzipConcurrency: z.number().int().positive().optional() + }) + .optional() + }) + .optional(); +``` + +- [ ] **Step 4: Run tests to verify they pass** + +```bash +yarn test __tests__/features/MigrationConfig/createConfig.test.ts +``` + +Expected: all tests pass including the 4 new ones. + +- [ ] **Step 5: Commit** + +```bash +git add src/features/MigrationConfig/schemas/shared.schema.ts \ + __tests__/features/MigrationConfig/createConfig.test.ts +git commit -m "feat: add tuning.flushEvery to schema" +``` + +--- + +## Task 2: Wire `MigrationConfig` into `PipelineRunner` + periodic flush + +**Files:** +- Modify: `src/features/PipelineRunner/PipelineRunner.ts` +- Modify: `__tests__/features/PipelineRunner/PipelineRunner.test.ts` + +- [ ] **Step 1: Write the failing tests** + +In `__tests__/features/PipelineRunner/PipelineRunner.test.ts`, make two changes: + +**2a. Add the `MigrationConfig` import** at the top of the file alongside the other imports: + +```typescript +import { MigrationConfig } from "~/features/MigrationConfig/abstractions/MigrationConfig.ts"; +``` + +**2b. Update `makeContainer`** to accept `flushEvery` and register `MigrationConfig`: + +Replace the existing signature and body: + +```typescript +function makeContainer(options: { runId?: string; flushEvery?: number } = {}): { + container: Container; + logger: TestLogger; +} { + const container = new Container(); + const logger = new TestLogger(); + container.registerInstance(ContainerToken, container); + container.registerInstance(Logger, logger); + container.registerInstance(TransferContext, { runId: options.runId ?? "test-run-id" }); + container.registerInstance(BaseTransformContextFactory, new FakeBaseContextFactory()); + container.registerInstance(SnapshotWriter, { + async write(): Promise {}, + async close(): Promise {} + }); + container.registerInstance(DroppedRecordLog, new MockDroppedRecordLog()); + container.registerInstance(TransferredRecordLog, new MockTransferredRecordLog()); + container.registerInstance( + MigrationConfig, + { + tuning: + options.flushEvery !== undefined ? { flushEvery: options.flushEvery } : undefined + } as unknown as MigrationConfig.Interface + ); + container.register(FakeScannerImpl).inSingletonScope(); + container.register(FakeProcessorImpl).inSingletonScope(); + container.register(FakeHookAImpl).inSingletonScope(); + container.register(FakeHookBImpl).inSingletonScope(); + PipelineBuilderFactoryFeature.register(container); + PipelineRunnerFeature.register(container); + return { container, logger }; +} +``` + +**2c. Add the new flush describe block** after the existing `describe("PipelineRunner.run()", ...)` block: + +```typescript +describe("PipelineRunner — periodic flush (flushEvery)", () => { + it("flushes mid-shard every flushEvery records", async () => { + const { container } = makeContainer({ flushEvery: 2 }); + const scanner = container.resolve(Scanner) as FakeScanner; + const processor = container.resolve(Processor) as FakeProcessor; + scanner.records = [ + { id: "r1", type: "foo" }, + { id: "r2", type: "foo" }, + { id: "r3", type: "foo" }, + { id: "r4", type: "foo" }, + { id: "r5", type: "foo" } + ]; + + const runner = container.resolve(PipelineRunner); + runner.register(buildPipeline(container, "flush-mid-shard")); + await runner.run(); + + // flushEvery=2, 5 records: flush at 2, flush at 4, final flush at 5 + expect(processor.executed).toHaveLength(3); + expect(processor.executed[0]?.size()).toBe(2); + expect(processor.executed[1]?.size()).toBe(2); + expect(processor.executed[2]?.size()).toBe(1); + }); + + it("flushes exactly N/flushEvery times when count is divisible", async () => { + const { container } = makeContainer({ flushEvery: 2 }); + const scanner = container.resolve(Scanner) as FakeScanner; + const processor = container.resolve(Processor) as FakeProcessor; + scanner.records = [ + { id: "r1", type: "foo" }, + { id: "r2", type: "foo" }, + { id: "r3", type: "foo" }, + { id: "r4", type: "foo" } + ]; + + const runner = container.resolve(PipelineRunner); + runner.register(buildPipeline(container, "flush-divisible")); + await runner.run(); + + // flushEvery=2, 4 records: flush at 2, flush at 4, no remainder + expect(processor.executed).toHaveLength(2); + expect(processor.executed[0]?.size()).toBe(2); + expect(processor.executed[1]?.size()).toBe(2); + }); + + it("no record loss across flush boundaries", async () => { + const { container } = makeContainer({ flushEvery: 2 }); + const scanner = container.resolve(Scanner) as FakeScanner; + const processor = container.resolve(Processor) as FakeProcessor; + scanner.records = [ + { id: "r1", type: "foo" }, + { id: "r2", type: "foo" }, + { id: "r3", type: "foo" } + ]; + + const runner = container.resolve(PipelineRunner); + runner.register(buildPipeline(container, "flush-no-loss")); + await runner.run(); + + const totalCommands = processor.executed.reduce((sum, c) => sum + c.size(), 0); + expect(totalCommands).toBe(3); + }); + + it("afterShard fires exactly once regardless of flush count", async () => { + const { container } = makeContainer({ flushEvery: 2 }); + const scanner = container.resolve(Scanner) as FakeScanner; + const processor = container.resolve(Processor) as FakeProcessor; + scanner.records = [ + { id: "r1", type: "foo" }, + { id: "r2", type: "foo" }, + { id: "r3", type: "foo" }, + { id: "r4", type: "foo" }, + { id: "r5", type: "foo" } + ]; + + const runner = container.resolve(PipelineRunner); + runner.register(buildPipeline(container, "flush-aftershard")); + await runner.run(); + + expect(processor.afterShardCalls).toHaveLength(1); + expect(processor.afterShardCalls[0]).toEqual({ segment: 0, totalSegments: 1 }); + }); + + it("without flushEvery set, uses a single shard-end flush (default 500 > record count)", async () => { + const { container } = makeContainer(); // no flushEvery → default 500 + const scanner = container.resolve(Scanner) as FakeScanner; + const processor = container.resolve(Processor) as FakeProcessor; + scanner.records = [ + { id: "r1", type: "foo" }, + { id: "r2", type: "foo" } + ]; + + const runner = container.resolve(PipelineRunner); + runner.register(buildPipeline(container, "flush-default")); + await runner.run(); + + // 2 records < 500 default → single execute call at shard end + expect(processor.executed).toHaveLength(1); + expect(processor.executed[0]?.size()).toBe(2); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +```bash +yarn test __tests__/features/PipelineRunner/PipelineRunner.test.ts +``` + +Expected: the 5 new tests fail (MigrationConfig not injected yet, no flush logic yet). Existing tests may also fail with a DI error about unresolved `MigrationConfig` — that's expected. + +- [ ] **Step 3: Add `MigrationConfig` dependency to `PipelineRunnerImpl`** + +In `src/features/PipelineRunner/PipelineRunner.ts`: + +Add the import at the top: +```typescript +import { MigrationConfig } from "~/features/MigrationConfig/abstractions/MigrationConfig.ts"; +``` + +Add `config` as the second constructor parameter (after `container`): +```typescript +public constructor( + private readonly container: Container, + private readonly config: MigrationConfig.Interface, + private readonly logger: Logger.Interface, + private readonly transferContext: TransferContext.Interface, + private readonly baseContextFactory: BaseTransformContextFactory.Interface, + private readonly snapshotWriter: SnapshotWriter.Interface, + private readonly droppedLog: DroppedRecordLog.Interface, + private readonly transferredLog: TransferredRecordLog.Interface +) {} +``` + +Update the `dependencies` array at the bottom of the file: +```typescript +export const PipelineRunner = PipelineRunnerAbstraction.createImplementation({ + implementation: PipelineRunnerImpl, + dependencies: [ + ContainerToken, + MigrationConfig, + Logger, + TransferContext, + BaseTransformContextFactory, + SnapshotWriter, + DroppedRecordLog, + TransferredRecordLog + ] +}); +``` + +- [ ] **Step 4: Add `flushShard` helper and rewrite `runShard`** + +Replace the `runShard` method body and add `flushShard` after the `runShard` closing brace. The full replacement for `runShard` (currently lines 240–346): + +```typescript +private async runShard(params: RunShardParams): Promise { + const { mergeGroupId, pipelines, scanner, shard, pipelineProcessors, shardCtx } = params; + + const flushEvery = this.config.tuning?.flushEvery ?? 500; + const processorOrder = this.collectProcessorOrder(pipelines, pipelineProcessors); + let pendingCommands = new Commands(); + let recordCount = 0; + + const perPipelineTransferred: Map = new Map(); + const perPipelineBlackholed: Map = new Map(); + const unmatchedByType: Map = new Map(); + + for await (const record of scanner.scan(shard)) { + let matched = false; + for (const pipeline of pipelines) { + if (!pipeline.accepts(record)) { + continue; + } + matched = true; + const processors = pipelineProcessors.get(pipeline)!; + await this.snapshotWriter.write( + `${pipeline.name}/segment-${shardCtx.segment}.source.jsonl`, + record + ); + const result = await this.runRecord( + pipeline, + processors, + record, + pendingCommands, + shardCtx + ); + if (result instanceof RecordDisposition.Blackholed) { + this.droppedLog.add(record, result); + perPipelineBlackholed.set( + pipeline.name, + (perPipelineBlackholed.get(pipeline.name) ?? 0) + 1 + ); + } else { + perPipelineTransferred.set( + pipeline.name, + (perPipelineTransferred.get(pipeline.name) ?? 0) + 1 + ); + this.transferredLog.add(record, pipeline.name); + } + break; + } + if (!matched) { + const { PK, SK, TYPE } = record as any; + const typeKey: string = TYPE && TYPE !== "unknown" ? TYPE : `${PK}:${SK}`; + unmatchedByType.set(typeKey, (unmatchedByType.get(typeKey) ?? 0) + 1); + this.logger.warn(`unmatched record — TYPE=${typeKey} PK=${PK} SK=${SK}`); + await this.snapshotWriter.write( + `dropped/segment-${shardCtx.segment}.jsonl`, + record + ); + this.droppedLog.add(record, new RecordDisposition.Unmatched()); + } + + recordCount++; + if (recordCount % flushEvery === 0) { + await this.flushShard(pendingCommands, processorOrder); + pendingCommands = new Commands(); + } + } + + this.logShardSummary( + mergeGroupId, + shardCtx, + perPipelineTransferred, + perPipelineBlackholed, + unmatchedByType + ); + + if (pendingCommands.size() > 0) { + await this.flushShard(pendingCommands, processorOrder); + } + + for (const processor of processorOrder) { + if (!processor.afterShard) { + continue; + } + await processor.afterShard(shardCtx); + } + + this.droppedLog.flush(shardCtx.segment); + this.transferredLog.flush(shardCtx.segment); + + return { + transferred: perPipelineTransferred, + blackholed: perPipelineBlackholed, + unmatched: unmatchedByType + }; +} + +private async flushShard(commands: Commands, processors: ProcessorInstance[]): Promise { + for (const processor of processors) { + await processor.execute(commands); + } + this.warnUnclaimedKeys(commands); +} +``` + +Also delete the old `collectProcessorOrder` call that was at lines 322–325 (it is now at the top of `runShard`) and the old `warnUnclaimedKeys(shardCommands)` call at line 337 (now inside `flushShard`). + +- [ ] **Step 5: Run tests to verify they pass** + +```bash +yarn test __tests__/features/PipelineRunner/PipelineRunner.test.ts +``` + +Expected: all tests pass — the 5 new flush tests plus all pre-existing runner tests. + +- [ ] **Step 6: Run the full test suite** + +```bash +yarn test +``` + +Expected: all tests pass. + +- [ ] **Step 7: Commit** + +```bash +git add src/features/PipelineRunner/PipelineRunner.ts \ + __tests__/features/PipelineRunner/PipelineRunner.test.ts +git commit -m "feat: periodic shard flush via tuning.flushEvery" +``` + +--- + +## Task 3: Wire `FLUSH_EVERY` in config templates + +**Files:** +- Modify: `templates/projects/example/config.ts` +- Modify: `templates/internal-project/config.ts` +- Modify: `templates/projects/example/.env.example` +- Modify: `templates/internal-project/.env.example` + +No tests — templates are scaffolding, not executed by the test suite. + +- [ ] **Step 1: Update `templates/projects/example/config.ts`** + +Add a `tuning` section after `pipeline`. Replace the closing brace of `createConfig({...})` to add: + +```typescript + pipeline: { + segments: numberFromEnv("SEGMENTS", 4), + modelsDir: fromEnv("MODELS_DIR", "./models"), + presetsDir: "./presets" + }, + tuning: { + flushEvery: numberFromEnv("FLUSH_EVERY", 500) + } +``` + +(The rest of the file — imports, `loadEnv`, `source`, `target`, and the commented `debug` block — remains unchanged.) + +- [ ] **Step 2: Update `templates/internal-project/config.ts`** + +Add a `tuning` section after `pipeline`. Replace: + +```typescript + pipeline: { + segments: numberFromEnv("SEGMENTS", 4), + modelsDir: fromEnv("MODELS_DIR", "./models"), + presetsDir: "./presets" + } +}); +``` + +with: + +```typescript + pipeline: { + segments: numberFromEnv("SEGMENTS", 4), + modelsDir: fromEnv("MODELS_DIR", "./models"), + presetsDir: "./presets" + }, + tuning: { + flushEvery: numberFromEnv("FLUSH_EVERY", 500) + } +}); +``` + +- [ ] **Step 3: Update `templates/projects/example/.env.example`** + +Append `FLUSH_EVERY` to the `# --- Tuning ---` section. After the existing `SEGMENTS={{SEGMENTS}}` line add: + +``` +# Records read per shard before flushing writes to the target. Lower values +# reduce peak memory on large tables; higher values reduce write round-trips. +# FLUSH_EVERY=500 +``` + +- [ ] **Step 4: Update `templates/internal-project/.env.example`** + +Same addition after `SEGMENTS={{SEGMENTS}}`: + +``` +# Records read per shard before flushing writes to the target. Lower values +# reduce peak memory on large tables; higher values reduce write round-trips. +# FLUSH_EVERY=500 +``` + +- [ ] **Step 5: Commit** + +```bash +git add templates/projects/example/config.ts \ + templates/internal-project/config.ts \ + templates/projects/example/.env.example \ + templates/internal-project/.env.example +git commit -m "feat: wire FLUSH_EVERY env var in config templates" +``` diff --git a/docs/superpowers/specs/2026-05-11-flush-every-design.md b/docs/superpowers/specs/2026-05-11-flush-every-design.md new file mode 100644 index 00000000..e2d0cf3f --- /dev/null +++ b/docs/superpowers/specs/2026-05-11-flush-every-design.md @@ -0,0 +1,87 @@ +# Periodic shard flush (`flushEvery`) + +**Date:** 2026-05-11 +**Status:** Approved + +## Problem + +`PipelineRunner.runShard` accumulates a single `Commands` buffer for the entire shard before calling `processor.execute` at shard end. For a 100 GB DynamoDB table split across 20 segments, each shard holds ~5 GB of `PutRecord` commands in memory before any writes occur. This causes OOM on large tables and hammers the target table with a single write burst at the end of each shard. + +## Solution + +Add a `tuning.flushEvery` config field (default 500 records). Every N records scanned, `PipelineRunner` calls `processor.execute` on the pending commands buffer and resets it. This bounds memory to `N × avg_record_size` and spreads writes evenly across the scan. + +## Design + +### Config — `shared.schema.ts` + +Add `flushEvery` to `tuningSchema`: + +```typescript +tuning: z.object({ + flushEvery: z.number().int().positive().optional(), + ddb: ..., + s3: ..., + os: ... +}).optional() +``` + +No schema-level default — the runner owns the 500 fallback. Users wire it via: + +```typescript +tuning: { + flushEvery: numberFromEnv("FLUSH_EVERY", 500) +} +``` + +Env var: `FLUSH_EVERY`. Integer, positive. If absent, defaults to 500. + +### PipelineRunner — `PipelineRunner.ts` + +`runShard` changes: + +- Replace the single `shardCommands` buffer with a `pendingCommands` buffer and a `recordCount` counter. +- After every `flushEvery` records (all scanned records, matched or not), call `this.flushShard(pendingCommands, processorOrder, shardCtx)` and reset `pendingCommands = new Commands()`. +- At shard end, flush the remainder via the same helper. +- `afterShard` is unchanged — fires once after the loop and final flush. + +New private helper: + +```typescript +private async flushShard( + commands: Commands, + processors: ProcessorInstance[], + shardCtx: Processor.AfterShardContext +): Promise { + for (const processor of processors) { + await processor.execute(commands); + } + this.warnUnclaimedKeys(commands); +} +``` + +`warnUnclaimedKeys` moves into `flushShard` so it runs per flush. The `unclaimedWarned: Set` deduplicates warnings across calls — the first flush that sees an unclaimed key warns; subsequent flushes for the same key are silent. + +`flushEvery` is read once at the top of `runShard`: + +```typescript +const flushEvery = this.config.tuning?.flushEvery ?? 500; +``` + +### Processor interfaces — no changes + +`Processor.Interface.execute(commands)` is unchanged. `DdbProcessor.execute` and `OsProcessor.execute` are stateless — they drain the commands buffer and write. Calling them multiple times per shard is safe. `afterShard` contract is unchanged. + +## Files changed + +| File | Change | +|---|---| +| `src/features/MigrationConfig/schemas/shared.schema.ts` | Add `flushEvery` to `tuningSchema` | +| `src/features/PipelineRunner/PipelineRunner.ts` | Periodic flush in `runShard`, private `flushShard` helper | + +## Trade-offs + +- **Memory ceiling:** `flushEvery × avg_record_size`. At default 500 × 10 KB = ~5 MB; worst-case (500 × 400 KB DDB max) = 200 MB. Users on large-record tables can tune down to 100. +- **Write smoothing:** Writes are spread across the scan instead of a single burst at shard end. No change to DDB batch size (still 25 items per `BatchWriteItem`). +- **Round-trips:** More `BatchWriteItem` calls vs. today's single drain. At 500 records / 25 per batch = 20 calls per flush. Negligible at DDB throughput. +- **Time-based alternative rejected:** A time-based interval doesn't bound memory — fast scans can accumulate millions of records in 30 seconds. Record-count is the only knob that directly controls buffer size. diff --git a/package.json b/package.json index 17287ec8..68b8f0a6 100644 --- a/package.json +++ b/package.json @@ -33,8 +33,8 @@ "license": "MIT", "dependencies": { "@aws-sdk/credential-providers": "^3.1045.0", - "@inquirer/core": "^11.1.9", - "@inquirer/prompts": "^8.4.2", + "@inquirer/core": "^11.1.10", + "@inquirer/prompts": "^8.4.3", "@opensearch-project/opensearch": "^3.6.0", "@webiny/api-headless-cms-ddb-es": "^6.3.0", "@webiny/api-opensearch": "^6.3.0", @@ -59,7 +59,7 @@ "@aws-sdk/client-s3": "^3.1045.0", "@aws-sdk/lib-dynamodb": "^3.1045.0", "@faker-js/faker": "^10.4.0", - "@smithy/util-stream": "^4.5.25", + "@smithy/util-stream": "^4.6.0", "@types/jsdom": "^28.0.1", "@types/node": "^24.12.3", "@types/yargs": "^17.0.35", diff --git a/projects/v5-to-v6/.env.example b/projects/v5-to-v6/.env.example index fbdeebd7..0a711b73 100644 --- a/projects/v5-to-v6/.env.example +++ b/projects/v5-to-v6/.env.example @@ -17,6 +17,7 @@ SOURCE_REGION=us-east-1 # SOURCE_AWS_SESSION_TOKEN= SOURCE_DDB_TABLE=webiny-v5-table SOURCE_S3_BUCKET=webiny-v5-files +SOURCE_AUDIT_LOGS_TABLE= SOURCE_OS_TABLE=webiny-v5-es-table # --- Target (v6 environment) ------------------------------------------------ @@ -27,6 +28,7 @@ TARGET_REGION=us-east-1 # TARGET_AWS_SESSION_TOKEN= TARGET_DDB_TABLE=webiny-v6-table TARGET_S3_BUCKET=webiny-v6-files +TARGET_AUDIT_LOGS_TABLE=webiny-v6-audit-logs TARGET_OS_TABLE=webiny-v6-es-table TARGET_OS_ENDPOINT=https://search-my-domain.us-east-1.es.amazonaws.com TARGET_OS_INDEX_PREFIX= diff --git a/projects/v5-to-v6/config.ts b/projects/v5-to-v6/config.ts new file mode 100644 index 00000000..604fe3a9 --- /dev/null +++ b/projects/v5-to-v6/config.ts @@ -0,0 +1,65 @@ +import { createConfig, fromAwsProfile, fromEnv, loadEnv, numberFromEnv } from "~/index.ts"; + +// Loads projects/v5-to-v6/.env (next to this file). `.env*` is gitignored. +// Region / tables / buckets come from .env. AWS credentials come from +// ~/.aws/credentials via `fromAwsProfile` — set SOURCE_PROFILE / +// TARGET_PROFILE in .env to pick a specific profile, or leave them blank +// to use the default profile. Vars without a default throw fast when +// missing instead of silently passing `undefined` to the AWS SDK. +loadEnv(import.meta.url); + +const DEFAULT_REGION = "eu-central-1"; +const DEFAULT_PROFILE = "default"; + +const sourceOsTable = fromEnv("SOURCE_OS_TABLE", null); +const sourceAuditLogTable = fromEnv("SOURCE_AUDIT_LOGS_TABLE", null); +const targetOsTable = fromEnv("TARGET_OS_TABLE", null); +const targetOsEndpoint = fromEnv("TARGET_OS_ENDPOINT", null); +const targetAuditLogTable = fromEnv("TARGET_AUDIT_LOGS_TABLE", null); + +export default createConfig({ + debug: { + logLevel: "debug", + logFile: true + }, + source: { + region: fromEnv("SOURCE_REGION", DEFAULT_REGION), + credentials: fromAwsProfile({ + profile: fromEnv("SOURCE_PROFILE", DEFAULT_PROFILE) + }), + dynamodb: { + tableName: fromEnv("SOURCE_DDB_TABLE") + }, + s3: { + bucket: fromEnv("SOURCE_S3_BUCKET") + }, + auditLog: sourceAuditLogTable ? { dynamodb: { tableName: sourceAuditLogTable } } : null, + opensearch: sourceOsTable ? { tableName: sourceOsTable } : null + }, + target: { + region: fromEnv("TARGET_REGION", DEFAULT_REGION), + credentials: fromAwsProfile({ + profile: fromEnv("TARGET_PROFILE", DEFAULT_PROFILE) + }), + dynamodb: { + tableName: fromEnv("TARGET_DDB_TABLE") + }, + s3: { + bucket: fromEnv("TARGET_S3_BUCKET") + }, + auditLog: targetAuditLogTable ? { dynamodb: { tableName: targetAuditLogTable } } : null, + opensearch: + targetOsTable && targetOsEndpoint + ? { + endpoint: targetOsEndpoint, + tableName: targetOsTable, + service: "opensearch" as const, + indexPrefix: fromEnv("TARGET_OS_INDEX_PREFIX", "") + } + : null + }, + pipeline: { + segments: numberFromEnv("SEGMENTS", 4), + modelsDir: fromEnv("MODELS_DIR", "./models") + } +}); diff --git a/projects/v5-to-v6/ddb.transfer.config.ts b/projects/v5-to-v6/ddb.transfer.config.ts deleted file mode 100644 index 4fb7b8c8..00000000 --- a/projects/v5-to-v6/ddb.transfer.config.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { loadEnv, createDdbConfig, fromAwsProfile, fromEnv, numberFromEnv } from "~/index.ts"; - -// Loads projects/v5-to-v6/.env (next to this file). `.env*` is gitignored. -// Region / tables / buckets come from .env. AWS credentials come from -// ~/.aws/credentials via `fromAwsProfile` — set SOURCE_PROFILE / -// TARGET_PROFILE in .env to pick a specific profile, or leave them blank -// to use the default profile. Vars without a default throw fast when -// missing instead of silently passing `undefined` to the AWS SDK. -loadEnv(import.meta.url); - -const DEFAULT_REGION = "eu-central-1"; -const DEFAULT_PROFILE = "default"; - -export default createDdbConfig({ - debug: { - logLevel: "debug", - logFile: true - }, - source: { - region: fromEnv("SOURCE_REGION", DEFAULT_REGION), - credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", DEFAULT_PROFILE) }), - dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, - s3: { bucket: fromEnv("SOURCE_S3_BUCKET") } - }, - target: { - region: fromEnv("TARGET_REGION", DEFAULT_REGION), - credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", DEFAULT_PROFILE) }), - dynamodb: { tableName: fromEnv("TARGET_DDB_TABLE") }, - s3: { bucket: fromEnv("TARGET_S3_BUCKET") }, - auditLog: { - dynamodb: { - tableName: fromEnv("TARGET_AUDIT_LOGS_TABLE") - } - } - }, - pipeline: { - preset: "v5-to-v6-ddb", - segments: numberFromEnv("SEGMENTS", 4), - modelsDir: fromEnv("MODELS_DIR", "./models") - } -}); diff --git a/projects/v5-to-v6/os.transfer.config.ts b/projects/v5-to-v6/os.transfer.config.ts deleted file mode 100644 index 5a05c8ab..00000000 --- a/projects/v5-to-v6/os.transfer.config.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { loadEnv, createOsConfig, fromAwsProfile, fromEnv, numberFromEnv } from "~/index.ts"; - -loadEnv(import.meta.url); - -const DEFAULT_REGION = "eu-central-1"; -const DEFAULT_PROFILE = "default"; - -export default createOsConfig({ - debug: { - logLevel: "debug", - logFile: true - }, - source: { - region: fromEnv("SOURCE_REGION", DEFAULT_REGION), - credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", DEFAULT_PROFILE) }), - dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, - opensearch: { tableName: fromEnv("SOURCE_OS_TABLE") } - }, - target: { - region: fromEnv("TARGET_REGION", DEFAULT_REGION), - credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", DEFAULT_PROFILE) }), - opensearch: { - endpoint: fromEnv("TARGET_OS_ENDPOINT"), - tableName: fromEnv("TARGET_OS_TABLE"), - service: "opensearch", - indexPrefix: fromEnv("TARGET_OS_INDEX_PREFIX", "") - } - }, - pipeline: { - preset: "v5-to-v6-os", - segments: numberFromEnv("SEGMENTS", 4), - modelsDir: fromEnv("MODELS_DIR", "./models") - } -}); diff --git a/src/base/Result.ts b/src/base/Result.ts index 5698c306..7a584b58 100644 --- a/src/base/Result.ts +++ b/src/base/Result.ts @@ -97,7 +97,7 @@ export class Result { */ public map(fn: (value: TValue) => U): Result { if (this.isOk()) { - return Result.ok(fn(this._value as TValue)); + return Result.ok(fn(this._value)); } return Result.fail(this._error as TError); @@ -112,7 +112,7 @@ export class Result { */ public mapError(fn: (error: TError) => F): Result { if (this.isFail()) { - return Result.fail(fn(this._error as TError)); + return Result.fail(fn(this._error)); } return Result.ok(this._value as TValue); @@ -129,7 +129,7 @@ export class Result { */ public flatMap(fn: (value: TValue) => Result): Result { if (this.isOk()) { - return fn(this._value as TValue); + return fn(this._value); } return Result.fail(this._error as TError); @@ -144,7 +144,7 @@ export class Result { */ public match(handlers: { ok: (value: TValue) => U; fail: (error: TError) => U }): U { if (this.isOk()) { - return handlers.ok(this._value as TValue); + return handlers.ok(this._value); } return handlers.fail(this._error as TError); diff --git a/src/base/index.ts b/src/base/index.ts index 2ec8a7df..fb886058 100644 --- a/src/base/index.ts +++ b/src/base/index.ts @@ -6,5 +6,11 @@ export { ResultAsync } from "./ResultAsync.js"; export { BaseError } from "./BaseError.js"; export { ContainerToken } from "./Container.ts"; export { formatError } from "./formatError.ts"; -export { isRetryableAwsError, isTokenBucketExhausted } from "./isRetryableAwsError.ts"; +export { + isRetryableAwsError, + isThrottlingError, + isAccessDeniedError, + isTokenBucketExhausted, + type AwsErrorLike +} from "./isRetryableAwsError.ts"; export { retryBackoffMs } from "./retryBackoff.ts"; diff --git a/src/base/isRetryableAwsError.ts b/src/base/isRetryableAwsError.ts index 599793bc..c3cd0333 100644 --- a/src/base/isRetryableAwsError.ts +++ b/src/base/isRetryableAwsError.ts @@ -1,3 +1,22 @@ +// Auth/permission errors are never transient — fail immediately without retrying. +const TERMINAL_ERROR_NAMES = new Set([ + "AccessDenied", + "AccessDeniedException", + "AuthorizationError", + "AuthorizationErrorException", + "UnauthorizedOperation" +]); + +const THROTTLING_ERROR_NAMES = new Set([ + "ProvisionedThroughputExceededException", + "ThrottlingException", + "RequestLimitExceeded", + "SlowDown", + "TooManyRequestsException", + "LimitExceededException", + "Throttling" +]); + const RETRYABLE_ERROR_NAMES = new Set([ // DynamoDB throttles "ProvisionedThroughputExceededException", @@ -30,7 +49,7 @@ const RETRYABLE_ERROR_NAMES = new Set([ const RETRYABLE_STATUS_CODES = new Set([408, 425, 429, 500, 502, 503, 504]); -interface AwsErrorLike { +export interface AwsErrorLike { name?: string; code?: string; message?: string; @@ -52,6 +71,16 @@ export function isRetryableAwsError(error: unknown): boolean { } const candidate = error as AwsErrorLike; + // Auth/permission failures are permanent — never retry. + const nameForTerminal = candidate.name ?? candidate.code; + if (typeof nameForTerminal === "string" && TERMINAL_ERROR_NAMES.has(nameForTerminal)) { + return false; + } + const httpStatus = candidate.$metadata?.httpStatusCode; + if (httpStatus === 403) { + return false; + } + if (candidate.$retryable?.throttling === true) { return true; } @@ -74,6 +103,45 @@ export function isRetryableAwsError(error: unknown): boolean { return false; } +/** + * Returns true when the error is a service-level throttle (rate limit exceeded), + * as opposed to a transient network or server error. + */ +export function isThrottlingError(error: unknown): boolean { + if (!error || typeof error !== "object") { + return false; + } + const candidate = error as AwsErrorLike; + + if (candidate.$retryable?.throttling === true) { + return true; + } + + const name = candidate.name ?? candidate.code; + if (typeof name === "string" && THROTTLING_ERROR_NAMES.has(name)) { + return true; + } + + const status = candidate.$metadata?.httpStatusCode; + return status === 429; +} + +/** + * Returns true when the error indicates an IAM / credentials access denial. + * Covers DynamoDB AccessDeniedException, S3 AccessDenied, and HTTP 403. + */ +export function isAccessDeniedError(error: unknown): boolean { + if (!error || typeof error !== "object") { + return false; + } + const candidate = error as AwsErrorLike; + const name = candidate.name ?? candidate.code; + if (typeof name === "string" && TERMINAL_ERROR_NAMES.has(name)) { + return true; + } + return candidate.$metadata?.httpStatusCode === 403; +} + /** * Returns true when the AWS SDK adaptive retry token bucket is depleted. * Callers should use a longer minimum backoff (≥10s) so the bucket has time diff --git a/src/bootstrap.ts b/src/bootstrap.ts index d42676d2..02733725 100644 --- a/src/bootstrap.ts +++ b/src/bootstrap.ts @@ -31,6 +31,7 @@ import { OsRecordDecompressorFeature } from "~/features/OsRecordDecompressor/ind import { OsScannerFeature } from "~/features/OsScanner/index.ts"; import { OsProcessorFeature } from "~/features/OsProcessor/index.ts"; import { TouchedIndexesFeature } from "~/features/TouchedIndexes/index.ts"; +import { AccessCheckerFeature } from "~/features/AccessChecker/index.ts"; import { DroppedRecordLogFeature } from "~/features/DroppedRecordLog/index.ts"; import { TransferredRecordLogFeature } from "~/features/TransferredRecordLog/index.ts"; import { CompressionFeature } from "@webiny/utils/features/compression/feature.js"; @@ -59,7 +60,7 @@ export function bootstrap(options: BootstrapOptions): Container { // Tools LoggerFeature.register(container, { - logLevel: options.logLevel || config.debug?.logLevel || "info", + logLevel: options.logLevel || config.debug?.logLevel || "debug", json: options.json || false, logFile: resolveLogFile(config, options.runId) }); @@ -82,22 +83,20 @@ export function bootstrap(options: BootstrapOptions): Container { }); DynamoDbClientFeature.register(container); - if (config.storage === "ddb") { - container.registerInstance(S3ClientConfig, { - source: { - region: config.source.region, - credentials: config.source.credentials - }, - target: { - region: config.target.region, - credentials: config.target.credentials - }, - tuning: config.tuning?.s3 - }); - S3ClientFeature.register(container); - } + container.registerInstance(S3ClientConfig, { + source: { + region: config.source.region, + credentials: config.source.credentials + }, + target: { + region: config.target.region, + credentials: config.target.credentials + }, + tuning: config.tuning?.s3 + }); + S3ClientFeature.register(container); - if (config.storage === "os") { + if (config.target.opensearch != null) { container.registerInstance(OpenSearchClientConfig, { endpoint: config.target.opensearch.endpoint, region: config.target.region, @@ -122,44 +121,40 @@ export function bootstrap(options: BootstrapOptions): Container { TransferredRecordLogFeature.register(container); PipelineRunnerFeature.register(container); - if (config.storage === "ddb") { - DdbExecutorFeature.register(container); - S3ProcessorFeature.register(container); - DdbScannerFeature.register(container); - DdbProcessorFeature.register(container); - AuditLogProcessorFeature.register(container); - } else { - TouchedIndexesFeature.register(container); - DdbExecutorFeature.register(container); - OsRecordDecompressorFeature.register(container); - OsScannerFeature.register(container); - OsProcessorFeature.register(container); - } + DdbExecutorFeature.register(container); + S3ProcessorFeature.register(container); + DdbScannerFeature.register(container); + DdbProcessorFeature.register(container); + AuditLogProcessorFeature.register(container); + TouchedIndexesFeature.register(container); + OsRecordDecompressorFeature.register(container); + OsScannerFeature.register(container); + OsProcessorFeature.register(container); + AccessCheckerFeature.register(container); return container; } /** * Turn `config.debug.logFile` into an absolute path for the pino file - * stream. `true` → `.transfer//logs/.log`. - * Workers are detected via `--segment ` in argv so each one writes - * to its own file (concurrent appends to a shared file can interleave). + * stream. Writes to `.transfer//logs/.log` + * by default — set `logFile: false` to opt out. A string value overrides + * the path entirely. Workers are detected via `--segment ` in argv so + * each one writes to its own file (concurrent appends to a shared file + * can interleave). */ function resolveLogFile( config: MigrationConfig.Interface, runId: string | undefined ): string | undefined { const raw = config.debug?.logFile; - if (!raw) { + if (raw === false) { return undefined; } if (typeof raw === "string") { return isAbsolute(raw) ? raw : joinPath(process.cwd(), raw); } if (!runId) { - // Default path needs a runId to anchor the directory; without - // it there's nowhere sensible to write. Silent no-op keeps the - // feature opt-in-forgiving. return undefined; } const kind = detectProcessKind(); diff --git a/src/commands/init/handler.ts b/src/commands/init/handler.ts index eb34e164..32301f9a 100644 --- a/src/commands/init/handler.ts +++ b/src/commands/init/handler.ts @@ -58,8 +58,7 @@ export async function handler(folderName: string): Promise { console.log(` ├── .env.example`); console.log(` ├── projects/`); console.log(` │ └── example/`); - console.log(` │ ├── ddb.transfer.config.ts`); - console.log(` │ ├── os.transfer.config.ts`); + console.log(` │ ├── config.ts`); console.log(` │ ├── setup.ts # optional custom DI wiring`); console.log(` │ ├── models/ # custom CMS model JSON overrides`); console.log(` │ └── .env.example`); @@ -71,5 +70,5 @@ export async function handler(folderName: string): Promise { console.log(` yarn install # or npm install`); console.log(` cp projects/example/.env.example projects/example/.env`); console.log(` # Edit projects/example/.env with your AWS credentials`); - console.log(` yarn transfer --config=./projects/example/ddb.transfer.config.ts\n`); + console.log(` yarn transfer --config=./projects/example/config.ts\n`); } diff --git a/src/commands/initProject/handler.ts b/src/commands/initProject/handler.ts index 7a86dfe5..1bc6c696 100644 --- a/src/commands/initProject/handler.ts +++ b/src/commands/initProject/handler.ts @@ -11,8 +11,7 @@ export async function handler(projectName: string): Promise { console.log(`\nCreated "projects/${projectName}" with the following structure:\n`); console.log(` projects/${projectName}/`); console.log(` ├── README.md`); - console.log(` ├── ddb.transfer.config.ts`); - console.log(` ├── os.transfer.config.ts`); + console.log(` ├── config.ts`); console.log(` ├── .env.example`); console.log(` ├── models/`); console.log(` └── presets/\n`); @@ -23,7 +22,9 @@ export async function handler(projectName: string): Promise { console.log(` (from: yarn webiny output core --json in each Webiny project)`); console.log(` source.pulumi.json + target.pulumi.json`); console.log(` (from: .pulumi/apps/core/.pulumi/stacks/core/.json)`); - console.log(` Mixed formats (e.g. source.webiny.json + target.pulumi.json) are allowed.\n`); + console.log(` Mixed formats (e.g. source.webiny.json + target.pulumi.json) are allowed.`); + console.log(` You can also drop CMS model exports into projects/${projectName}/models/`); + console.log(` (export from Webiny Admin → CMS → Models → Export)\n`); console.log(` 2. Run the wizard — it validates the JSON files and writes .env:`); console.log(` yarn transfer\n`); console.log(` 3. Review projects/${projectName}/.env, then run again:`); @@ -31,5 +32,5 @@ export async function handler(projectName: string): Promise { console.log(`To set up manually instead:`); console.log(` cp projects/${projectName}/.env.example projects/${projectName}/.env`); console.log(` # Edit .env — fill in region, table names, and AWS credentials`); - console.log(` yarn transfer --config=./projects/${projectName}/ddb.transfer.config.ts\n`); + console.log(` yarn transfer --config=./projects/${projectName}/config.ts\n`); } diff --git a/src/commands/processSegment/handler.ts b/src/commands/processSegment/handler.ts index c552c3c2..100c2a5e 100644 --- a/src/commands/processSegment/handler.ts +++ b/src/commands/processSegment/handler.ts @@ -16,7 +16,9 @@ export interface ProcessSegmentArgs { segment: number; total: number; config: string; + preset: string; logLevel?: string; + dryRun?: boolean; } export async function handler(argv: ProcessSegmentArgs): Promise { @@ -28,7 +30,7 @@ export async function handler(argv: ProcessSegmentArgs): Promise { | "error" | undefined; const container = bootstrap({ config, runId: argv.runId, logLevel: resolvedLogLevel }); - container.registerInstance(TransferContext, { runId: argv.runId }); + container.registerInstance(TransferContext, { runId: argv.runId, dryRun: argv.dryRun }); const logger = container.resolve(Logger).child(`[segment ${argv.segment}]`); const runner = container.resolve(PipelineRunner); @@ -39,7 +41,7 @@ export async function handler(argv: ProcessSegmentArgs): Promise { const beforeLoadPreset = container.resolve(BeforeLoadPresetHook); await beforeLoadPreset.execute(config); - const preset = await presetLoader.load(config.pipeline.preset); + const preset = await presetLoader.load(argv.preset); await preset.configure({ runner, pipelineBuilderFactory: container.resolve(PipelineBuilderFactory), @@ -49,13 +51,15 @@ export async function handler(argv: ProcessSegmentArgs): Promise { const afterLoadPreset = container.resolve(AfterLoadPresetHook); await afterLoadPreset.execute(config, preset); - logger.info(`Processing shard ${argv.segment + 1}/${argv.total}...`); + logger.info( + `Processing shard ${argv.segment + 1}/${argv.total}${argv.dryRun ? " (DRY RUN)" : ""}...` + ); try { await runner.run({ segment: argv.segment, totalSegments: argv.total }); } catch (error) { logger.error( - `Shard ${argv.segment} failed: ${formatError(error, resolvedLogLevel === "debug")}` + `Shard ${argv.segment} failed: ${formatError(error, (resolvedLogLevel ?? "debug") === "debug")}` ); process.exit(1); } diff --git a/src/commands/processSegment/register.ts b/src/commands/processSegment/register.ts index 4a352d2e..5677f80e 100644 --- a/src/commands/processSegment/register.ts +++ b/src/commands/processSegment/register.ts @@ -23,14 +23,29 @@ export function registerProcessSegmentCommand(yargs: Argv): Argv { demandOption: true, description: "Config file path" }) + .option("preset", { + type: "string", + demandOption: true, + description: "Preset name to use for this segment" + }) .option("log-level", { type: "string", choices: ["debug", "info", "warn", "error"] as const, description: "Log level" + }) + .option("dry-run", { + type: "boolean", + default: false, + description: "Skip all writes to the target system" }); }, async argv => { - await handler({ ...argv, logLevel: argv["log-level"] as string | undefined }); + await handler({ + ...argv, + logLevel: argv["log-level"] as string | undefined, + preset: argv.preset, + dryRun: argv["dry-run"] + }); } ); } diff --git a/src/commands/run/handler.ts b/src/commands/run/handler.ts index 4882d0ac..44ac22aa 100644 --- a/src/commands/run/handler.ts +++ b/src/commands/run/handler.ts @@ -2,9 +2,12 @@ import { fileURLToPath } from "node:url"; import { join } from "node:path"; import { readdir, readFile } from "node:fs/promises"; import { execa } from "execa"; +import { confirm } from "@inquirer/prompts"; import { bootstrap } from "~/bootstrap.ts"; import { formatError } from "~/base/index.ts"; import type { RunStats } from "~/features/PipelineRunner/abstractions/PipelineRunner.ts"; +import { PipelineRunner } from "~/features/PipelineRunner/index.ts"; +import { PipelineBuilderFactory } from "~/features/PipelineBuilderFactory/index.ts"; import { loadConfig } from "~/features/MigrationConfig/loadConfig.ts"; import { Logger } from "~/tools/Logger/index.ts"; import { MigrationConfig } from "~/features/MigrationConfig/index.ts"; @@ -14,13 +17,23 @@ import { TransferContext } from "~/features/TransferLifecycle/index.ts"; import { PresetLoader } from "~/features/PresetLoader/index.ts"; +import { AccessChecker } from "~/features/AccessChecker/index.ts"; import { loadUserSetup } from "~/utils/loadUserSetup.ts"; import { resolveSegmentsToRun } from "./segmentsFilter.ts"; +class AccessCheckError extends Error { + public constructor(count: number) { + super(`Access check failed — ${count} resource(s) denied or missing`); + this.name = "AccessCheckError"; + } +} + export async function handler( configPath: string, + presetName: string, segmentsFilter?: number[], - logLevel?: string + logLevel?: string, + dryRun = false ): Promise { const runId = String(Date.now()); let container; @@ -42,14 +55,14 @@ export async function handler( } catch (error) { // Config-load / Zod validation failures happen before we have a logger // — write directly to stderr so the user sees the friendly format. - const verbose = (logLevel ?? "info") === "debug"; + const verbose = (logLevel ?? "debug") === "debug"; process.stderr.write(`\n${formatError(error, verbose)}\n`); process.exit(1); } - const resolvedLogLevel = (logLevel ?? config.debug?.logLevel) as string | undefined; - const verbose = resolvedLogLevel === "debug"; - const segments = config.pipeline.segments || 1; + const resolvedLogLevel = logLevel ?? config.debug?.logLevel; + const verbose = (resolvedLogLevel ?? "debug") === "debug"; + const segments = config.pipeline?.segments || 1; let segmentsToRun: number[]; try { @@ -59,7 +72,11 @@ export async function handler( process.exit(1); } - container.registerInstance(TransferContext, { runId }); + container.registerInstance(TransferContext, { runId, dryRun }); + + if (dryRun) { + logger.warn("DRY RUN: no writes will be made to the target system."); + } logConfig({ logger, @@ -67,6 +84,7 @@ export async function handler( runId, segments, segmentsToRun, + presetName, logLevel: logLevel ?? config.debug?.logLevel }); @@ -76,14 +94,63 @@ export async function handler( await loadUserSetup(configPath, container, logger); const presetLoader = container.resolve(PresetLoader); - await presetLoader.load(config.pipeline.preset); + const preset = await presetLoader.load(presetName); + + const runner = container.resolve(PipelineRunner); + const pipelineBuilderFactory = container.resolve(PipelineBuilderFactory); + await preset.configure({ runner, pipelineBuilderFactory, container }); + + const accessChecker = container.resolve(AccessChecker); + const accessReport = await accessChecker.run(); + + if (accessReport.length > 0) { + logger.info("Pre-transfer access check:"); + for (const entry of accessReport) { + if (entry.status === "ok") { + logger.info(` ok ${entry.label}`); + } else if (entry.status === "denied") { + logger.error(` DENIED ${entry.label}`); + } else if (entry.status === "missing") { + logger.error(` MISSING ${entry.label}`); + } else { + logger.warn(` unknown ${entry.label}`); + } + } + } + + const blocked = accessReport.filter(e => e.status === "denied" || e.status === "missing"); + if (blocked.length > 0) { + throw new AccessCheckError(blocked.length); + } + + const guardWarnings = ( + await Promise.all(runner.getProcessors().map(p => p.getGuardWarning?.() ?? null)) + ).filter((w): w is string => w !== null); + + if (guardWarnings.length > 0) { + for (const warning of guardWarnings) { + logger.warn(warning); + } + const proceed = await confirm({ message: "Proceed with transfer?" }); + if (!proceed) { + process.exit(0); + } + } const beforeHook = container.resolve(BeforeTransferHook); logger.info("Running before-transfer hooks..."); await beforeHook.execute(); const workers = segmentsToRun.map(segment => - spawnWorker(segment, segments, runId, configPath, logLevel ?? config.debug?.logLevel) + spawnWorker( + segment, + segments, + runId, + configPath, + presetName, + logLevel ?? config.debug?.logLevel, + dryRun + ) ); const results = await Promise.allSettled(workers); @@ -132,6 +199,7 @@ interface LogConfigParams { runId: string; segments: number; segmentsToRun: number[]; + presetName: string; logLevel?: string; } @@ -141,31 +209,29 @@ function logConfig({ runId, segments, segmentsToRun, + presetName, logLevel }: LogConfigParams): void { logger.info("Starting transfer with configuration:"); logger.info(` Run ID: ${runId}`); - logger.info(` Storage: ${config.storage}`); - logger.info(` Preset: ${config.pipeline.preset}`); - logger.info(` Log Level: ${logLevel ?? "info"}`); + logger.info(` Preset: ${presetName}`); + logger.info(` Log Level: ${logLevel ?? "debug"}`); if (segmentsToRun.length === segments) { logger.info(` Segments: ${segments}`); } else { logger.info(` Segments: ${segments} (running only [${segmentsToRun.join(", ")}])`); } - if (config.storage === "ddb") { - logger.info(` Source Region: ${config.source.region}`); - logger.info(` Source Table: ${config.source.dynamodb.tableName}`); - logger.info(` Source Bucket: ${config.source.s3.bucket}`); - logger.info(` Target Region: ${config.target.region}`); - logger.info(` Target Table: ${config.target.dynamodb.tableName}`); - logger.info(` Target Bucket: ${config.target.s3.bucket}`); - } else { - logger.info(` Source Region: ${config.source.region}`); - logger.info(` Source Primary Table: ${config.source.dynamodb.tableName}`); + logger.info(` Source Region: ${config.source.region}`); + logger.info(` Source DDB Table: ${config.source.dynamodb.tableName}`); + logger.info(` Source S3 Bucket: ${config.source.s3.bucket}`); + if (config.source.opensearch) { logger.info(` Source OS Table: ${config.source.opensearch.tableName}`); - logger.info(` Target Region: ${config.target.region}`); + } + logger.info(` Target Region: ${config.target.region}`); + logger.info(` Target DDB Table: ${config.target.dynamodb.tableName}`); + logger.info(` Target S3 Bucket: ${config.target.s3.bucket}`); + if (config.target.opensearch) { logger.info(` Target OS Table: ${config.target.opensearch.tableName}`); logger.info(` OS Endpoint: ${config.target.opensearch.endpoint}`); } @@ -176,7 +242,9 @@ async function spawnWorker( total: number, runId: string, configPath: string, - logLevel?: string + presetName: string, + logLevel?: string, + dryRun = false ): Promise { const binPath = fileURLToPath(new URL("../../../bin.js", import.meta.url)); @@ -191,7 +259,10 @@ async function spawnWorker( total.toString(), "--config", configPath, - ...(logLevel ? ["--log-level", logLevel] : []) + "--preset", + presetName, + ...(logLevel ? ["--log-level", logLevel] : []), + ...(dryRun ? ["--dry-run"] : []) ]; const { exitCode } = await execa("node", args, { diff --git a/src/commands/run/register.ts b/src/commands/run/register.ts index 0bb7be9e..decd1805 100644 --- a/src/commands/run/register.ts +++ b/src/commands/run/register.ts @@ -29,18 +29,19 @@ export function registerRunCommand(yargs: Argv): Argv { }); }, async argv => { - if (argv.config) { - await handler(argv.config, argv.segments, argv["log-level"] as string | undefined); - return; - } - const wizard = new TransferWizard(process.cwd()); try { - const configPath = await wizard.run(); - if (configPath === null) { + const result = await wizard.run(); + if (result === null) { process.exit(0); } - await handler(configPath, argv.segments, argv["log-level"] as string | undefined); + await handler( + result.configPath, + result.preset, + argv.segments, + argv["log-level"] as string | undefined, + result.dryRun + ); } catch (err) { if (err instanceof ExitPromptError) { process.exit(0); diff --git a/src/commands/run/wizard/TransferWizard.ts b/src/commands/run/wizard/TransferWizard.ts index 6baba0a4..46b54bd4 100644 --- a/src/commands/run/wizard/TransferWizard.ts +++ b/src/commands/run/wizard/TransferWizard.ts @@ -1,14 +1,16 @@ import { join, relative, resolve } from "node:path"; -import { access, stat } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import { stat } from "node:fs/promises"; import { existsSync } from "node:fs"; -import { select, input } from "@inquirer/prompts"; +import { select, input, confirm } from "@inquirer/prompts"; import { discoverProjects } from "./projectDiscovery.ts"; -import { discoverConfigs } from "./configDiscovery.ts"; +import { discoverConfig } from "./configDiscovery.ts"; +import { listAvailablePresetsWithDescriptions } from "./presetDiscovery.ts"; import { writeEnv } from "./envWriter.ts"; import { extractFromWebinyOutput } from "./sources/WebinyOutputSource.ts"; import { extractFromPulumiState } from "./sources/PulumiStateSource.ts"; import { scaffoldProject } from "~/commands/initProject/scaffoldProject.ts"; -import type { RawOutputValues, EnvValues } from "./types.ts"; +import type { RawOutputValues, EnvValues, WizardResult } from "./types.ts"; async function fileNonEmpty(path: string): Promise { try { @@ -49,6 +51,7 @@ async function resolveRawValues( "region", "primaryDynamodbTableName", "fileManagerBucketId", + "auditLogTableName", "osTableName", "osEndpoint" ] as const) { @@ -62,13 +65,15 @@ async function resolveRawValues( ); } - // Consistent — prefer webiny, but fill in OS fields from pulumi if webiny lacks them + // Consistent — prefer webiny, but fill in fields from pulumi if webiny lacks them return { region: webinyVals!.region, primaryDynamodbTableName: webinyVals!.primaryDynamodbTableName, fileManagerBucketId: webinyVals!.fileManagerBucketId, + auditLogTableName: webinyVals!.auditLogTableName ?? pulumiVals!.auditLogTableName, osTableName: webinyVals!.osTableName || pulumiVals!.osTableName, - osEndpoint: webinyVals!.osEndpoint || pulumiVals!.osEndpoint + osEndpoint: webinyVals!.osEndpoint || pulumiVals!.osEndpoint, + accountId: webinyVals!.accountId ?? pulumiVals!.accountId }; } @@ -87,6 +92,9 @@ Option B — Pulumi state file (use when you don't have Webiny CLI access): State files are at: .pulumi/apps/core/.pulumi/stacks/core/.json You can mix formats (e.g. source.webiny.json + target.pulumi.json). + +Optionally, drop CMS model exports into ${rel}/models/ + (export from Webiny Admin → CMS → Models → Export) `); } @@ -99,7 +107,7 @@ export class TransferWizard { this.cwd = cwd; } - public async run(): Promise { + public async run(): Promise { const projects = await discoverProjects(this.cwd); const selected = await select({ @@ -153,7 +161,20 @@ export class TransferWizard { const envExists = await fileNonEmpty(join(projectDir, ".env")); if (!justCreated && sourceValsInitial === null && targetValsInitial === null && envExists) { - return await this.runConfigSelection(projectName); + return await this.runPresetSelection(projectName); + } + + if (!justCreated && envExists && sourceValsInitial !== null && targetValsInitial !== null) { + const choice = await select({ + message: ".env already exists. What would you like to do?", + choices: [ + { value: "existing", name: "Use existing .env" }, + { value: "repopulate", name: "Repopulate .env from JSON files" } + ] + }); + if (choice === "existing") { + return await this.runPresetSelection(projectName); + } } let sourceVals: RawOutputValues | null = sourceValsInitial; @@ -166,6 +187,23 @@ export class TransferWizard { targetVals = await resolveRawValues(projectDir, "target"); } + if ( + sourceVals.accountId && + targetVals.accountId && + sourceVals.accountId !== targetVals.accountId + ) { + const bold = "\x1b[1m"; + const yellow = "\x1b[33m"; + const dim = "\x1b[2m"; + const reset = "\x1b[0m"; + console.warn( + `\n${bold}${yellow}⚠ Source and target are in different AWS accounts:${reset}` + + `\n ${dim}source:${reset} ${bold}${sourceVals.accountId}${reset}` + + `\n ${dim}target:${reset} ${bold}${targetVals.accountId}${reset}` + + `\n ${dim}Set SOURCE_PROFILE and TARGET_PROFILE in .env to use the correct credentials.${reset}\n` + ); + } + const osPresent = !!(sourceVals.osTableName || targetVals.osTableName); const segmentsRaw = await input({ @@ -192,25 +230,20 @@ export class TransferWizard { sourceRegion: sourceVals.region, sourceDdbTable: sourceVals.primaryDynamodbTableName, sourceS3Bucket: sourceVals.fileManagerBucketId, + sourceAuditLogTable: sourceVals.auditLogTableName ?? "", sourceOsTable: sourceVals.osTableName, + sourceAccountId: sourceVals.accountId ?? "", targetRegion: targetVals.region, targetDdbTable: targetVals.primaryDynamodbTableName, targetS3Bucket: targetVals.fileManagerBucketId, + targetAuditLogTable: targetVals.auditLogTableName ?? "", targetOsTable: targetVals.osTableName, targetOsEndpoint: targetVals.osEndpoint, targetOsIndexPrefix, + targetAccountId: targetVals.accountId ?? "", segments: Number(segmentsRaw) }; - try { - await access(join(projectDir, ".env")); - console.warn( - "\n⚠ .env already exists and will be overwritten. Manual edits will be lost.\n" - ); - } catch { - // no existing .env — silent - } - await writeEnv(projectDir, envValues); console.log( @@ -221,25 +254,46 @@ export class TransferWizard { return null; } - private async runConfigSelection(projectName: string): Promise { + private async runPresetSelection(projectName: string): Promise { const projectDir = resolve(join(this.cwd, "projects", projectName)); - const configs = await discoverConfigs(projectDir); + const configPath = await discoverConfig(projectDir); - if (configs.length === 0) { + if (!configPath) { console.error( - `\nNo transfer configs found in projects/${projectName}/.\n` + - `Add a ddb.transfer.config.ts or os.transfer.config.ts.\n` + `\nNo config.ts found in projects/${projectName}/.\n` + + `Run "yarn transfer" to set up the project first.\n` ); process.exit(1); } - if (configs.length === 1) { - return configs[0].path; + let presetsDir: string | undefined; + try { + const mod = await import(pathToFileURL(configPath).href); + presetsDir = mod.default?.pipeline?.presetsDir; + } catch { + // ignore — presets from built-ins only + } + + const presets = await listAvailablePresetsWithDescriptions(presetsDir); + + if (presets.length === 0) { + console.error("\nNo presets available. Check your presetsDir configuration.\n"); + process.exit(1); } - return select({ - message: "Which transfer do you want to run?", - choices: configs.map(c => ({ value: c.path, name: c.label })) + const preset = await select({ + message: "Which preset do you want to run?", + choices: presets.map(p => ({ + value: p.name, + name: p.description ? `${p.name} — ${p.description}` : p.name + })) }); + + const dryRun = await confirm({ + message: "Dry run? (reads source, skips all writes to target)", + default: false + }); + + return { configPath, preset, dryRun }; } } diff --git a/src/commands/run/wizard/configDiscovery.ts b/src/commands/run/wizard/configDiscovery.ts index 8b7c943a..9932bca7 100644 --- a/src/commands/run/wizard/configDiscovery.ts +++ b/src/commands/run/wizard/configDiscovery.ts @@ -1,46 +1,12 @@ -import { readdir } from "node:fs/promises"; -import { basename, join, resolve } from "node:path"; -import { pathToFileURL } from "node:url"; +import { access } from "node:fs/promises"; +import { join, resolve } from "node:path"; -export interface ConfigEntry { - path: string; - label: string; -} - -const STORAGE_LABELS: Record = { - ddb: "DynamoDB Transfer", - os: "OpenSearch Transfer" -}; - -interface ConfigModule { - storage?: string; -} - -export async function discoverConfigs(projectDir: string): Promise { - let entries; +export async function discoverConfig(projectDir: string): Promise { + const configPath = resolve(join(projectDir, "config.ts")); try { - entries = await readdir(projectDir, { withFileTypes: true }); + await access(configPath); + return configPath; } catch { - return []; - } - - const configFiles = entries - .filter(e => e.isFile() && e.name.endsWith(".config.ts")) - .map(e => resolve(join(projectDir, e.name))); - - const results: ConfigEntry[] = []; - for (const filePath of configFiles) { - try { - const mod = await import(pathToFileURL(filePath).href); - const config = mod.default as ConfigModule | undefined; - const storage = config?.storage ?? ""; - const label = STORAGE_LABELS[storage] ?? basename(filePath); - results.push({ path: filePath, label }); - } catch (err) { - console.warn( - `Warning: could not import config ${filePath} — ${err instanceof Error ? err.message : String(err)} — skipping.` - ); - } + return null; } - return results; } diff --git a/src/commands/run/wizard/envWriter.ts b/src/commands/run/wizard/envWriter.ts index d887f905..a0582632 100644 --- a/src/commands/run/wizard/envWriter.ts +++ b/src/commands/run/wizard/envWriter.ts @@ -6,13 +6,17 @@ const TOKEN_MAP: Record = { SOURCE_REGION: "sourceRegion", SOURCE_DDB_TABLE: "sourceDdbTable", SOURCE_S3_BUCKET: "sourceS3Bucket", + SOURCE_AUDIT_LOGS_TABLE: "sourceAuditLogTable", SOURCE_OS_TABLE: "sourceOsTable", + SOURCE_ACCOUNT_ID: "sourceAccountId", TARGET_REGION: "targetRegion", TARGET_DDB_TABLE: "targetDdbTable", TARGET_S3_BUCKET: "targetS3Bucket", + TARGET_AUDIT_LOGS_TABLE: "targetAuditLogTable", TARGET_OS_TABLE: "targetOsTable", TARGET_OS_ENDPOINT: "targetOsEndpoint", TARGET_OS_INDEX_PREFIX: "targetOsIndexPrefix", + TARGET_ACCOUNT_ID: "targetAccountId", SEGMENTS: "segments" }; @@ -40,6 +44,8 @@ SOURCE_REGION={{SOURCE_REGION}} SOURCE_DDB_TABLE={{SOURCE_DDB_TABLE}} SOURCE_S3_BUCKET={{SOURCE_S3_BUCKET}} +SOURCE_ACCOUNT_ID={{SOURCE_ACCOUNT_ID}} +SOURCE_AUDIT_LOGS_TABLE={{SOURCE_AUDIT_LOGS_TABLE}} SOURCE_OS_TABLE={{SOURCE_OS_TABLE}} # --- Target environment ------------------------------------------------ @@ -53,6 +59,8 @@ TARGET_REGION={{TARGET_REGION}} TARGET_DDB_TABLE={{TARGET_DDB_TABLE}} TARGET_S3_BUCKET={{TARGET_S3_BUCKET}} +TARGET_ACCOUNT_ID={{TARGET_ACCOUNT_ID}} +TARGET_AUDIT_LOGS_TABLE={{TARGET_AUDIT_LOGS_TABLE}} TARGET_OS_TABLE={{TARGET_OS_TABLE}} TARGET_OS_ENDPOINT={{TARGET_OS_ENDPOINT}} TARGET_OS_INDEX_PREFIX={{TARGET_OS_INDEX_PREFIX}} diff --git a/src/commands/run/wizard/presetDiscovery.ts b/src/commands/run/wizard/presetDiscovery.ts new file mode 100644 index 00000000..0e4c2d41 --- /dev/null +++ b/src/commands/run/wizard/presetDiscovery.ts @@ -0,0 +1,85 @@ +import { dirname, join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { existsSync, readdirSync } from "node:fs"; + +const BUILTIN_PRESETS_DIR = join(dirname(fileURLToPath(import.meta.url)), "../../../presets"); + +const PRESET_EXTENSIONS: ReadonlySet = new Set([".ts", ".js"]); + +export interface PresetEntry { + name: string; + description: string; +} + +function stripExtension(filename: string): string | null { + for (const ext of PRESET_EXTENSIONS) { + if (filename.endsWith(ext)) { + return filename.slice(0, -ext.length); + } + } + return null; +} + +function scanDir(dir: string): string[] { + if (!existsSync(dir)) { + return []; + } + try { + return readdirSync(dir) + .map(stripExtension) + .filter((name): name is string => name !== null); + } catch { + return []; + } +} + +function resolvePresetPath(name: string, presetsDir?: string): string | null { + for (const ext of PRESET_EXTENSIONS) { + const builtIn = join(BUILTIN_PRESETS_DIR, `${name}${ext}`); + if (existsSync(builtIn)) { + return builtIn; + } + } + if (presetsDir) { + for (const ext of PRESET_EXTENSIONS) { + const user = join(presetsDir, `${name}${ext}`); + if (existsSync(user)) { + return user; + } + } + } + return null; +} + +async function loadDescription(name: string, presetsDir?: string): Promise { + const filePath = resolvePresetPath(name, presetsDir); + if (!filePath) { + return ""; + } + try { + const mod = await import(pathToFileURL(filePath).href); + const preset = mod.default ?? mod.preset; + return typeof preset?.description === "string" ? preset.description : ""; + } catch { + return ""; + } +} + +export function listAvailablePresets(presetsDir?: string): string[] { + const builtIns = scanDir(BUILTIN_PRESETS_DIR); + const userPresets = presetsDir ? scanDir(presetsDir) : []; + const all = new Set([...builtIns, ...userPresets]); + return [...all].sort(); +} + +export async function listAvailablePresetsWithDescriptions( + presetsDir?: string +): Promise { + const names = listAvailablePresets(presetsDir); + return Promise.all( + names.map(async name => ({ + name, + description: await loadDescription(name, presetsDir) + })) + ); +} diff --git a/src/commands/run/wizard/schemas/webinyOutput.schema.ts b/src/commands/run/wizard/schemas/webinyOutput.schema.ts index 9269ad18..d8c6176d 100644 --- a/src/commands/run/wizard/schemas/webinyOutput.schema.ts +++ b/src/commands/run/wizard/schemas/webinyOutput.schema.ts @@ -5,7 +5,9 @@ export const webinyOutputSchema = z .object({ region: z.string().min(1), primaryDynamodbTableName: z.string().min(1), + primaryDynamodbTableArn: z.string().optional(), fileManagerBucketId: z.string().min(1), + auditLogsDynamodbTableName: z.string().optional(), opensearchDynamodbTableName: z.string().optional(), elasticsearchDynamodbTableName: z.string().optional(), opensearchDomainEndpoint: z.string().optional(), @@ -15,13 +17,23 @@ export const webinyOutputSchema = z export type WebinyOutputs = z.infer; +function extractAccountId(arn: string | undefined): string | undefined { + if (!arn) { + return undefined; + } + const parts = arn.split(":"); + return parts.length >= 5 && parts[4] ? parts[4] : undefined; +} + export function normalizeOutputs(outputs: WebinyOutputs): RawOutputValues { return { region: outputs.region, primaryDynamodbTableName: outputs.primaryDynamodbTableName, fileManagerBucketId: outputs.fileManagerBucketId, + auditLogTableName: outputs.auditLogsDynamodbTableName, osTableName: outputs.opensearchDynamodbTableName ?? outputs.elasticsearchDynamodbTableName ?? "", - osEndpoint: outputs.opensearchDomainEndpoint ?? outputs.elasticsearchDomainEndpoint ?? "" + osEndpoint: outputs.opensearchDomainEndpoint ?? outputs.elasticsearchDomainEndpoint ?? "", + accountId: extractAccountId(outputs.primaryDynamodbTableArn) }; } diff --git a/src/commands/run/wizard/types.ts b/src/commands/run/wizard/types.ts index 04efd95d..5fd90321 100644 --- a/src/commands/run/wizard/types.ts +++ b/src/commands/run/wizard/types.ts @@ -2,20 +2,32 @@ export interface RawOutputValues { region: string; primaryDynamodbTableName: string; fileManagerBucketId: string; + auditLogTableName?: string; osTableName: string; osEndpoint: string; + accountId?: string; } export interface EnvValues { sourceRegion: string; sourceDdbTable: string; sourceS3Bucket: string; + sourceAuditLogTable: string; sourceOsTable: string; + sourceAccountId: string; targetRegion: string; targetDdbTable: string; targetS3Bucket: string; + targetAuditLogTable: string; targetOsTable: string; targetOsEndpoint: string; targetOsIndexPrefix: string; + targetAccountId: string; segments: number; } + +export interface WizardResult { + configPath: string; + preset: string; + dryRun: boolean; +} diff --git a/src/domain/pipeline/abstractions/Processor.ts b/src/domain/pipeline/abstractions/Processor.ts index 9bc80f7a..b0e78eac 100644 --- a/src/domain/pipeline/abstractions/Processor.ts +++ b/src/domain/pipeline/abstractions/Processor.ts @@ -31,6 +31,33 @@ interface IProcessor< */ onEnd?(ctx: TBaseContext & TSlice): void | Promise; + /** + * Pre-transfer access check. Called once in the orchestrator process before + * any segment worker is spawned. + * + * Returns one entry per probed resource (table, bucket, cluster endpoint). + * Mandatory (not optional) so every processor author must consciously decide + * what to check — return `[]` if the processor has no AWS resources to probe. + * + * Status meanings: + * "ok" — probe succeeded; proceed + * "denied" — IAM / credentials error; transfer will be aborted + * "missing" — resource does not exist; transfer will be aborted + * "unknown" — probe failed for an unclassified reason; warn and proceed + * + * Implementations must not throw — catch all errors and return an "unknown" + * entry instead. + */ + checkAccess(): Promise; + + /** + * Pre-transfer guard check. Called in the orchestrator before any segment + * workers are spawned. Return a human-readable warning string when the + * processor detects a condition that requires user confirmation (e.g. + * cross-account S3 copy), or null to proceed silently. + */ + getGuardWarning?(): Promise; + /** * Drain the processor's commands from the bag and write to target. The * act of calling commands.get(key) marks that key as "claimed" — the @@ -51,6 +78,17 @@ interface IProcessor< export const Processor = createAbstraction>("Core/Processor"); +export namespace AccessCheck { + export type Status = "ok" | "denied" | "missing" | "unknown"; + + export interface Entry { + label: string; + status: Status; + } + + export type Report = Entry[]; +} + export namespace Processor { export type Interface< TBaseContext extends BaseTransformContext.Interface = diff --git a/src/domain/transform/filters.ts b/src/domain/transform/filters.ts index 1c7b2af0..2df11b19 100644 --- a/src/domain/transform/filters.ts +++ b/src/domain/transform/filters.ts @@ -116,3 +116,14 @@ export const isMigrationRecord = (record: BaseRecord): boolean => { } return record.PK.startsWith("MIGRATION"); }; + +export const isFormBuilderRecord = (record: BaseRecord): boolean => { + if (record.PK.includes("#FB#")) { + return true; + } + const type = record.TYPE; + if (!type) { + return false; + } + return type.startsWith("fb.form.") || type.startsWith("fb.formSubmission"); +}; diff --git a/src/features/AccessChecker/AccessChecker.ts b/src/features/AccessChecker/AccessChecker.ts new file mode 100644 index 00000000..6ee458f8 --- /dev/null +++ b/src/features/AccessChecker/AccessChecker.ts @@ -0,0 +1,24 @@ +import { AccessChecker as AccessCheckerAbstraction } from "./abstractions/AccessChecker.ts"; +import { PipelineRunner } from "~/features/PipelineRunner/index.ts"; +import type { AccessCheck } from "~/domain/pipeline/abstractions/Processor.ts"; + +class AccessCheckerImpl implements AccessCheckerAbstraction.Interface { + public constructor(private readonly runner: PipelineRunner.Interface) {} + + public async run(): Promise { + const processors = this.runner.getProcessors(); + const results = await Promise.allSettled(processors.map(p => p.checkAccess())); + return results.flatMap((result, i) => { + if (result.status === "fulfilled") { + return result.value; + } + const label = processors[i].constructor.name ?? "unknown processor"; + return [{ label, status: "unknown" as const }]; + }); + } +} + +export const AccessChecker = AccessCheckerAbstraction.createImplementation({ + implementation: AccessCheckerImpl, + dependencies: [PipelineRunner] +}); diff --git a/src/features/AccessChecker/abstractions/AccessChecker.ts b/src/features/AccessChecker/abstractions/AccessChecker.ts new file mode 100644 index 00000000..fd789b46 --- /dev/null +++ b/src/features/AccessChecker/abstractions/AccessChecker.ts @@ -0,0 +1,14 @@ +import { createAbstraction } from "~/base/index.ts"; +import type { AccessCheck } from "~/domain/pipeline/abstractions/Processor.ts"; + +interface IAccessChecker { + run(): Promise; +} + +export const AccessChecker = createAbstraction("Core/AccessChecker"); + +export namespace AccessChecker { + export type Interface = IAccessChecker; + export type Report = AccessCheck.Report; + export type Entry = AccessCheck.Entry; +} diff --git a/src/features/AccessChecker/feature.ts b/src/features/AccessChecker/feature.ts new file mode 100644 index 00000000..2ea29af9 --- /dev/null +++ b/src/features/AccessChecker/feature.ts @@ -0,0 +1,9 @@ +import { createFeature } from "~/base/index.ts"; +import { AccessChecker } from "./AccessChecker.ts"; + +export const AccessCheckerFeature = createFeature({ + name: "Core/AccessCheckerFeature", + register(container) { + container.register(AccessChecker).inSingletonScope(); + } +}); diff --git a/src/features/AccessChecker/index.ts b/src/features/AccessChecker/index.ts new file mode 100644 index 00000000..76d0548d --- /dev/null +++ b/src/features/AccessChecker/index.ts @@ -0,0 +1,2 @@ +export { AccessChecker } from "./abstractions/AccessChecker.ts"; +export { AccessCheckerFeature } from "./feature.ts"; diff --git a/src/features/AuditLogProcessor/AuditLogProcessor.ts b/src/features/AuditLogProcessor/AuditLogProcessor.ts index 83c26c80..fc6a0f00 100644 --- a/src/features/AuditLogProcessor/AuditLogProcessor.ts +++ b/src/features/AuditLogProcessor/AuditLogProcessor.ts @@ -1,10 +1,12 @@ -import { Processor } from "~/domain/pipeline/abstractions/Processor.ts"; +import { AccessCheck, Processor } from "~/domain/pipeline/abstractions/Processor.ts"; import { DdbExecutor } from "~/features/DdbExecutor/abstractions/DdbExecutor.ts"; import { MigrationConfig } from "~/features/MigrationConfig/abstractions/MigrationConfig.ts"; import { AuditLogPutRecord } from "~/domain/transform/commands/AuditLogPutRecord.ts"; import { PutRecord } from "~/domain/transform/commands/PutRecord.ts"; import type { Commands } from "~/domain/transform/commands/Commands.ts"; import type { BaseTransformContext } from "~/features/TransformContext/abstractions/BaseTransformContext.ts"; +import { DynamoDB } from "@aws-sdk/client-dynamodb"; +import { isAccessDeniedError, type AwsErrorLike } from "~/base/index.ts"; interface AuditLogProcessorSlice { putAuditLog(record: Record): void; @@ -20,10 +22,7 @@ class AuditLogProcessorImpl implements Processor.Interface< ) {} public extendContext(base: BaseTransformContext.Interface): AuditLogProcessorSlice { - const tableName = - this.config.storage === "ddb" - ? (this.config.target.auditLog?.dynamodb?.tableName ?? null) - : null; + const tableName = this.config.target.auditLog?.dynamodb?.tableName ?? null; return { putAuditLog(record: Record): void { if (!tableName) { @@ -41,11 +40,35 @@ class AuditLogProcessorImpl implements Processor.Interface< ctx.putAuditLog(ctx.record as Record); } + public async checkAccess(): Promise { + const tableName = this.config.target.auditLog?.dynamodb?.tableName ?? null; + if (!tableName) { + return []; + } + const label = `DynamoDB audit log table: ${tableName}`; + const client = new DynamoDB({ + region: this.config.target.region, + credentials: this.config.target.credentials as never + }); + try { + await client.describeTable({ TableName: tableName }); + return [{ label, status: "ok" }]; + } catch (error) { + if (isAccessDeniedError(error)) { + return [{ label, status: "denied" }]; + } + const errName = (error as AwsErrorLike).name ?? (error as AwsErrorLike).code; + if (errName === "ResourceNotFoundException") { + return [{ label, status: "missing" }]; + } + return [{ label, status: "unknown" }]; + } finally { + client.destroy(); + } + } + public async execute(commands: Commands): Promise { - const tableName = - this.config.storage === "ddb" - ? (this.config.target.auditLog?.dynamodb?.tableName ?? null) - : null; + const tableName = this.config.target.auditLog?.dynamodb?.tableName ?? null; if (!tableName) { return; } diff --git a/src/features/DdbProcessor/DdbProcessor.ts b/src/features/DdbProcessor/DdbProcessor.ts index 4363bad6..4388ce3b 100644 --- a/src/features/DdbProcessor/DdbProcessor.ts +++ b/src/features/DdbProcessor/DdbProcessor.ts @@ -1,13 +1,16 @@ -import { Processor } from "~/domain/pipeline/abstractions/Processor.ts"; +import { AccessCheck, Processor } from "~/domain/pipeline/abstractions/Processor.ts"; import { DdbExecutor } from "~/features/DdbExecutor/abstractions/DdbExecutor.ts"; import { MigrationConfig } from "~/features/MigrationConfig/abstractions/MigrationConfig.ts"; import { SourceDynamoDbClient, TargetDynamoDbClient } from "~/services/DynamoDbClient/abstractions/DynamoDbClient.ts"; +import { TransferContext } from "~/features/TransferLifecycle/abstractions/TransferContext.ts"; import { PutRecord } from "~/domain/transform/commands/PutRecord.ts"; import type { Commands } from "~/domain/transform/commands/Commands.ts"; import type { BaseTransformContext } from "~/features/TransformContext/abstractions/BaseTransformContext.ts"; +import { DynamoDB } from "@aws-sdk/client-dynamodb"; +import { isAccessDeniedError, type AwsErrorLike } from "~/base/index.ts"; interface DdbProcessorSlice { putRecord(record: Record): void; @@ -29,13 +32,11 @@ class DdbProcessorImpl implements Processor.Interface< private readonly executor: DdbExecutor.Interface, private readonly config: MigrationConfig.Interface, private readonly sourceDb: SourceDynamoDbClient.Interface, - private readonly targetDb: TargetDynamoDbClient.Interface + private readonly targetDb: TargetDynamoDbClient.Interface, + private readonly transferContext: TransferContext.Interface ) {} public extendContext(base: BaseTransformContext.Interface): DdbProcessorSlice { - if (this.config.storage !== "ddb") { - throw new Error("DdbProcessor can only be used in ddb mode"); - } const sourceTable = this.config.source.dynamodb.tableName; const targetTable = this.config.target.dynamodb.tableName; const sourceDb = this.sourceDb; @@ -65,7 +66,53 @@ class DdbProcessorImpl implements Processor.Interface< ctx.putRecord(ctx.record as Record); } + public async checkAccess(): Promise { + const [sourceEntry, targetEntry] = await Promise.all([ + this.describeTable( + this.config.source.credentials, + this.config.source.region, + this.config.source.dynamodb.tableName, + "source" + ), + this.describeTable( + this.config.target.credentials, + this.config.target.region, + this.config.target.dynamodb.tableName, + "target" + ) + ]); + return [sourceEntry, targetEntry]; + } + + private async describeTable( + credentials: MigrationConfig.Interface["source"]["credentials"], + region: string, + tableName: string, + side: string + ): Promise { + const label = `DynamoDB ${side} table: ${tableName}`; + const client = new DynamoDB({ region, credentials: credentials as never }); + try { + await client.describeTable({ TableName: tableName }); + return { label, status: "ok" }; + } catch (error) { + if (isAccessDeniedError(error)) { + return { label, status: "denied" }; + } + const errName = (error as AwsErrorLike).name ?? (error as AwsErrorLike).code; + if (errName === "ResourceNotFoundException") { + return { label, status: "missing" }; + } + return { label, status: "unknown" }; + } finally { + client.destroy(); + } + } + public async execute(commands: Commands): Promise { + if (this.transferContext.dryRun) { + return; + } const puts = commands.get(PutRecord.key); await this.executor.execute(puts); } @@ -73,5 +120,11 @@ class DdbProcessorImpl implements Processor.Interface< export const DdbProcessor = Processor.createImplementation({ implementation: DdbProcessorImpl, - dependencies: [DdbExecutor, MigrationConfig, SourceDynamoDbClient, TargetDynamoDbClient] + dependencies: [ + DdbExecutor, + MigrationConfig, + SourceDynamoDbClient, + TargetDynamoDbClient, + TransferContext + ] }); diff --git a/src/features/DdbScanner/DdbScanner.ts b/src/features/DdbScanner/DdbScanner.ts index 8cfb8046..5d14f183 100644 --- a/src/features/DdbScanner/DdbScanner.ts +++ b/src/features/DdbScanner/DdbScanner.ts @@ -11,7 +11,7 @@ class DdbScannerImpl implements Scanner.Interface { ) {} public async listShards(): Promise { - const total = this.config.pipeline.segments ?? 1; + const total = this.config.pipeline?.segments ?? 1; const shards: DdbShard[] = []; for (let i = 0; i < total; i++) { shards.push({ segment: i, total }); @@ -20,9 +20,6 @@ class DdbScannerImpl implements Scanner.Interface { } public async *scan(shard: DdbShard): AsyncIterable { - if (this.config.storage !== "ddb") { - throw new Error("DdbScanner: source is not in DDB storage mode; check config.storage"); - } yield* this.source.scan(this.config.source.dynamodb.tableName, { segment: shard.segment, totalSegments: shard.total diff --git a/src/features/MigrationConfig/createConfig.ts b/src/features/MigrationConfig/createConfig.ts new file mode 100644 index 00000000..0cfc53f1 --- /dev/null +++ b/src/features/MigrationConfig/createConfig.ts @@ -0,0 +1,9 @@ +import { unifiedTransferInputSchema } from "./schemas/unified.schema.ts"; +import type { MigrationConfiguration } from "./validation.ts"; +import { z } from "zod"; + +export function createConfig( + input: z.input +): MigrationConfiguration { + return unifiedTransferInputSchema.parse(input); +} diff --git a/src/features/MigrationConfig/createDdbConfig.ts b/src/features/MigrationConfig/createDdbConfig.ts deleted file mode 100644 index 09805336..00000000 --- a/src/features/MigrationConfig/createDdbConfig.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { ddbTransferInputSchema, type DdbConfigInput } from "./schemas/ddb.schema.ts"; -import type { DdbMigrationConfiguration } from "./validation.ts"; - -/** - * Create a DynamoDB transfer configuration. - * - * Validates the input at creation time and returns a typed config - * with `storage: "ddb"` set automatically. - * - * @example - * ```typescript - * import { createDdbConfig } from "@webiny/data-transfer"; - * - * export default createDdbConfig({ - * source: { - * region: "us-east-1", - * credentials: { accessKeyId: "...", secretAccessKey: "..." }, - * dynamodb: { tableName: "webiny-v5-table" }, - * s3: { bucket: "webiny-v5-files" } - * }, - * target: { - * region: "us-east-1", - * credentials: { accessKeyId: "...", secretAccessKey: "..." }, - * dynamodb: { tableName: "webiny-v6-table" }, - * s3: { bucket: "webiny-v6-files" } - * }, - * pipeline: { preset: "v5-to-v6", segments: 4 } - * }); - * ``` - */ -export function createDdbConfig(input: DdbConfigInput): DdbMigrationConfiguration { - const parsed = ddbTransferInputSchema.parse(input); - return { - storage: "ddb", - ...parsed - }; -} diff --git a/src/features/MigrationConfig/createOsConfig.ts b/src/features/MigrationConfig/createOsConfig.ts deleted file mode 100644 index f57c94f2..00000000 --- a/src/features/MigrationConfig/createOsConfig.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { osTransferInputSchema, type OsConfigInput } from "./schemas/os.schema.ts"; -import type { OsMigrationConfiguration } from "./validation.ts"; - -/** - * Create an OpenSearch transfer configuration. - * - * Validates the input at creation time and returns a typed config - * with `storage: "os"` set automatically. - * - * @example - * ```typescript - * import { createOsConfig } from "@webiny/data-transfer"; - * - * export default createOsConfig({ - * source: { - * region: "us-east-1", - * credentials: { accessKeyId: "...", secretAccessKey: "..." }, - * dynamodb: { tableName: "webiny-v5-table" }, - * opensearch: { tableName: "webiny-v5-es-table" } - * }, - * target: { - * region: "us-east-1", - * credentials: { accessKeyId: "...", secretAccessKey: "..." }, - * opensearch: { - * endpoint: "https://search-xxx.us-east-1.es.amazonaws.com", - * tableName: "webiny-v6-es-table", - * service: "opensearch", - * indexPrefix: "" // empty string = no prefix - * } - * }, - * pipeline: { preset: "v5-to-v6-os", segments: 4 } - * }); - * ``` - */ -export function createOsConfig(input: OsConfigInput): OsMigrationConfiguration { - const parsed = osTransferInputSchema.parse(input); - return { - storage: "os", - ...parsed - }; -} diff --git a/src/features/MigrationConfig/index.ts b/src/features/MigrationConfig/index.ts index 58aed940..f802890d 100644 --- a/src/features/MigrationConfig/index.ts +++ b/src/features/MigrationConfig/index.ts @@ -1,5 +1,5 @@ export { MigrationConfig } from "./abstractions/index.ts"; export { MigrationConfigFeature } from "./feature.ts"; export { loadConfig } from "./loadConfig.ts"; -export { createDdbConfig } from "./createDdbConfig.ts"; -export { createOsConfig } from "./createOsConfig.ts"; +export { createConfig } from "./createConfig.ts"; +export { migrationConfigSchema, type MigrationConfiguration } from "./validation.ts"; diff --git a/src/features/MigrationConfig/loadConfig.ts b/src/features/MigrationConfig/loadConfig.ts index 94fb4f29..62c891f6 100644 --- a/src/features/MigrationConfig/loadConfig.ts +++ b/src/features/MigrationConfig/loadConfig.ts @@ -1,57 +1,41 @@ import { pathToFileURL } from "node:url"; import { dirname, resolve } from "node:path"; -import { MigrationConfig } from "./abstractions/MigrationConfig.ts"; +import type { MigrationConfig } from "./abstractions/MigrationConfig.ts"; +import { migrationConfigSchema } from "./validation.ts"; -/** - * Load a transfer configuration file. - * - * The config file should use `createDdbConfig` or `createOsConfig` - * to create and validate the config. The loader performs a lightweight - * check — the builder functions handle full validation. - */ export async function loadConfig(configPath: string): Promise { const absolutePath = resolve(process.cwd(), configPath); const fileUrl = pathToFileURL(absolutePath).href; - try { - const module = await import(fileUrl); - const config = module.default; + const module = await import(fileUrl).catch((err: unknown) => { + throw new Error( + `Failed to load config from ${configPath}: ${err instanceof Error ? err.message : String(err)}` + ); + }); - if (!config) { - throw new Error( - `Config file ${configPath} must have a default export. ` + - `Use createDdbConfig() or createOsConfig() to create your config.` - ); - } + const raw = module.default; - if (!config.storage || (config.storage !== "ddb" && config.storage !== "os")) { - throw new Error( - `Config file ${configPath} has invalid or missing "storage" field. ` + - `Use createDdbConfig() or createOsConfig() to create your config.` - ); - } + if (!raw) { + throw new Error( + `Config file ${configPath} must have a default export. Use createConfig() to create your config.` + ); + } - // Resolve path-shaped pipeline fields relative to the config file's - // directory. Built-in preset NAMES (e.g. "v5-to-v6") are left alone. - const configDir = dirname(absolutePath); - if (config.pipeline?.modelsDir) { - config.pipeline.modelsDir = resolve(configDir, config.pipeline.modelsDir); - } - if (config.pipeline?.presetsDir) { - config.pipeline.presetsDir = resolve(configDir, config.pipeline.presetsDir); - } - if ( - typeof config.pipeline?.preset === "string" && - (config.pipeline.preset.endsWith(".ts") || config.pipeline.preset.endsWith(".js")) - ) { - config.pipeline.preset = resolve(configDir, config.pipeline.preset); - } + const parsed = migrationConfigSchema.safeParse(raw); + if (!parsed.success) { + throw new Error(`Invalid config in ${configPath}:\n${parsed.error.message}`); + } - return config as MigrationConfig.Interface; - } catch (error) { - if (error instanceof Error) { - throw new Error(`Failed to load config from ${configPath}: ${error.message}`); + const config = parsed.data; + const configDir = dirname(absolutePath); + const pipeline = config.pipeline ?? {}; + + return { + ...config, + pipeline: { + ...pipeline, + ...(pipeline.modelsDir ? { modelsDir: resolve(configDir, pipeline.modelsDir) } : {}), + ...(pipeline.presetsDir ? { presetsDir: resolve(configDir, pipeline.presetsDir) } : {}) } - throw error; - } + }; } diff --git a/src/features/MigrationConfig/schemas/ddb.schema.ts b/src/features/MigrationConfig/schemas/ddb.schema.ts deleted file mode 100644 index 8502c98e..00000000 --- a/src/features/MigrationConfig/schemas/ddb.schema.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { z } from "zod"; -import { - credentialsOrProviderSchema, - debugSettingsSchema, - pipelineSettingsSchema, - trimmedString, - tuningSchema -} from "./shared.schema.ts"; - -const ddbSourceAccountConfigSchema = z.object({ - region: trimmedString(), - // Required. Either a literal credentials object, or a provider - // function (e.g. fromAwsProfile({profile: "dev"})). - credentials: credentialsOrProviderSchema, - dynamodb: z.object({ tableName: trimmedString() }), - s3: z.object({ bucket: trimmedString() }) -}); - -const ddbTargetAccountConfigSchema = ddbSourceAccountConfigSchema.extend({ - // Audit log table config. Set tableName to null to skip audit log - // transfer — records will be intercepted (blackholed) and NOT written - // to any target. NOTE: if you want audit logs transferred, you must - // provide a valid tableName here. - auditLog: z - .object({ - dynamodb: z.object({ tableName: trimmedString().nullable() }) - }) - .nullable() -}); - -export const ddbTransferInputSchema = z - .object({ - source: ddbSourceAccountConfigSchema, - target: ddbTargetAccountConfigSchema, - pipeline: pipelineSettingsSchema, - tuning: tuningSchema, - debug: debugSettingsSchema - }) - .superRefine((data, ctx) => { - // S3 bucket names are globally unique — same name means same bucket, - // regardless of region or account. Writing the transformed stream - // into the source bucket would overwrite originals. - if (data.source.s3.bucket === data.target.s3.bucket) { - ctx.addIssue({ - code: "custom", - path: ["target", "s3", "bucket"], - message: `Target S3 bucket "${data.target.s3.bucket}" is the same as source — would overwrite source files. Use a different bucket.` - }); - } - // Same region + same table name is the classic copy-paste misconfig. - // Could technically be safe across different AWS accounts, but - // requiring different names (or regions) makes the intent explicit - // instead of trusting that the user actually meant it. - if ( - data.source.region === data.target.region && - data.source.dynamodb.tableName === data.target.dynamodb.tableName - ) { - ctx.addIssue({ - code: "custom", - path: ["target", "dynamodb", "tableName"], - message: `Target DynamoDB table "${data.target.dynamodb.tableName}" in region "${data.target.region}" matches source. If these are different AWS accounts, rename one or change the target region to make the intent explicit.` - }); - } - // Audit log table must differ from the main target table to avoid - // writing audit log records into the primary data table. - if ( - data.target.auditLog?.dynamodb?.tableName != null && - data.target.auditLog.dynamodb.tableName === data.target.dynamodb.tableName - ) { - ctx.addIssue({ - code: "custom", - path: ["target", "auditLog", "dynamodb", "tableName"], - message: `Audit log DynamoDB table "${data.target.auditLog.dynamodb.tableName}" must differ from the main target table.` - }); - } - }); - -export type DdbConfigInput = z.infer; diff --git a/src/features/MigrationConfig/schemas/index.ts b/src/features/MigrationConfig/schemas/index.ts index da586ebe..29a2a8a8 100644 --- a/src/features/MigrationConfig/schemas/index.ts +++ b/src/features/MigrationConfig/schemas/index.ts @@ -1,2 +1 @@ -export { ddbTransferInputSchema, type DdbConfigInput } from "./ddb.schema.ts"; -export { osTransferInputSchema, type OsConfigInput } from "./os.schema.ts"; +export { unifiedTransferInputSchema } from "./unified.schema.ts"; diff --git a/src/features/MigrationConfig/schemas/os.schema.ts b/src/features/MigrationConfig/schemas/os.schema.ts deleted file mode 100644 index ecc4b04d..00000000 --- a/src/features/MigrationConfig/schemas/os.schema.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { z } from "zod"; -import { - credentialsOrProviderSchema, - debugSettingsSchema, - pipelineSettingsSchema, - trimmedString, - tuningSchema -} from "./shared.schema.ts"; - -const osSourceAccountConfigSchema = z.object({ - region: trimmedString(), - credentials: credentialsOrProviderSchema, - dynamodb: z.object({ tableName: trimmedString() }), - opensearch: z.object({ tableName: trimmedString() }) -}); - -const osTargetAccountConfigSchema = z.object({ - region: trimmedString(), - credentials: credentialsOrProviderSchema, - opensearch: z.object({ - // Zod's `.url()` doesn't trim — wrap through trimmedString first. - endpoint: trimmedString().url(), - tableName: trimmedString(), - service: z.enum(["opensearch", "opensearch-serverless"]), - indexPrefix: z.string().trim() - }) -}); - -export const osTransferInputSchema = z - .object({ - source: osSourceAccountConfigSchema, - target: osTargetAccountConfigSchema, - pipeline: pipelineSettingsSchema, - tuning: tuningSchema, - debug: debugSettingsSchema - }) - .superRefine((data, ctx) => { - // Same region + same OS companion DDB table is the classic copy- - // paste misconfig. Different accounts would technically be safe, - // but requiring different names (or regions) makes the intent - // explicit instead of trusting that the user actually meant it. - if ( - data.source.region === data.target.region && - data.source.opensearch.tableName === data.target.opensearch.tableName - ) { - ctx.addIssue({ - code: "custom", - path: ["target", "opensearch", "tableName"], - message: `Target OpenSearch DDB table "${data.target.opensearch.tableName}" in region "${data.target.region}" matches source. If these are different AWS accounts, rename one or change the target region to make the intent explicit.` - }); - } - }); - -export type OsConfigInput = z.infer; diff --git a/src/features/MigrationConfig/schemas/shared.schema.ts b/src/features/MigrationConfig/schemas/shared.schema.ts index ae9ad598..5d1ccd1b 100644 --- a/src/features/MigrationConfig/schemas/shared.schema.ts +++ b/src/features/MigrationConfig/schemas/shared.schema.ts @@ -47,12 +47,13 @@ export const credentialsOrProviderSchema = z.union([ }) ]); -export const pipelineSettingsSchema = z.object({ - preset: trimmedString(), - segments: z.number().int().positive().optional(), - modelsDir: trimmedString().optional(), - presetsDir: trimmedString().optional() -}); +export const pipelineSettingsSchema = z + .object({ + segments: z.number().int().positive().optional(), + modelsDir: trimmedString().optional(), + presetsDir: trimmedString().optional() + }) + .optional(); /** * Snapshot settings — when enabled, the runner dumps per-record JSONL @@ -74,11 +75,10 @@ export const debugSettingsSchema = z .object({ snapshot: z.union([z.boolean(), snapshotSettingsSchema]).optional(), /** - * When set, the runner writes raw pino JSONL to a log file in - * addition to stdout. `true` → default path - * (`.transfer//logs/.log`). - * String → explicit path; user is on their own for gitignore / - * cleanup. + * Controls log file output. By default logs are written to + * `.transfer//logs/.log`. + * Set to `false` to disable file logging. Set to a string to + * write to a custom path instead. */ logFile: z.union([z.boolean(), trimmedString()]).optional(), logLevel: z.enum(["debug", "info", "warn", "error"]).optional() @@ -90,6 +90,7 @@ export const debugSettingsSchema = z // rate-limited AWS account. export const tuningSchema = z .object({ + flushEvery: z.number().int().positive().optional(), ddb: z .object({ maxRetries: z.number().int().nonnegative().optional(), diff --git a/src/features/MigrationConfig/schemas/unified.schema.ts b/src/features/MigrationConfig/schemas/unified.schema.ts new file mode 100644 index 00000000..a8d874dd --- /dev/null +++ b/src/features/MigrationConfig/schemas/unified.schema.ts @@ -0,0 +1,113 @@ +import { z } from "zod"; +import { + credentialsOrProviderSchema, + debugSettingsSchema, + pipelineSettingsSchema, + trimmedString, + tuningSchema +} from "./shared.schema.ts"; + +const opensearchSourceSchema = z.object({ + tableName: trimmedString() +}); + +const opensearchTargetSchema = z.object({ + endpoint: trimmedString().url(), + tableName: trimmedString(), + service: z.enum(["opensearch", "opensearch-serverless"]), + indexPrefix: z.string().trim() +}); + +const sourceSchema = z.object({ + region: trimmedString(), + credentials: credentialsOrProviderSchema, + accountId: z.string().optional(), + dynamodb: z.object({ tableName: trimmedString() }), + s3: z.object({ bucket: trimmedString() }), + auditLog: z + .object({ + dynamodb: z.object({ tableName: trimmedString().nullable() }) + }) + .nullable() + .optional(), + opensearch: opensearchSourceSchema.nullable().optional() +}); + +const targetSchema = z.object({ + region: trimmedString(), + credentials: credentialsOrProviderSchema, + accountId: z.string().optional(), + dynamodb: z.object({ tableName: trimmedString() }), + s3: z.object({ bucket: trimmedString() }), + opensearch: opensearchTargetSchema.nullable().optional(), + auditLog: z + .object({ + dynamodb: z.object({ tableName: trimmedString().nullable() }) + }) + .nullable() + .optional() +}); + +export const unifiedTransferInputSchema = z + .object({ + source: sourceSchema, + target: targetSchema, + pipeline: pipelineSettingsSchema, + tuning: tuningSchema, + debug: debugSettingsSchema + }) + .superRefine((data, ctx) => { + if (data.source.s3.bucket === data.target.s3.bucket) { + ctx.addIssue({ + code: "custom", + path: ["target", "s3", "bucket"], + message: `Target S3 bucket "${data.target.s3.bucket}" is the same as source — would overwrite source files. Use a different bucket.` + }); + } + + if ( + data.source.region === data.target.region && + data.source.dynamodb.tableName === data.target.dynamodb.tableName + ) { + ctx.addIssue({ + code: "custom", + path: ["target", "dynamodb", "tableName"], + message: `Target DynamoDB table "${data.target.dynamodb.tableName}" in region "${data.target.region}" matches source. If these are different AWS accounts, rename one or change the target region to make the intent explicit.` + }); + } + + if ( + data.target.auditLog?.dynamodb?.tableName != null && + data.target.auditLog.dynamodb.tableName === data.target.dynamodb.tableName + ) { + ctx.addIssue({ + code: "custom", + path: ["target", "auditLog", "dynamodb", "tableName"], + message: `Audit log DynamoDB table "${data.target.auditLog.dynamodb.tableName}" must differ from the main target table.` + }); + } + + const hasSourceOs = data.source.opensearch != null; + const hasTargetOs = data.target.opensearch != null; + if (hasSourceOs !== hasTargetOs) { + ctx.addIssue({ + code: "custom", + path: hasSourceOs ? ["target", "opensearch"] : ["source", "opensearch"], + message: + "source.opensearch and target.opensearch must both be set or both be absent." + }); + } + + if ( + hasSourceOs && + hasTargetOs && + data.source.region === data.target.region && + data.source.opensearch!.tableName === data.target.opensearch!.tableName + ) { + ctx.addIssue({ + code: "custom", + path: ["target", "opensearch", "tableName"], + message: `Target OpenSearch DDB table "${data.target.opensearch!.tableName}" in region "${data.target.region}" matches source. If these are different AWS accounts, rename one or change the target region to make the intent explicit.` + }); + } + }); diff --git a/src/features/MigrationConfig/validation.ts b/src/features/MigrationConfig/validation.ts index d14bfd50..3664b908 100644 --- a/src/features/MigrationConfig/validation.ts +++ b/src/features/MigrationConfig/validation.ts @@ -1,28 +1,5 @@ import { z } from "zod"; -import { ddbTransferInputSchema } from "./schemas/ddb.schema.ts"; -import { osTransferInputSchema } from "./schemas/os.schema.ts"; - -// ============================================================================ -// Full config schemas (with storage discriminator) -// ============================================================================ - -const ddbConfigSchema = ddbTransferInputSchema.extend({ - storage: z.literal("ddb") -}); - -const osConfigSchema = osTransferInputSchema.extend({ - storage: z.literal("os") -}); - -export const migrationConfigSchema = z.discriminatedUnion("storage", [ - ddbConfigSchema, - osConfigSchema -]); - -// ============================================================================ -// Inferred Types -// ============================================================================ +import { unifiedTransferInputSchema } from "./schemas/unified.schema.ts"; +export const migrationConfigSchema = unifiedTransferInputSchema; export type MigrationConfiguration = z.infer; -export type DdbMigrationConfiguration = z.infer; -export type OsMigrationConfiguration = z.infer; diff --git a/src/features/ModelProvider/ModelProvider.ts b/src/features/ModelProvider/ModelProvider.ts index 2f43d4a0..b8b66d6a 100644 --- a/src/features/ModelProvider/ModelProvider.ts +++ b/src/features/ModelProvider/ModelProvider.ts @@ -28,7 +28,7 @@ class ModelProviderImpl implements ModelProviderAbstraction.Interface { config: MigrationConfig.Interface ) { this.tableName = config.source.dynamodb.tableName; - this.modelsDir = config.pipeline.modelsDir; + this.modelsDir = config.pipeline?.modelsDir; } public async preloadModels(tenantLocales: Map): Promise { diff --git a/src/features/OsProcessor/OsIndexPrefixHook.ts b/src/features/OsProcessor/OsIndexPrefixHook.ts index 3c62204c..ceaa2401 100644 --- a/src/features/OsProcessor/OsIndexPrefixHook.ts +++ b/src/features/OsProcessor/OsIndexPrefixHook.ts @@ -1,13 +1,13 @@ import { MigrationConfig } from "~/features/MigrationConfig/abstractions/MigrationConfig.ts"; import { BeforeTransferHook } from "~/features/TransferLifecycle/index.ts"; -import type { OsMigrationConfiguration } from "~/features/MigrationConfig/validation.ts"; class OsIndexPrefixHookImpl implements BeforeTransferHook.Interface { public constructor(private readonly config: MigrationConfig.Interface) {} public async execute(): Promise { - const osConfig = this.config as OsMigrationConfiguration; - process.env.OPENSEARCH_INDEX_PREFIX = osConfig.target.opensearch.indexPrefix; + if (this.config.target.opensearch) { + process.env.OPENSEARCH_INDEX_PREFIX = this.config.target.opensearch.indexPrefix; + } } } diff --git a/src/features/OsProcessor/OsProcessor.ts b/src/features/OsProcessor/OsProcessor.ts index 6d1eb376..35308d1d 100644 --- a/src/features/OsProcessor/OsProcessor.ts +++ b/src/features/OsProcessor/OsProcessor.ts @@ -1,7 +1,8 @@ import { join } from "node:path"; +import { Container } from "@webiny/di"; import { getBaseConfiguration } from "@webiny/api-opensearch/indexConfiguration/index.js"; -import { isRetryableAwsError } from "~/base/index.ts"; -import { Processor } from "~/domain/pipeline/abstractions/Processor.ts"; +import { isRetryableAwsError, ContainerToken } from "~/base/index.ts"; +import { AccessCheck, Processor } from "~/domain/pipeline/abstractions/Processor.ts"; import { DdbExecutor } from "~/features/DdbExecutor/abstractions/DdbExecutor.ts"; import { SourceDynamoDbClient, @@ -19,6 +20,10 @@ import type { Commands } from "~/domain/transform/commands/Commands.ts"; import type { BaseTransformContext } from "~/features/TransformContext/abstractions/BaseTransformContext.ts"; import { CompressionHandler } from "@webiny/utils/exports/api.js"; +interface OpenSearchErrorLike { + statusCode?: number; +} + const DEFAULT_RETRY_SCHEDULE: number[] = [5000, 10000, 20000, 30000, 30000]; const DEFAULT_REFRESH_INTERVAL = "1s"; const DISABLED_REFRESH_INTERVAL = "-1"; @@ -40,10 +45,12 @@ class OsProcessorImpl implements Processor.Interface< BaseTransformContext.Interface, OsProcessorSlice > { + private _osClient: OpenSearchClient.Interface | null = null; + public constructor( private readonly logger: Logger.Interface, private readonly ddbExecutor: DdbExecutor.Interface, - private readonly osClient: OpenSearchClient.Interface, + private readonly container: Container, private readonly compression: CompressionHandler.Interface, private readonly touchedIndexes: TouchedIndexes.Interface, private readonly config: MigrationConfig.Interface, @@ -54,9 +61,19 @@ class OsProcessorImpl implements Processor.Interface< private readonly targetDb: TargetDynamoDbClient.Interface ) {} + private get osClient(): OpenSearchClient.Interface { + if (!this._osClient) { + this._osClient = this.container.resolve(OpenSearchClient); + } + return this._osClient; + } + public extendContext(base: BaseTransformContext.Interface): OsProcessorSlice { - if (this.config.storage !== "os") { - throw new Error("OsProcessor can only be used in os mode"); + if (!this.config.target.opensearch) { + throw new Error("OsProcessor: config.target.opensearch is not configured."); + } + if (!this.config.source.opensearch) { + throw new Error("OsProcessor: config.source.opensearch is not configured."); } const sourceTable = this.config.source.opensearch.tableName; const targetTable = this.config.target.opensearch.tableName; @@ -88,6 +105,9 @@ class OsProcessorImpl implements Processor.Interface< } public async execute(commands: Commands): Promise { + if (this.transferContext.dryRun) { + return; + } const puts = commands.get(PutRecord.key); if (puts.length === 0) { return; @@ -104,6 +124,28 @@ class OsProcessorImpl implements Processor.Interface< await this.ddbExecutor.execute(gzippedPuts); } + public async checkAccess(): Promise { + if (!this.config.target.opensearch) { + return []; + } + // Source OS data is read indirectly via the source DDB table; no source-side OS probe needed. + const endpoint = this.config.target.opensearch.endpoint; + const label = `OpenSearch cluster: ${endpoint}`; + try { + await this.osClient.listIndexes(); + return [{ label, status: "ok" }]; + } catch (error) { + const statusCode = (error as OpenSearchErrorLike).statusCode; + if (statusCode === 401 || statusCode === 403) { + return [{ label, status: "denied" }]; + } + if (statusCode === 404) { + return [{ label, status: "missing" }]; + } + return [{ label, status: "unknown" }]; + } + } + public afterShard(ctx: Processor.AfterShardContext): void { const items = this.touchedIndexes.all(); if (items.length === 0) { @@ -254,7 +296,7 @@ export const OsProcessor = Processor.createImplementation({ dependencies: [ Logger, DdbExecutor, - OpenSearchClient, + ContainerToken, CompressionHandler, TouchedIndexes, MigrationConfig, diff --git a/src/features/OsScanner/OsScanner.ts b/src/features/OsScanner/OsScanner.ts index 6336697c..7b1419d0 100644 --- a/src/features/OsScanner/OsScanner.ts +++ b/src/features/OsScanner/OsScanner.ts @@ -14,7 +14,7 @@ class OsScannerImpl implements Scanner.Interface { ) {} public async listShards(): Promise { - const total = this.config.pipeline.segments ?? 1; + const total = this.config.pipeline?.segments ?? 1; const shards: OsShard[] = []; for (let i = 0; i < total; i++) { shards.push({ segment: i, total }); @@ -23,8 +23,8 @@ class OsScannerImpl implements Scanner.Interface { } public async *scan(shard: OsShard): AsyncIterable { - if (this.config.storage !== "os") { - throw new Error("OsScanner: source is not in OS storage mode; check config.storage"); + if (!this.config.source.opensearch) { + throw new Error("OsScanner: config.source.opensearch is not configured."); } const tableName = this.config.source.opensearch.tableName; for await (const raw of this.source.scan(tableName, { diff --git a/src/features/PipelineRunner/PipelineRunner.ts b/src/features/PipelineRunner/PipelineRunner.ts index 8bd6c01d..a7242926 100644 --- a/src/features/PipelineRunner/PipelineRunner.ts +++ b/src/features/PipelineRunner/PipelineRunner.ts @@ -13,6 +13,7 @@ import { SnapshotWriter } from "~/features/SnapshotWriter/abstractions/SnapshotW import { DroppedRecordLog } from "~/features/DroppedRecordLog/index.ts"; import { TransferredRecordLog } from "~/features/TransferredRecordLog/index.ts"; import { RecordDisposition } from "~/domain/pipeline/index.ts"; +import { MigrationConfig } from "~/features/MigrationConfig/abstractions/MigrationConfig.ts"; import { PipelineRunner as PipelineRunnerAbstraction, type RunOptions, @@ -49,6 +50,7 @@ class PipelineRunnerImpl implements PipelineRunnerAbstraction.Interface { public constructor( private readonly container: Container, + private readonly config: MigrationConfig.Interface, private readonly logger: Logger.Interface, private readonly transferContext: TransferContext.Interface, private readonly baseContextFactory: BaseTransformContextFactory.Interface, @@ -239,19 +241,13 @@ class PipelineRunnerImpl implements PipelineRunnerAbstraction.Interface { private async runShard(params: RunShardParams): Promise { const { mergeGroupId, pipelines, scanner, shard, pipelineProcessors, shardCtx } = params; - // Single shared command buffer for the whole shard. Per-record - // transformers + processor.onEnd hooks push into it via slice - // helpers / addCommand. At shard end, each processor.execute drains - // its own keys via commands.get(key) — which also marks them claimed. - // After all processors drain, commands.unclaimedKeys() reports any - // keys that nobody handled (transformer pushed X but pipeline lacks - // the processor that drains X). - const shardCommands = new Commands(); - - // Track per-shard dispatch counts — aggregate at shard end instead - // of per-record so a real prod run surfaces silent drops (records - // matching no pipeline filter) in the default `info` log instead - // of being invisible at `debug`. + + const flushEvery = this.config.tuning?.flushEvery ?? 500; + const processorOrder = this.collectProcessorOrder(pipelines, pipelineProcessors); + let pendingCommands = new Commands(); + let recordCount = 0; + let periodicFlushCount = 0; + const perPipelineTransferred: Map = new Map(); const perPipelineBlackholed: Map = new Map(); const unmatchedByType: Map = new Map(); @@ -272,7 +268,7 @@ class PipelineRunnerImpl implements PipelineRunnerAbstraction.Interface { pipeline, processors, record, - shardCommands, + pendingCommands, shardCtx ); if (result instanceof RecordDisposition.Blackholed) { @@ -288,16 +284,10 @@ class PipelineRunnerImpl implements PipelineRunnerAbstraction.Interface { ); this.transferredLog.add(record, pipeline.name); } - // First-match-wins: subsequent pipelines in this group are - // skipped for this record. Pipeline registration order - // determines priority. break; } if (!matched) { const { PK, SK, TYPE } = record as any; - // TYPE="unknown" is a real stored value in v5 but carries no - // useful identity — fall back to PK:SK so the summary entry - // identifies the actual record rather than grouping under "unknown". const typeKey: string = TYPE && TYPE !== "unknown" ? TYPE : `${PK}:${SK}`; unmatchedByType.set(typeKey, (unmatchedByType.get(typeKey) ?? 0) + 1); this.logger.warn(`unmatched record — TYPE=${typeKey} PK=${PK} SK=${SK}`); @@ -307,6 +297,13 @@ class PipelineRunnerImpl implements PipelineRunnerAbstraction.Interface { ); this.droppedLog.add(record, new RecordDisposition.Unmatched()); } + + recordCount++; + if (recordCount % flushEvery === 0) { + await this.flushShard(pendingCommands, processorOrder); + pendingCommands = new Commands(); + periodicFlushCount++; + } } this.logShardSummary( @@ -317,16 +314,10 @@ class PipelineRunnerImpl implements PipelineRunnerAbstraction.Interface { unmatchedByType ); - // Shard end: each unique processor (across pipelines in this group) - // drains the shared buffer in first-seen registration order. - const processorOrder = this.collectProcessorOrder(pipelines, pipelineProcessors); - for (const processor of processorOrder) { - await processor.execute(shardCommands); + if (pendingCommands.size() > 0 || periodicFlushCount === 0) { + await this.flushShard(pendingCommands, processorOrder); } - // Per-shard terminal hooks: each processor persists its own - // cross-boundary state (e.g., OsProcessor writes touchedIndexes). - // Sequential, processor array order — same as execute(). for (const processor of processorOrder) { if (!processor.afterShard) { continue; @@ -334,7 +325,6 @@ class PipelineRunnerImpl implements PipelineRunnerAbstraction.Interface { await processor.afterShard(shardCtx); } - this.warnUnclaimedKeys(shardCommands); this.droppedLog.flush(shardCtx.segment); this.transferredLog.flush(shardCtx.segment); @@ -345,6 +335,13 @@ class PipelineRunnerImpl implements PipelineRunnerAbstraction.Interface { }; } + private async flushShard(commands: Commands, processors: ProcessorInstance[]): Promise { + for (const processor of processors) { + await processor.execute(commands); + } + this.warnUnclaimedKeys(commands); + } + private async runRecord( pipeline: AnyPipeline, processors: ProcessorInstance[], @@ -543,6 +540,7 @@ export const PipelineRunner = PipelineRunnerAbstraction.createImplementation({ implementation: PipelineRunnerImpl, dependencies: [ ContainerToken, + MigrationConfig, Logger, TransferContext, BaseTransformContextFactory, diff --git a/src/features/PresetLoader/PresetLoader.ts b/src/features/PresetLoader/PresetLoader.ts index fd6acd10..dae533cf 100644 --- a/src/features/PresetLoader/PresetLoader.ts +++ b/src/features/PresetLoader/PresetLoader.ts @@ -71,7 +71,7 @@ class PresetLoaderImpl implements PresetLoaderAbstraction.Interface { return builtInPath; } - const presetsDir = this.config.pipeline.presetsDir; + const presetsDir = this.config.pipeline?.presetsDir; if (presetsDir) { const userPath = this.findUserPresetPath(presetNameOrPath, presetsDir); if (userPath) { diff --git a/src/features/S3Processor/S3Processor.ts b/src/features/S3Processor/S3Processor.ts index 8c89c9b7..97f51fca 100644 --- a/src/features/S3Processor/S3Processor.ts +++ b/src/features/S3Processor/S3Processor.ts @@ -1,6 +1,9 @@ -import { Processor } from "~/domain/pipeline/abstractions/Processor.ts"; +import { S3 } from "@webiny/aws-sdk/client-s3/index.js"; +import { AccessCheck, Processor } from "~/domain/pipeline/abstractions/Processor.ts"; +import { isAccessDeniedError, type AwsErrorLike } from "~/base/index.ts"; import { SourceS3Client, TargetS3Client } from "~/services/S3Client/abstractions/S3Client.ts"; import { MigrationConfig } from "~/features/MigrationConfig/abstractions/MigrationConfig.ts"; +import { TransferContext } from "~/features/TransferLifecycle/abstractions/TransferContext.ts"; import { S3Copy } from "~/domain/transform/commands/S3Copy.ts"; import type { Commands } from "~/domain/transform/commands/Commands.ts"; import type { BaseTransformContext } from "~/features/TransformContext/abstractions/BaseTransformContext.ts"; @@ -17,13 +20,11 @@ class S3ProcessorImpl implements Processor.Interface< public constructor( private readonly sourceS3: SourceS3Client.Interface, private readonly targetS3: TargetS3Client.Interface, - private readonly config: MigrationConfig.Interface + private readonly config: MigrationConfig.Interface, + private readonly transferContext: TransferContext.Interface ) {} public extendContext(base: BaseTransformContext.Interface): S3ProcessorSlice { - if (this.config.storage !== "ddb") { - throw new Error("S3Processor can only be used in ddb mode"); - } const sourceBucket = this.config.source.s3.bucket; const targetBucket = this.config.target.s3.bucket; const sourceS3 = this.sourceS3; @@ -42,7 +43,67 @@ class S3ProcessorImpl implements Processor.Interface< // No onEnd — S3 has no sensible per-record default. Transformers call // ctx.copyFile(...) explicitly when they want to emit a copy. + public async getGuardWarning(): Promise { + const sourceAccount = this.config.source.accountId || null; + const targetAccount = this.config.target.accountId || null; + if (sourceAccount === null || targetAccount === null || sourceAccount === targetAccount) { + return null; + } + return ( + `S3 file copy is cross-account: source account ${sourceAccount} → target account ${targetAccount}.\n` + + `CopyObject runs with target credentials — the source bucket "${this.config.source.s3.bucket}"\n` + + `must have a bucket policy granting account ${targetAccount} s3:GetObject access.` + ); + } + + public async checkAccess(): Promise { + const [sourceEntry, targetEntry] = await Promise.all([ + this.headBucket( + this.config.source.credentials, + this.config.source.region, + this.config.source.s3.bucket, + "source" + ), + this.headBucket( + this.config.target.credentials, + this.config.target.region, + this.config.target.s3.bucket, + "target" + ) + ]); + return [sourceEntry, targetEntry]; + } + + private async headBucket( + credentials: MigrationConfig.Interface["source"]["credentials"], + region: string, + bucket: string, + side: string + ): Promise { + const label = `S3 ${side} bucket: ${bucket}`; + const client = new S3({ region, credentials: credentials as never }); + try { + await client.headBucket({ Bucket: bucket }); + return { label, status: "ok" }; + } catch (error) { + if (isAccessDeniedError(error)) { + return { label, status: "denied" }; + } + const errName = (error as AwsErrorLike).name ?? (error as AwsErrorLike).code; + const httpStatus = (error as AwsErrorLike).$metadata?.httpStatusCode; + if (errName === "NoSuchBucket" || httpStatus === 404) { + return { label, status: "missing" }; + } + return { label, status: "unknown" }; + } finally { + client.destroy(); + } + } + public async execute(commands: Commands): Promise { + if (this.transferContext.dryRun) { + return; + } const copies = commands.get(S3Copy.key); if (copies.length === 0) { return; @@ -60,5 +121,5 @@ class S3ProcessorImpl implements Processor.Interface< export const S3Processor = Processor.createImplementation({ implementation: S3ProcessorImpl, - dependencies: [SourceS3Client, TargetS3Client, MigrationConfig] + dependencies: [SourceS3Client, TargetS3Client, MigrationConfig, TransferContext] }); diff --git a/src/features/TransferLifecycle/abstractions/TransferContext.ts b/src/features/TransferLifecycle/abstractions/TransferContext.ts index 02654f0c..9125f0d8 100644 --- a/src/features/TransferLifecycle/abstractions/TransferContext.ts +++ b/src/features/TransferLifecycle/abstractions/TransferContext.ts @@ -2,6 +2,7 @@ import { createAbstraction } from "~/base/index.ts"; interface ITransferContext { runId: string; + dryRun?: boolean; } export const TransferContext = createAbstraction("Transfer/TransferContext"); diff --git a/src/features/TransformContext/BaseTransformContextFactory.ts b/src/features/TransformContext/BaseTransformContextFactory.ts index b92088eb..b0e57cc9 100644 --- a/src/features/TransformContext/BaseTransformContextFactory.ts +++ b/src/features/TransformContext/BaseTransformContextFactory.ts @@ -27,7 +27,7 @@ class BaseTransformContextFactoryImpl implements BaseTransformContextFactoryAbst const ctx: BaseTransformContextAbstraction.Interface = { record: structuredClone(params.record), - original: Object.freeze(structuredClone(params.record)) as Readonly, + original: Object.freeze(structuredClone(params.record)), modelProvider, cache, logger, diff --git a/src/index.ts b/src/index.ts index e1396c6d..cc12f255 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,13 +2,14 @@ * Public API for config file authors. * * ```typescript - * import { createDdbConfig } from "@webiny/data-transfer"; + * import { createConfig } from "@webiny/data-transfer"; * - * export default createDdbConfig({ ... }); + * export default createConfig({ ... }); * ``` */ -export { createDdbConfig } from "./features/MigrationConfig/createDdbConfig.ts"; -export { createOsConfig } from "./features/MigrationConfig/createOsConfig.ts"; +export { createConfig } from "./features/MigrationConfig/createConfig.ts"; +export { migrationConfigSchema } from "./features/MigrationConfig/validation.ts"; +export type { MigrationConfiguration } from "./features/MigrationConfig/validation.ts"; export { loadEnv } from "./utils/load-env.ts"; export { fromEnv, numberFromEnv } from "./utils/fromEnv.ts"; export { initDataTransfer, type InitDataTransferContext } from "./utils/initDataTransfer.ts"; @@ -64,7 +65,8 @@ export { isOsBackgroundTask, isOsMailerSettings, isAuditLogEntry, - isMigrationRecord + isMigrationRecord, + isFormBuilderRecord } from "./domain/transform/filters.ts"; // Scanner / processor implementation tokens — passed into diff --git a/src/presets/copy-files.ts b/src/presets/copy-files.ts index 4abe8a52..3063b0b5 100644 --- a/src/presets/copy-files.ts +++ b/src/presets/copy-files.ts @@ -1,17 +1,21 @@ import { createTransferPreset } from "~/utils/createTransferPreset.ts"; import { DdbScanner } from "~/features/DdbScanner/index.ts"; import { S3Processor } from "~/features/S3Processor/index.ts"; +import { isFmFile } from "~/domain/transform/filters.ts"; +import { createFilter } from "~/domain/pipeline/index.js"; +import { DdbProcessor } from "~/features/DdbProcessor/index.js"; export default createTransferPreset({ name: "copy-files", - description: "Copy all the S3 files loaded via DynamoDB S3 files.", + description: "Copy all the S3 files loaded via DynamoDB regular table - pure copy.", configure({ runner, pipelineBuilderFactory: factory }): void { const everything = factory .create({ name: "S3 Files", scanner: DdbScanner, - processors: [S3Processor] + processors: [DdbProcessor, S3Processor] }) + .filter(createFilter(isFmFile)) .build(); runner.register(everything); diff --git a/src/presets/v5-to-v6-ddb.ts b/src/presets/v5-to-v6-ddb.ts index da3945f3..d6473fbe 100644 --- a/src/presets/v5-to-v6-ddb.ts +++ b/src/presets/v5-to-v6-ddb.ts @@ -16,6 +16,7 @@ import { isCmsModel, isFlpRecord, isFmFile, + isFormBuilderRecord, isMigrationRecord, isSecurityTeam } from "~/domain/transform/filters.ts"; @@ -57,7 +58,8 @@ import { export default createTransferPreset({ name: "v5-to-v6-ddb", - description: "Webiny v5 to v6 migration with all necessary transformations - DynamoDB only.", + description: + "Webiny v5 to v6 migration with all necessary transformations - Regular DynamoDb table.", configure({ runner, pipelineBuilderFactory: factory, container }): void { // ======================================================================== // Migration records — blackhole all PKs starting with "MIGRATION" @@ -89,7 +91,7 @@ export default createTransferPreset({ .filter(createFilter(isAuditLogEntry)) .use(auditLogTransformers) .blackhole(() => { - return config.storage !== "ddb" || !config.target.auditLog?.dynamodb?.tableName; + return !config.target.auditLog?.dynamodb?.tableName; }) .build(); @@ -268,6 +270,23 @@ export default createTransferPreset({ .use(addLiveField) .build(); + // ======================================================================== + // Form Builder — blackhole (no v6 migration path yet) + // Matches by PK (#FB# segment) first, then TYPE prefix fb.form.* and + // the standalone fb.formSubmission type. + // IMPORTANT: Must be registered AFTER CmsEntries because FB forms are + // CMS entries and would otherwise be claimed first. + // ======================================================================== + const formBuilderRecords = factory + .create({ + name: "FormBuilderRecords", + scanner: DdbScanner, + processors: [DdbProcessor] + }) + .filter(createFilter(isFormBuilderRecord)) + .blackhole() + .build(); + // ======================================================================== // Register pipelines with runner // IMPORTANT: Order matters due to first-match-wins behavior @@ -279,12 +298,13 @@ export default createTransferPreset({ .register(contentModelGroups) .register(backgroundTasks) .register(fmSettings) - .register(fmFiles) // Before cmsEntries + .register(fmFiles) .register(mailerSettings) .register(securityGroups) .register(securityTeams) .register(cmsModels) .register(folderPermissions) - .register(cmsEntries); // After fmFiles + .register(cmsEntries) + .register(formBuilderRecords); } }); diff --git a/src/services/DynamoDbClient/DynamoDbClient.ts b/src/services/DynamoDbClient/DynamoDbClient.ts index ce375e75..887284d9 100644 --- a/src/services/DynamoDbClient/DynamoDbClient.ts +++ b/src/services/DynamoDbClient/DynamoDbClient.ts @@ -12,7 +12,12 @@ import { import { QueryCommand } from "@aws-sdk/lib-dynamodb"; import { SourceDynamoDbClient } from "./abstractions/DynamoDbClient.ts"; import { DynamoDbClientConfig } from "./abstractions/DynamoDbClientConfig.ts"; -import { isRetryableAwsError, isTokenBucketExhausted, retryBackoffMs } from "~/base/index.ts"; +import { + isRetryableAwsError, + isThrottlingError, + isTokenBucketExhausted, + retryBackoffMs +} from "~/base/index.ts"; import type { Logger } from "~/tools/Logger/abstractions/Logger.ts"; import type { BaseRecord } from "~/domain/transform/types/records.ts"; @@ -204,9 +209,15 @@ export class DynamoDbClientImpl implements SourceDynamoDbClient.Interface { // Token bucket needs time to refill — enforce a minimum 10s wait. const backoff = isTokenBucketExhausted(error) ? Math.max(base, 10000) : base; const err = error as { message?: string; name?: string }; - this.logger.warn( - `DynamoDB retry ${attempt + 1}/${this.maxRetries}: ${err.name ?? "Error"} — ${err.message ?? String(error)} (backoff ${backoff}ms)` - ); + if (isThrottlingError(error)) { + this.logger.debug( + `DDB throttled — ${err.name ?? "ThrottlingError"} (attempt ${attempt + 1}/${this.maxRetries}, backoff ${backoff}ms)` + ); + } else { + this.logger.warn( + `DDB retry ${attempt + 1}/${this.maxRetries}: ${err.name ?? "Error"} — ${err.message ?? String(error)} (backoff ${backoff}ms)` + ); + } await new Promise(resolve => setTimeout(resolve, backoff)); } } diff --git a/src/services/OpenSearchClient/OpenSearchClient.ts b/src/services/OpenSearchClient/OpenSearchClient.ts index c98f59dd..ae4c19b8 100644 --- a/src/services/OpenSearchClient/OpenSearchClient.ts +++ b/src/services/OpenSearchClient/OpenSearchClient.ts @@ -2,11 +2,12 @@ import { Client } from "@opensearch-project/opensearch"; import { AwsSigv4Signer } from "@opensearch-project/opensearch/aws"; import { OpenSearchClient as OpenSearchClientAbstraction } from "./abstractions/OpenSearchClient.ts"; import { OpenSearchClientConfig } from "./abstractions/OpenSearchClientConfig.ts"; +import { Logger } from "~/tools/Logger/abstractions/Logger.ts"; class OpenSearchClientImpl implements OpenSearchClientAbstraction.Interface { private client: Client; - public constructor(config: OpenSearchClientConfig.Interface) { + public constructor(config: OpenSearchClientConfig.Interface, logger: Logger.Interface) { // Normalize credentials: user config may pass either a literal // object or a provider function (fromAwsProfile). AwsSigv4Signer // wants a `getCredentials` async function, so wrap either shape. @@ -32,6 +33,12 @@ class OpenSearchClientImpl implements OpenSearchClientAbstraction.Interface { node: config.endpoint, maxRetries: config.maxRetries ?? 3 }); + + this.client.on("response", (err, _meta) => { + if (err && "statusCode" in err && (err as { statusCode?: number }).statusCode === 429) { + logger.debug(`OpenSearch throttled — 429 Too Many Requests`); + } + }); } public async indexExists(index: string): Promise { @@ -78,5 +85,5 @@ class OpenSearchClientImpl implements OpenSearchClientAbstraction.Interface { export const OpenSearchClient = OpenSearchClientAbstraction.createImplementation({ implementation: OpenSearchClientImpl, - dependencies: [OpenSearchClientConfig] + dependencies: [OpenSearchClientConfig, Logger] }); diff --git a/src/services/S3Client/S3Client.ts b/src/services/S3Client/S3Client.ts index 8aeb19e3..2b6e8a47 100644 --- a/src/services/S3Client/S3Client.ts +++ b/src/services/S3Client/S3Client.ts @@ -6,7 +6,12 @@ import { } from "@webiny/aws-sdk/client-s3/index.js"; import { SourceS3Client } from "./abstractions/S3Client.ts"; import { S3ClientConfig } from "./abstractions/S3ClientConfig.ts"; -import { isRetryableAwsError, isTokenBucketExhausted, retryBackoffMs } from "~/base/index.ts"; +import { + isRetryableAwsError, + isThrottlingError, + isTokenBucketExhausted, + retryBackoffMs +} from "~/base/index.ts"; import type { Logger } from "~/tools/Logger/abstractions/Logger.ts"; // See DynamoDbClient for the rationale on 6 retries + the jittered @@ -135,9 +140,15 @@ export class S3ClientImpl implements SourceS3Client.Interface { const base = retryBackoffMs(attempt, this.initialBackoff); const backoff = isTokenBucketExhausted(error) ? Math.max(base, 10000) : base; const err = error as { message?: string; name?: string }; - this.logger.warn( - `S3 retry ${attempt + 1}/${this.maxRetries}: ${err.name ?? "Error"} — ${err.message ?? String(error)} (backoff ${backoff}ms)` - ); + if (isThrottlingError(error)) { + this.logger.debug( + `S3 throttled — ${err.name ?? "ThrottlingError"} (attempt ${attempt + 1}/${this.maxRetries}, backoff ${backoff}ms)` + ); + } else { + this.logger.warn( + `S3 retry ${attempt + 1}/${this.maxRetries}: ${err.name ?? "Error"} — ${err.message ?? String(error)} (backoff ${backoff}ms)` + ); + } await new Promise(resolve => setTimeout(resolve, backoff)); } } diff --git a/src/tools/Logger/PinoLogger.ts b/src/tools/Logger/PinoLogger.ts index fdf1353f..2f9667f8 100644 --- a/src/tools/Logger/PinoLogger.ts +++ b/src/tools/Logger/PinoLogger.ts @@ -1,6 +1,11 @@ -import pino, { multistream, type LevelWithSilentOrString, type StreamEntry } from "pino"; +import pino, { + multistream, + type DestinationStream, + type LevelWithSilentOrString, + type StreamEntry +} from "pino"; import pretty from "pino-pretty"; -import { createWriteStream, mkdirSync } from "node:fs"; +import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; import { Writable } from "node:stream"; import { Logger } from "./abstractions/Logger.ts"; @@ -63,9 +68,9 @@ const createPrettyDestination = (): Writable => { }); }; -const createFileDestination = (path: string): Writable => { +const createFileDestination = (path: string): DestinationStream => { mkdirSync(dirname(path), { recursive: true }); - return createWriteStream(path, { flags: "a" }); + return pino.destination({ dest: path, append: true, periodicFlush: 5000 }); }; export class PinoLogger implements Logger.Interface { diff --git a/src/utils/fromEnv.ts b/src/utils/fromEnv.ts index 257a49d6..42363c78 100644 --- a/src/utils/fromEnv.ts +++ b/src/utils/fromEnv.ts @@ -15,11 +15,15 @@ */ export function fromEnv(name: string): string; export function fromEnv(name: string, defaultValue: string): string; -export function fromEnv(name: string, defaultValue?: string): string { - const value = readEnvValue(name) ?? defaultValue; +export function fromEnv(name: string, defaultValue: null): string | null; +export function fromEnv(name: string, defaultValue?: string | null): string | null { + const value = readEnvValue(name); if (value !== undefined) { return value; } + if (defaultValue !== undefined) { + return defaultValue; + } throw missingVariableError(name); } diff --git a/src/utils/load-env.ts b/src/utils/load-env.ts index cdbf2562..ac2a758a 100644 --- a/src/utils/load-env.ts +++ b/src/utils/load-env.ts @@ -11,11 +11,11 @@ import { fileURLToPath } from "node:url"; * * @example * ```typescript - * import { loadEnv, createDdbConfig } from "@webiny/data-transfer"; + * import { loadEnv, createConfig } from "@webiny/data-transfer"; * * loadEnv(import.meta.url); * - * export default createDdbConfig({ ... }); + * export default createConfig({ ... }); * ``` */ export function loadEnv(importMetaUrl: string): void { diff --git a/templates/.claude/skills/writing-data-transfer-config/SKILL.md b/templates/.claude/skills/writing-data-transfer-config/SKILL.md index e4424f20..ff2214d8 100644 --- a/templates/.claude/skills/writing-data-transfer-config/SKILL.md +++ b/templates/.claude/skills/writing-data-transfer-config/SKILL.md @@ -1,23 +1,22 @@ --- name: writing-data-transfer-config -description: Use when writing or editing a @webiny/data-transfer config file (ddb.transfer.config.ts / os.transfer.config.ts / custom.transfer.config.ts). Covers createDdbConfig / createOsConfig signatures, credential shapes (fromAwsProfile vs literal), fromEnv / numberFromEnv helpers, loadEnv, source/target collision + trimming rules, pointing at a preset, tuning knobs. +description: Use when writing or editing a @webiny/data-transfer config file (config.ts). Covers createConfig signature, credential shapes (fromAwsProfile vs literal), fromEnv / numberFromEnv helpers, loadEnv, source/target collision + trimming rules, tuning knobs. --- # Writing a `@webiny/data-transfer` config -A config is a `.ts` file that `export default`s one of the two factory calls: +A config is a `config.ts` file (one per project folder) that `export default`s `createConfig(...)`. -- **`createDdbConfig(...)`** — DDB primary table (+ S3 files). Covers CMS + security + file manager + tenancy. -- **`createOsConfig(...)`** — OpenSearch companion DDB table. CMS entries only, gzipped records. +`createConfig` validates with Zod at import time — invalid configs fail fast with a useful message, before any AWS call. -Both validate with Zod at import time — invalid configs fail fast with a useful message, before any AWS call. +OpenSearch fields (`source.opensearch` / `target.opensearch`) are optional — omit them entirely for a DDB-only transfer. ## Minimal shape ```ts import { loadEnv, - createDdbConfig, + createConfig, fromAwsProfile, fromEnv, numberFromEnv @@ -25,22 +24,31 @@ import { loadEnv(import.meta.url); -export default createDdbConfig({ +export default createConfig({ source: { - region: fromEnv("SOURCE_REGION", "us-east-1"), + region: fromEnv("SOURCE_REGION", "eu-central-1"), credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", "default") }), dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, s3: { bucket: fromEnv("SOURCE_S3_BUCKET") } + // opensearch: { tableName: fromEnv("SOURCE_OS_TABLE") } }, target: { - region: fromEnv("TARGET_REGION", "us-east-1"), + region: fromEnv("TARGET_REGION", "eu-central-1"), credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", "default") }), dynamodb: { tableName: fromEnv("TARGET_DDB_TABLE") }, - s3: { bucket: fromEnv("TARGET_S3_BUCKET") } + s3: { bucket: fromEnv("TARGET_S3_BUCKET") }, + auditLog: { dynamodb: { tableName: fromEnv("TARGET_AUDIT_LOGS_TABLE") } } + // opensearch: { + // endpoint: fromEnv("TARGET_OS_ENDPOINT"), + // tableName: fromEnv("TARGET_OS_TABLE"), + // service: "opensearch", + // indexPrefix: fromEnv("TARGET_OS_INDEX_PREFIX", "") + // } }, pipeline: { - preset: "v5-to-v6-ddb", // built-in, OR "./presets/my-preset.ts" - segments: numberFromEnv("SEGMENTS", 4) + segments: numberFromEnv("SEGMENTS", 4), + modelsDir: fromEnv("MODELS_DIR", "./models"), + presetsDir: "./presets" } }); ``` @@ -112,26 +120,24 @@ Loads the `.env` file **next to the config file** (not the one at the repo root) Enforced by Zod at build time: -- **All string fields are trimmed** (`region`, `tableName`, `bucket`, `endpoint`, creds, preset). A trailing-space paste error doesn't silently corrupt anything. +- **All string fields are trimmed** (`region`, `tableName`, `bucket`, `endpoint`, creds). A trailing-space paste error doesn't silently corrupt anything. - **Whitespace-only rejected** — empty-after-trim is treated as missing. - **Source/target collision guard**: - Same S3 bucket on both sides → rejected (would overwrite source files). - Same region + same DDB / OS-DDB table name → rejected (would read and write to the same table). Same table name across DIFFERENT regions is allowed — distinct physical tables. -## Pointing at a preset +## `pipeline.presetsDir` — preset discovery -`pipeline.preset` takes one of: -- **A built-in name**: `"v5-to-v6-ddb"` or `"v5-to-v6-os"` (filename in `src/presets/` without extension). The runner auto-discovers built-ins. -- **A file path**: `"./presets/my-preset.ts"` or `"../shared/presets/foo.ts"`. Resolved relative to the CONFIG file's directory. +`pipeline.presetsDir` points at a directory of preset files (e.g., `"./presets"`). The runner discovers them at startup; the transfer wizard lets you pick one at runtime. No preset path is needed in the config file itself. ## `pipeline.modelsDir` — CMS model definitions -Required by the OS preset (`v5-to-v6-os`) and by built-in transformers that inspect field types (`fixBrokenStorageKeys`, `transformRichText`, `addLiveField`). Point at a directory of exported model definitions. +Used by built-in transformers that inspect field types (`fixBrokenStorageKeys`, `transformRichText`, `addLiveField`). Point at a directory of exported model definitions. ```ts pipeline: { - preset: "v5-to-v6-os", - modelsDir: fromEnv("MODELS_DIR", "./models") + modelsDir: fromEnv("MODELS_DIR", "./models"), + presetsDir: "./presets" } ``` @@ -182,6 +188,7 @@ Post-run inspection: `cat .transfer//logs/*.log | pino-pretty`. Default p ```ts tuning: { + flushEvery: numberFromEnv("FLUSH_EVERY", 500), // records per shard flush — bounds peak memory ddb: { maxRetries: 3, initialBackoffMs: 100 }, s3: { concurrency: 10, maxRetries: 3, initialBackoffMs: 100 }, os: { @@ -194,12 +201,14 @@ tuning: { All optional; absent = built-in defaults. AWS SDK `retryMode: "adaptive"` is always on for DDB + S3 — it self-tunes backoff based on real throttle signals, so you usually don't need to tune these. +**`flushEvery`** caps peak per-shard memory. The runner calls `processor.execute()` every N records and resets the pending-commands buffer. Default 500 (≈ 5 MB at a 10 KB average record). Lower to 100 for tables with very large records (approaching the 400 KB DDB max). + ## Running it From the user project root: ```bash -yarn transfer --config=./projects//ddb.transfer.config.ts +yarn transfer --config=./projects//config.ts ``` Or with a specific AWS profile pre-set in `.env`: @@ -211,9 +220,9 @@ TARGET_PROFILE=staging-writer ## Common patterns -- **DDB first, then OS** — run them as separate transfers with separate config files. They don't share state. DDB uses `v5-to-v6-ddb`; OS uses `v5-to-v6-os`. Both can share the same `.env`. -- **Multiple target environments** — duplicate the project folder under `projects/` with different `.env`. Configs stay identical. -- **Custom preset** — if the built-in doesn't match your needs, write one (see `writing-data-transfer-preset` skill) and point `pipeline.preset` at its file path. +- **One config per project** — a single `config.ts` handles both DDB and OS transfers. The wizard picks the preset at runtime. +- **Multiple target environments** — duplicate the project folder under `projects/` with different `.env`. The `config.ts` stays identical. +- **Custom preset** — write one (see `writing-data-transfer-preset` skill), drop it in `presetsDir`, and the wizard will discover it automatically. ## Anti-patterns diff --git a/templates/internal-project/.env.example b/templates/internal-project/.env.example index 2bc7648c..fd847727 100644 --- a/templates/internal-project/.env.example +++ b/templates/internal-project/.env.example @@ -20,6 +20,7 @@ SOURCE_REGION={{SOURCE_REGION}} SOURCE_DDB_TABLE={{SOURCE_DDB_TABLE}} SOURCE_S3_BUCKET={{SOURCE_S3_BUCKET}} +SOURCE_AUDIT_LOGS_TABLE={{SOURCE_AUDIT_LOGS_TABLE}} SOURCE_OS_TABLE={{SOURCE_OS_TABLE}} # --- Target environment ------------------------------------------------ @@ -33,6 +34,7 @@ TARGET_REGION={{TARGET_REGION}} TARGET_DDB_TABLE={{TARGET_DDB_TABLE}} TARGET_S3_BUCKET={{TARGET_S3_BUCKET}} +TARGET_AUDIT_LOGS_TABLE={{TARGET_AUDIT_LOGS_TABLE}} TARGET_OS_TABLE={{TARGET_OS_TABLE}} TARGET_OS_ENDPOINT={{TARGET_OS_ENDPOINT}} TARGET_OS_INDEX_PREFIX={{TARGET_OS_INDEX_PREFIX}} @@ -40,3 +42,5 @@ TARGET_OS_INDEX_PREFIX={{TARGET_OS_INDEX_PREFIX}} # --- Tuning ------------------------------------------------------------------ # Number of parallel worker processes (DDB parallel-scan segments). SEGMENTS={{SEGMENTS}} +# Records to accumulate per shard before flushing writes (bounds peak memory). +# FLUSH_EVERY=500 diff --git a/templates/internal-project/config.ts b/templates/internal-project/config.ts new file mode 100644 index 00000000..74ce67fb --- /dev/null +++ b/templates/internal-project/config.ts @@ -0,0 +1,56 @@ +import { + createConfig, + fromAwsProfile, + fromEnv, + loadEnv, + numberFromEnv +} from "@webiny/data-transfer"; + +// Loads .env from the same directory as this file. `.env*` is gitignored. +loadEnv(import.meta.url); + +const DEFAULT_REGION = "eu-central-1"; +const DEFAULT_PROFILE = "default"; + +const sourceOsTable = fromEnv("SOURCE_OS_TABLE", null); +const sourceAuditLogTable = fromEnv("SOURCE_AUDIT_LOGS_TABLE", null); +const targetOsTable = fromEnv("TARGET_OS_TABLE", null); +const targetOsEndpoint = fromEnv("TARGET_OS_ENDPOINT", null); +const targetAuditLogTable = fromEnv("TARGET_AUDIT_LOGS_TABLE", null); + +export default createConfig({ + source: { + region: fromEnv("SOURCE_REGION", DEFAULT_REGION), + credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", DEFAULT_PROFILE) }), + accountId: fromEnv("SOURCE_ACCOUNT_ID", ""), + dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, + s3: { bucket: fromEnv("SOURCE_S3_BUCKET") }, + auditLog: sourceAuditLogTable ? { dynamodb: { tableName: sourceAuditLogTable } } : null, + opensearch: sourceOsTable ? { tableName: sourceOsTable } : null + }, + target: { + region: fromEnv("TARGET_REGION", DEFAULT_REGION), + credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", DEFAULT_PROFILE) }), + accountId: fromEnv("TARGET_ACCOUNT_ID", ""), + dynamodb: { tableName: fromEnv("TARGET_DDB_TABLE") }, + s3: { bucket: fromEnv("TARGET_S3_BUCKET") }, + auditLog: targetAuditLogTable ? { dynamodb: { tableName: targetAuditLogTable } } : null, + opensearch: + targetOsTable && targetOsEndpoint + ? { + endpoint: targetOsEndpoint, + tableName: targetOsTable, + service: "opensearch" as const, + indexPrefix: fromEnv("TARGET_OS_INDEX_PREFIX", "") + } + : null + }, + pipeline: { + segments: numberFromEnv("SEGMENTS", 4), + modelsDir: fromEnv("MODELS_DIR", "./models"), + presetsDir: "./presets" + }, + tuning: { + flushEvery: numberFromEnv("FLUSH_EVERY", 500) + } +}); diff --git a/templates/internal-project/ddb.transfer.config.ts b/templates/internal-project/ddb.transfer.config.ts deleted file mode 100644 index c3794df7..00000000 --- a/templates/internal-project/ddb.transfer.config.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { loadEnv, createDdbConfig, fromAwsProfile, fromEnv, numberFromEnv } from "~/index.ts"; - -loadEnv(import.meta.url); - -const DEFAULT_REGION = "eu-central-1"; -const DEFAULT_PROFILE = "default"; - -export default createDdbConfig({ - source: { - region: fromEnv("SOURCE_REGION", DEFAULT_REGION), - // Profile-based credentials — reads ~/.aws/credentials. - // To use literal credentials instead, replace with: - // { accessKeyId: fromEnv("SOURCE_AWS_ACCESS_KEY_ID"), - // secretAccessKey: fromEnv("SOURCE_AWS_SECRET_ACCESS_KEY") } - credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", DEFAULT_PROFILE) }), - dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, - s3: { bucket: fromEnv("SOURCE_S3_BUCKET") } - }, - target: { - region: fromEnv("TARGET_REGION", DEFAULT_REGION), - credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", DEFAULT_PROFILE) }), - dynamodb: { tableName: fromEnv("TARGET_DDB_TABLE") }, - s3: { bucket: fromEnv("TARGET_S3_BUCKET") }, - // Audit log table. Set tableName to transfer audit logs to a separate - // target table, or keep null to skip (audit log records are dropped). - auditLog: null - // auditLog: { dynamodb: { tableName: fromEnv("TARGET_AUDIT_LOGS_TABLE") } } - }, - pipeline: { - // Uses the example preset in ./presets/ddb.ts (copies all records + S3 files verbatim). - // To use a built-in preset instead: preset: "v5-to-v6-ddb" - preset: "ddb", - // presetsDir lets you reference custom presets by name (without a path). - // Drop .ts files into ./presets/ and use their filename as the preset name. - presetsDir: "./presets", - segments: numberFromEnv("SEGMENTS", 4) - // modelsDir: "./models" // uncomment to load CMS model JSON overrides - } -}); diff --git a/templates/internal-project/os.transfer.config.ts b/templates/internal-project/os.transfer.config.ts deleted file mode 100644 index 111c1f50..00000000 --- a/templates/internal-project/os.transfer.config.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { loadEnv, createOsConfig, fromAwsProfile, fromEnv, numberFromEnv } from "~/index.ts"; - -loadEnv(import.meta.url); - -const DEFAULT_REGION = "eu-central-1"; -const DEFAULT_PROFILE = "default"; - -export default createOsConfig({ - source: { - region: fromEnv("SOURCE_REGION", DEFAULT_REGION), - // Profile-based credentials — reads ~/.aws/credentials. - // To use literal credentials instead, replace with: - // { accessKeyId: fromEnv("SOURCE_AWS_ACCESS_KEY_ID"), - // secretAccessKey: fromEnv("SOURCE_AWS_SECRET_ACCESS_KEY") } - credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", DEFAULT_PROFILE) }), - dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, - opensearch: { tableName: fromEnv("SOURCE_OS_TABLE") } - }, - target: { - region: fromEnv("TARGET_REGION", DEFAULT_REGION), - credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", DEFAULT_PROFILE) }), - opensearch: { - endpoint: fromEnv("TARGET_OS_ENDPOINT"), - tableName: fromEnv("TARGET_OS_TABLE"), - // "opensearch" for a managed domain; "opensearch-serverless" for serverless. - service: "opensearch", - indexPrefix: fromEnv("TARGET_OS_INDEX_PREFIX", "") - } - }, - pipeline: { - // Uses the example preset in ./presets/os.ts (copies all OS records verbatim). - // To use a built-in preset instead: preset: "v5-to-v6-os" - // Run AFTER the DDB transfer completes. - preset: "os", - // presetsDir lets you reference custom presets by name (without a path). - // Drop .ts files into ./presets/ and use their filename as the preset name. - presetsDir: "./presets", - segments: numberFromEnv("SEGMENTS", 4) - // modelsDir: "./models" // required when using OS transformers that read CMS models - } -}); diff --git a/templates/projects/example/.env.example b/templates/projects/example/.env.example index e537e863..3e17406d 100644 --- a/templates/projects/example/.env.example +++ b/templates/projects/example/.env.example @@ -42,3 +42,5 @@ TARGET_OS_INDEX_PREFIX={{TARGET_OS_INDEX_PREFIX}} # --- Tuning ------------------------------------------------------------------ # Number of parallel worker processes (DDB parallel-scan segments). SEGMENTS={{SEGMENTS}} +# Records to accumulate per shard before flushing writes (bounds peak memory). +# FLUSH_EVERY=500 diff --git a/templates/projects/example/ddb.transfer.config.ts b/templates/projects/example/config.ts similarity index 72% rename from templates/projects/example/ddb.transfer.config.ts rename to templates/projects/example/config.ts index aeb288f1..2970f99a 100644 --- a/templates/projects/example/ddb.transfer.config.ts +++ b/templates/projects/example/config.ts @@ -1,6 +1,6 @@ import { loadEnv, - createDdbConfig, + createConfig, fromAwsProfile, fromEnv, numberFromEnv @@ -12,9 +12,9 @@ import { // transfer from the repository root. loadEnv(import.meta.url); -export default createDdbConfig({ +export default createConfig({ source: { - region: fromEnv("SOURCE_REGION", "us-east-1"), + region: fromEnv("SOURCE_REGION", "eu-central-1"), // AWS credentials — TWO SHAPES ACCEPTED. Pick one. // // A) Profile-based (default below): reads ~/.aws/credentials, @@ -34,24 +34,39 @@ export default createDdbConfig({ // // Optional — only set for temporary STS credentials: // // sessionToken: fromEnv("SOURCE_AWS_SESSION_TOKEN") // }, + accountId: fromEnv("SOURCE_ACCOUNT_ID", ""), dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, s3: { bucket: fromEnv("SOURCE_S3_BUCKET") } + // Uncomment if your Webiny project uses OpenSearch (Elasticsearch): + // opensearch: { tableName: fromEnv("SOURCE_OS_TABLE") } }, target: { - region: fromEnv("TARGET_REGION", "us-east-1"), + region: fromEnv("TARGET_REGION", "eu-central-1"), credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", "default") }), // credentials: { // accessKeyId: fromEnv("TARGET_AWS_ACCESS_KEY_ID"), // secretAccessKey: fromEnv("TARGET_AWS_SECRET_ACCESS_KEY"), // // sessionToken: fromEnv("TARGET_AWS_SESSION_TOKEN") // }, + accountId: fromEnv("TARGET_ACCOUNT_ID", ""), dynamodb: { tableName: fromEnv("TARGET_DDB_TABLE") }, - s3: { bucket: fromEnv("TARGET_S3_BUCKET") } + s3: { bucket: fromEnv("TARGET_S3_BUCKET") }, + auditLog: { dynamodb: { tableName: fromEnv("TARGET_AUDIT_LOGS_TABLE") } } + // Uncomment if your Webiny project uses OpenSearch (Elasticsearch): + // opensearch: { + // endpoint: fromEnv("TARGET_OS_ENDPOINT"), + // tableName: fromEnv("TARGET_OS_TABLE"), + // service: "opensearch", + // indexPrefix: fromEnv("TARGET_OS_INDEX_PREFIX", "") + // } }, pipeline: { - preset: "../../presets/example.ts", - segments: numberFromEnv("SEGMENTS", 4) - // modelsDir: "./models" + segments: numberFromEnv("SEGMENTS", 4), + modelsDir: fromEnv("MODELS_DIR", "./models"), + presetsDir: "./presets" + }, + tuning: { + flushEvery: numberFromEnv("FLUSH_EVERY", 500) } // // Optional debug helpers — uncomment either or both to enable. diff --git a/templates/projects/example/custom.transfer.config.ts b/templates/projects/example/custom.transfer.config.ts deleted file mode 100644 index 5df0285c..00000000 --- a/templates/projects/example/custom.transfer.config.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { - loadEnv, - createDdbConfig, - fromAwsProfile, - fromEnv, - numberFromEnv -} from "@webiny/data-transfer"; - -// Loads the .env file next to this config file. -loadEnv(import.meta.url); - -// Same source/target shape as ddb.transfer.config.ts — the only difference is -// `pipeline.preset` points at a file path (resolved relative to this config -// file's directory) instead of a built-in preset name. -export default createDdbConfig({ - source: { - region: fromEnv("SOURCE_REGION", "us-east-1"), - // Profile-based credentials. See ddb.transfer.config.ts for the - // literal `{accessKeyId, secretAccessKey, sessionToken?}` alternative. - credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", "default") }), - // credentials: { - // accessKeyId: fromEnv("SOURCE_AWS_ACCESS_KEY_ID"), - // secretAccessKey: fromEnv("SOURCE_AWS_SECRET_ACCESS_KEY"), - // // sessionToken: fromEnv("SOURCE_AWS_SESSION_TOKEN") - // }, - dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, - s3: { bucket: fromEnv("SOURCE_S3_BUCKET") } - }, - target: { - region: fromEnv("TARGET_REGION", "us-east-1"), - credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", "default") }), - // credentials: { - // accessKeyId: fromEnv("TARGET_AWS_ACCESS_KEY_ID"), - // secretAccessKey: fromEnv("TARGET_AWS_SECRET_ACCESS_KEY"), - // // sessionToken: fromEnv("TARGET_AWS_SESSION_TOKEN") - // }, - dynamodb: { tableName: fromEnv("TARGET_DDB_TABLE") }, - s3: { bucket: fromEnv("TARGET_S3_BUCKET") } - }, - pipeline: { - preset: "../../presets/example.ts", - segments: numberFromEnv("SEGMENTS", 1) - } - // - // Optional debug helpers — see ddb.transfer.config.ts for full - // comments. Both fields are opt-in. - // - // debug: { - // snapshot: true, // dump to .transfer//snapshot/ - // logFile: true // log to .transfer//logs/*.log - // } -}); diff --git a/templates/projects/example/os.transfer.config.ts b/templates/projects/example/os.transfer.config.ts deleted file mode 100644 index d1543619..00000000 --- a/templates/projects/example/os.transfer.config.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { - loadEnv, - createOsConfig, - fromAwsProfile, - fromEnv, - numberFromEnv -} from "@webiny/data-transfer"; - -// Loads the .env file from THIS directory (next to this config file). -// Using import.meta.url ensures each project folder loads its own .env, -// so config stays isolated between projects — even when you run the -// transfer from the repository root. -loadEnv(import.meta.url); - -export default createOsConfig({ - source: { - region: fromEnv("SOURCE_REGION", "us-east-1"), - // Profile-based credentials — reads ~/.aws/credentials. For the - // literal `{accessKeyId, secretAccessKey, sessionToken?}` shape, - // see the commented alternative in ddb.transfer.config.ts. - credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", "default") }), - // credentials: { - // accessKeyId: fromEnv("SOURCE_AWS_ACCESS_KEY_ID"), - // secretAccessKey: fromEnv("SOURCE_AWS_SECRET_ACCESS_KEY"), - // // sessionToken: fromEnv("SOURCE_AWS_SESSION_TOKEN") - // }, - dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, - opensearch: { tableName: fromEnv("SOURCE_OS_TABLE") } - }, - target: { - region: fromEnv("TARGET_REGION", "us-east-1"), - credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", "default") }), - // credentials: { - // accessKeyId: fromEnv("TARGET_AWS_ACCESS_KEY_ID"), - // secretAccessKey: fromEnv("TARGET_AWS_SECRET_ACCESS_KEY"), - // // sessionToken: fromEnv("TARGET_AWS_SESSION_TOKEN") - // }, - opensearch: { - endpoint: fromEnv("TARGET_OS_ENDPOINT"), - tableName: fromEnv("TARGET_OS_TABLE"), - service: "opensearch", - indexPrefix: fromEnv("TARGET_OS_INDEX_PREFIX", "") - } - }, - pipeline: { - preset: "v5-to-v6-os", - segments: numberFromEnv("SEGMENTS", 4), - modelsDir: fromEnv("MODELS_DIR", "./models") - } - // - // Optional debug helpers — see ddb.transfer.config.ts for full - // comments. Both fields are opt-in. - // - // debug: { - // snapshot: true, // dump to .transfer//snapshot/ - // logFile: true // log to .transfer//logs/*.log - // } -}); diff --git a/tsconfig.json b/tsconfig.json index b3ca568c..119bf617 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -20,7 +20,6 @@ "outDir": "./dist", "rootDirs": ["./src", "__tests__"], "paths": { - "@/*": ["./*"], "~/*": ["./src/*"] } }, diff --git a/vitest.config.ts b/vitest.config.ts index c19967c1..685cdd6f 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -20,10 +20,10 @@ export default defineConfig({ provider: "v8", reporter: ["text", "json", "html"], thresholds: { - lines: 77, - functions: 80, - branches: 70, - statements: 77 + lines: 79, + functions: 84, + branches: 71, + statements: 79 } } } diff --git a/yarn.lock b/yarn.lock index 808d7acc..02e88bff 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4021,12 +4021,12 @@ __metadata: languageName: node linkType: hard -"@inquirer/checkbox@npm:^5.1.4": - version: 5.1.4 - resolution: "@inquirer/checkbox@npm:5.1.4" +"@inquirer/checkbox@npm:^5.1.5": + version: 5.1.5 + resolution: "@inquirer/checkbox@npm:5.1.5" dependencies: "@inquirer/ansi": "npm:^2.0.5" - "@inquirer/core": "npm:^11.1.9" + "@inquirer/core": "npm:^11.1.10" "@inquirer/figures": "npm:^2.0.5" "@inquirer/type": "npm:^4.0.5" peerDependencies: @@ -4034,28 +4034,28 @@ __metadata: peerDependenciesMeta: "@types/node": optional: true - checksum: 10c0/38d6c8b49c392307c29fbb252d5dd09a819aecc34c83f01a7652efa09d7b8b901c448216496eda3962e319209243297ec575eff4e081a758bb08adb805f7e3d4 + checksum: 10c0/6cf3bfbc0e39b80b8a37b69e49231c22616877b31b77507a55be5363d14b33e91cd3c1eb4ebf0ba89435ab5ef8204ce634579a23ab7b186ee8c71d54a7c959ee languageName: node linkType: hard -"@inquirer/confirm@npm:^6.0.12": - version: 6.0.12 - resolution: "@inquirer/confirm@npm:6.0.12" +"@inquirer/confirm@npm:^6.0.13": + version: 6.0.13 + resolution: "@inquirer/confirm@npm:6.0.13" dependencies: - "@inquirer/core": "npm:^11.1.9" + "@inquirer/core": "npm:^11.1.10" "@inquirer/type": "npm:^4.0.5" peerDependencies: "@types/node": ">=18" peerDependenciesMeta: "@types/node": optional: true - checksum: 10c0/36e9b1ef60e08562f07bcbcd78ba0a681b2fa3e5f54fa5e12303fc5982e8ba875ed782c24161e2295028b2b404ba690e841837303d16eeeb28df7aa9eb8aa835 + checksum: 10c0/59f3c484f405b3ffe2e97a9e4927111d71d317ea84fa14dc0ffd2d7c902f934ad31f48b380765937f5ff85722ece475edcde8a64b8018c21f2ee3e33082cee8b languageName: node linkType: hard -"@inquirer/core@npm:^11.1.9": - version: 11.1.9 - resolution: "@inquirer/core@npm:11.1.9" +"@inquirer/core@npm:^11.1.10": + version: 11.1.10 + resolution: "@inquirer/core@npm:11.1.10" dependencies: "@inquirer/ansi": "npm:^2.0.5" "@inquirer/figures": "npm:^2.0.5" @@ -4069,15 +4069,15 @@ __metadata: peerDependenciesMeta: "@types/node": optional: true - checksum: 10c0/f7b162ce5f67fb75aab00a3668fdd8c8629aec790087840ea66ee8ead6009ab2066bec9cbf5bcc394ccdf130e6139051d6bace334b3a66c4f05349585213172c + checksum: 10c0/d1d4081cbb0bd3dc15a3c95c58560a4836079a14161c407bc20f9a1b0fdb93c2228daf61aa60acab7023bc78df1b85875fde94308dcac2c6bd0ffd24b5f05190 languageName: node linkType: hard -"@inquirer/editor@npm:^5.1.1": - version: 5.1.1 - resolution: "@inquirer/editor@npm:5.1.1" +"@inquirer/editor@npm:^5.1.2": + version: 5.1.2 + resolution: "@inquirer/editor@npm:5.1.2" dependencies: - "@inquirer/core": "npm:^11.1.9" + "@inquirer/core": "npm:^11.1.10" "@inquirer/external-editor": "npm:^3.0.0" "@inquirer/type": "npm:^4.0.5" peerDependencies: @@ -4085,22 +4085,22 @@ __metadata: peerDependenciesMeta: "@types/node": optional: true - checksum: 10c0/d9506fc4d16362ee060df0c8aa722ee85eae79e0be373253ecc49d0f51569f886bf0d09da9ae9910e3d5f79dca10767a5c1711c799af41c668b2a97866470221 + checksum: 10c0/5b24700b8d4339d4b9b5fe5991d4c35843c1977595bfc0ab242e95a2bd8c2463c9039050f3b7821e8f49786926c5b68ba4cb20d6d887292fc0b12ccb03bd3ca2 languageName: node linkType: hard -"@inquirer/expand@npm:^5.0.13": - version: 5.0.13 - resolution: "@inquirer/expand@npm:5.0.13" +"@inquirer/expand@npm:^5.0.14": + version: 5.0.14 + resolution: "@inquirer/expand@npm:5.0.14" dependencies: - "@inquirer/core": "npm:^11.1.9" + "@inquirer/core": "npm:^11.1.10" "@inquirer/type": "npm:^4.0.5" peerDependencies: "@types/node": ">=18" peerDependenciesMeta: "@types/node": optional: true - checksum: 10c0/108055f24dc4348a4981003003ca41fe793c7b3292ff80f583a1e1f1097bee64a0c2e5795cae969bcf77d4f34f03a3feeaa6315363eb01554f1c1eae6769c9f5 + checksum: 10c0/efdc9a93d57397f415529ed2b969de6fa78c2ab98901a89efd73f1cfc79eba3ea899c621a63a458c530ca4142c89fd61cfeb873384e906f9cc33c022a5a3f5be languageName: node linkType: hard @@ -4126,95 +4126,95 @@ __metadata: languageName: node linkType: hard -"@inquirer/input@npm:^5.0.12": - version: 5.0.12 - resolution: "@inquirer/input@npm:5.0.12" +"@inquirer/input@npm:^5.0.13": + version: 5.0.13 + resolution: "@inquirer/input@npm:5.0.13" dependencies: - "@inquirer/core": "npm:^11.1.9" + "@inquirer/core": "npm:^11.1.10" "@inquirer/type": "npm:^4.0.5" peerDependencies: "@types/node": ">=18" peerDependenciesMeta: "@types/node": optional: true - checksum: 10c0/bb8273925c774bc5dffd93d6edc9e24391ab03c3ddd0c604987abbb74a7efcf03ca10b3bd0607ccf6613369e004417b14cf3c031327601118cff2cc98c5098e6 + checksum: 10c0/df2d67a6f0a1b4cc22dfdc1e78f833560f613647bd75374d4664601b7947106dd977aceb8440a19a17204b58c11611e6084096eae3eb1873260f1f1d029a8a0a languageName: node linkType: hard -"@inquirer/number@npm:^4.0.12": - version: 4.0.12 - resolution: "@inquirer/number@npm:4.0.12" +"@inquirer/number@npm:^4.0.13": + version: 4.0.13 + resolution: "@inquirer/number@npm:4.0.13" dependencies: - "@inquirer/core": "npm:^11.1.9" + "@inquirer/core": "npm:^11.1.10" "@inquirer/type": "npm:^4.0.5" peerDependencies: "@types/node": ">=18" peerDependenciesMeta: "@types/node": optional: true - checksum: 10c0/e933967d9879792775d4ac8f4abcc9c2f263107a1e3c1cdd5ce85dd446a03bed2f6cb01c41ed1a3b45b80f385eb3a991d1442703d9816416c79c0ca6f20e57f3 + checksum: 10c0/9185d15c8b0ab820dc0456e7db6f189ae84bf8d5f5bce19398f3a858fca0276e66a53922a4ab118dbae65f1f4f6a29f06f3d34c6436ae9e095a98e9862590e2b languageName: node linkType: hard -"@inquirer/password@npm:^5.0.12": - version: 5.0.12 - resolution: "@inquirer/password@npm:5.0.12" +"@inquirer/password@npm:^5.0.13": + version: 5.0.13 + resolution: "@inquirer/password@npm:5.0.13" dependencies: "@inquirer/ansi": "npm:^2.0.5" - "@inquirer/core": "npm:^11.1.9" + "@inquirer/core": "npm:^11.1.10" "@inquirer/type": "npm:^4.0.5" peerDependencies: "@types/node": ">=18" peerDependenciesMeta: "@types/node": optional: true - checksum: 10c0/e1399196f01ff5fadc1d3e1bc6946a4f53a5753ccf8e39b2f11f1dae5c51fdb85e33ef44760d83c5b58dacc442de603f32e9c991b2e5f2f8a2a451ff62e1fe48 + checksum: 10c0/e06aa6ae4344e3e37630655c443001bcf4421b1e4821c15987fc747b8d0362d536a52a115d25d02b023bb7091fc88630a65d7b0ac02e15a2f70d933f6a863da3 languageName: node linkType: hard -"@inquirer/prompts@npm:^8.4.2": - version: 8.4.2 - resolution: "@inquirer/prompts@npm:8.4.2" - dependencies: - "@inquirer/checkbox": "npm:^5.1.4" - "@inquirer/confirm": "npm:^6.0.12" - "@inquirer/editor": "npm:^5.1.1" - "@inquirer/expand": "npm:^5.0.13" - "@inquirer/input": "npm:^5.0.12" - "@inquirer/number": "npm:^4.0.12" - "@inquirer/password": "npm:^5.0.12" - "@inquirer/rawlist": "npm:^5.2.8" - "@inquirer/search": "npm:^4.1.8" - "@inquirer/select": "npm:^5.1.4" +"@inquirer/prompts@npm:^8.4.3": + version: 8.4.3 + resolution: "@inquirer/prompts@npm:8.4.3" + dependencies: + "@inquirer/checkbox": "npm:^5.1.5" + "@inquirer/confirm": "npm:^6.0.13" + "@inquirer/editor": "npm:^5.1.2" + "@inquirer/expand": "npm:^5.0.14" + "@inquirer/input": "npm:^5.0.13" + "@inquirer/number": "npm:^4.0.13" + "@inquirer/password": "npm:^5.0.13" + "@inquirer/rawlist": "npm:^5.2.9" + "@inquirer/search": "npm:^4.1.9" + "@inquirer/select": "npm:^5.1.5" peerDependencies: "@types/node": ">=18" peerDependenciesMeta: "@types/node": optional: true - checksum: 10c0/0519d6a52e195e24b03949afda1ad7fc2ae752c72a7ba554d4bdbbac844fd47dc83d79e67f732c6bf3c56a407e3171b92bd3b0dc334fd35eea446a889d1100f7 + checksum: 10c0/fd77efb0b12a9293d6533cf332a1af10f69fefa97202c3790c2a7695a06c443ed5d4877d05efe512803e4a98cce0fa46fed9d372149512c2eccbfcf23f1c44bd languageName: node linkType: hard -"@inquirer/rawlist@npm:^5.2.8": - version: 5.2.8 - resolution: "@inquirer/rawlist@npm:5.2.8" +"@inquirer/rawlist@npm:^5.2.9": + version: 5.2.9 + resolution: "@inquirer/rawlist@npm:5.2.9" dependencies: - "@inquirer/core": "npm:^11.1.9" + "@inquirer/core": "npm:^11.1.10" "@inquirer/type": "npm:^4.0.5" peerDependencies: "@types/node": ">=18" peerDependenciesMeta: "@types/node": optional: true - checksum: 10c0/35b0a9e29972342669365af70ea8feebc4e13ce688ce53c1fae723cbd02a7082294553eeb49ee443f5e993c1a97be31fcbe366f7cb573c075557316717792527 + checksum: 10c0/10f1a23e5222a932d9965490beb8b37551b2c68dcb281290d8f0099dc0778b49d4ca671ad8b346c802cbff29924faffd01b62dc88f297020e3d1061eb00b7f78 languageName: node linkType: hard -"@inquirer/search@npm:^4.1.8": - version: 4.1.8 - resolution: "@inquirer/search@npm:4.1.8" +"@inquirer/search@npm:^4.1.9": + version: 4.1.9 + resolution: "@inquirer/search@npm:4.1.9" dependencies: - "@inquirer/core": "npm:^11.1.9" + "@inquirer/core": "npm:^11.1.10" "@inquirer/figures": "npm:^2.0.5" "@inquirer/type": "npm:^4.0.5" peerDependencies: @@ -4222,16 +4222,16 @@ __metadata: peerDependenciesMeta: "@types/node": optional: true - checksum: 10c0/7281c59e8953aa4663c5afe3d6a3b558ec3e97867569f29dcc0a67e89ff05b37f374ac34ebc7356a7137f76a5172d52c60469cadcc323cc733b4d635b5cf3334 + checksum: 10c0/0e0cd6f2f312cecfa02f7d5556a7ccf5cbffff442fbdd0828f320e0a0239b63d24db5e31e05ec77d2a969900ad40b2d115f6496b2c9c47561a15dc504298d9dc languageName: node linkType: hard -"@inquirer/select@npm:^5.1.4": - version: 5.1.4 - resolution: "@inquirer/select@npm:5.1.4" +"@inquirer/select@npm:^5.1.5": + version: 5.1.5 + resolution: "@inquirer/select@npm:5.1.5" dependencies: "@inquirer/ansi": "npm:^2.0.5" - "@inquirer/core": "npm:^11.1.9" + "@inquirer/core": "npm:^11.1.10" "@inquirer/figures": "npm:^2.0.5" "@inquirer/type": "npm:^4.0.5" peerDependencies: @@ -4239,7 +4239,7 @@ __metadata: peerDependenciesMeta: "@types/node": optional: true - checksum: 10c0/30b6b061acd08f3f4cfde08a626446d308271e32569acec03f8c87a17e04c21aeccbecf354c8b254a2decfbe7d2189d8ae4443bf10d0be4a2aedeb4e850574c2 + checksum: 10c0/871e05266c00151031798dc659acaed3d9c76d0040a25204cbcc99f7104559db1a8bc2a73ee04318a01f06f879a3d7e5986db50f563aeca23ae4cd5e8cff6057 languageName: node linkType: hard @@ -6032,8 +6032,8 @@ __metadata: linkType: hard "@pulumi/pulumi@npm:^3.142.0, @pulumi/pulumi@npm:^3.234.0": - version: 3.236.0 - resolution: "@pulumi/pulumi@npm:3.236.0" + version: 3.237.0 + resolution: "@pulumi/pulumi@npm:3.237.0" dependencies: "@grpc/grpc-js": "npm:^1.10.1" "@logdna/tail-file": "npm:^2.0.6" @@ -6070,7 +6070,7 @@ __metadata: optional: true typescript: optional: true - checksum: 10c0/a63076c377420a52f301b668c3c8f8fe6cb9034489f882f157bcda8d52b88068545b57d57e1b8a656c1ce04bb4f63786c6e0bd16d23212884fa5379c77ae92d2 + checksum: 10c0/bf34ef3230cdbe93ff7ae97df97e3fc9d38db01f8f2c3e2e7276b8f7b05bb9586376df4968c770fb7aef64edf5f21d4ae129924d33790db416eeb7192d5d3cee languageName: node linkType: hard @@ -6539,180 +6539,116 @@ __metadata: languageName: node linkType: hard -"@smithy/chunked-blob-reader-native@npm:^4.2.3": - version: 4.2.3 - resolution: "@smithy/chunked-blob-reader-native@npm:4.2.3" - dependencies: - "@smithy/util-base64": "npm:^4.3.2" - tslib: "npm:^2.6.2" - checksum: 10c0/cac49faa52e1692fb2c9837252c6a4cbbe1eaba2b90b267d5e36e935735fa419bfd83f98b8c933e0cf03cabb914a409c2002f4f55184ea1b5cae83cd8fa4f617 - languageName: node - linkType: hard - -"@smithy/chunked-blob-reader@npm:^5.2.2": - version: 5.2.2 - resolution: "@smithy/chunked-blob-reader@npm:5.2.2" - dependencies: - tslib: "npm:^2.6.2" - checksum: 10c0/ec1021d9e1d2cff7e3168a3c29267387cec8857e4ed1faa80e77f5671a50a241136098b4801a6a1d7ba8b348e8c936792627bd698ab4cf00e0ed73479eb51abd - languageName: node - linkType: hard - "@smithy/config-resolver@npm:^4.4.17": - version: 4.4.17 - resolution: "@smithy/config-resolver@npm:4.4.17" + version: 4.5.0 + resolution: "@smithy/config-resolver@npm:4.5.0" dependencies: - "@smithy/node-config-provider": "npm:^4.3.14" - "@smithy/types": "npm:^4.14.1" - "@smithy/util-config-provider": "npm:^4.2.2" - "@smithy/util-endpoints": "npm:^3.4.2" - "@smithy/util-middleware": "npm:^4.2.14" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/44e653e2ed31bf765f0b30ca404dc3e02f30ad800971f812a6363b71da8d37de5d7d8901d9db4d86d2493f25037904f35c01bbb1a83a2a72e7e7b201ce5c411a + checksum: 10c0/d85777a1ee614c4574cf269ace4bde4ebf7dadf2d6bfc853d6e916246cb7679a47056d1a1f91f6c5f4eab01cdadc49a7f56b0c889e4f2e3929a06efbaa8e19ec languageName: node linkType: hard -"@smithy/core@npm:^3.23.17": - version: 3.23.17 - resolution: "@smithy/core@npm:3.23.17" +"@smithy/core@npm:^3.23.17, @smithy/core@npm:^3.24.0": + version: 3.24.0 + resolution: "@smithy/core@npm:3.24.0" dependencies: - "@smithy/protocol-http": "npm:^5.3.14" + "@aws-crypto/crc32": "npm:5.2.0" "@smithy/types": "npm:^4.14.1" - "@smithy/url-parser": "npm:^4.2.14" - "@smithy/util-base64": "npm:^4.3.2" - "@smithy/util-body-length-browser": "npm:^4.2.2" - "@smithy/util-middleware": "npm:^4.2.14" - "@smithy/util-stream": "npm:^4.5.25" - "@smithy/util-utf8": "npm:^4.2.2" - "@smithy/uuid": "npm:^1.1.2" tslib: "npm:^2.6.2" - checksum: 10c0/223631835e93c314a8fa394db724673c0940d3ba9e5ffbd73bd7c09854d7e003d19de950b44ce408e099c72ed3c22eb5e41370abea4ac656f7037e6da07be3c1 + checksum: 10c0/f317e449a193e9e12afd76719ea58cdfb316d03fc46e6452b20a089665c86941a76f30e7c7a0bc9e2232334e673d432c08f647ca6651a2c145a33198ec0ee0d6 languageName: node linkType: hard "@smithy/credential-provider-imds@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/credential-provider-imds@npm:4.2.14" - dependencies: - "@smithy/node-config-provider": "npm:^4.3.14" - "@smithy/property-provider": "npm:^4.2.14" - "@smithy/types": "npm:^4.14.1" - "@smithy/url-parser": "npm:^4.2.14" - tslib: "npm:^2.6.2" - checksum: 10c0/62ced0249cb1ba64c6dd98a90b35b93dd9e3f1469d020752c46b5a83ecef38280f4b29c2f63e3b0c414a2fa2ec7631a19370415c80ed4d6ea18a9be040803126 - languageName: node - linkType: hard - -"@smithy/eventstream-codec@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/eventstream-codec@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/credential-provider-imds@npm:4.3.0" dependencies: - "@aws-crypto/crc32": "npm:5.2.0" + "@smithy/core": "npm:^3.24.0" "@smithy/types": "npm:^4.14.1" - "@smithy/util-hex-encoding": "npm:^4.2.2" tslib: "npm:^2.6.2" - checksum: 10c0/c2f9139004b3f75d3621b21e0ef80f850baf4955cce230e6d18faef171c2b4dc62694b1d86d4000a7ec1babb482afeca666520f69cf31455ed63259da6b2a3ee + checksum: 10c0/b3bc4c5f3123dee1043fe3ff22c1c24d8e00cad66f41739a6ab8489d1571167444bd0d4568c3b10ad624adb35c73f39a6fa1cfaf6d830aa3b0dbb4f8c654253c languageName: node linkType: hard "@smithy/eventstream-serde-browser@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/eventstream-serde-browser@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/eventstream-serde-browser@npm:4.3.0" dependencies: - "@smithy/eventstream-serde-universal": "npm:^4.2.14" - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/b624cf962ec84bbc61c419938de07548cbff3b1049fe5fb0c88097bdb516e6f3af22f6856ab3a5212cf352e7f1c69b0c5d03370cb4fe7f91a29dff975c385da5 + checksum: 10c0/a08f3f52a352b7a9011fe9383707f6b2bf13ca62f2aeda38043305acce75e821a8b6cb9779be370ecbae1dc7b8c61b5847aaf425e5ed296cfa8a81a7ff370c97 languageName: node linkType: hard "@smithy/eventstream-serde-config-resolver@npm:^4.3.14": - version: 4.3.14 - resolution: "@smithy/eventstream-serde-config-resolver@npm:4.3.14" + version: 4.4.0 + resolution: "@smithy/eventstream-serde-config-resolver@npm:4.4.0" dependencies: - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/d889a6e12797928c9507e99193f86ba0b7807441b67305643ec6b8b953c54237aa0ae7a4baaf00eb72ff07e3c71e88d6225de3db3d2b33729af38adb81b593f8 + checksum: 10c0/bbfa08ebdd1e85d1fd88cac12718a4a6287f0dd9d07bf28f2c951bc6d9059dca7a1568abee3b87efc929df8d9579b81cdbe730594dde84a6c2aaac791e607b45 languageName: node linkType: hard "@smithy/eventstream-serde-node@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/eventstream-serde-node@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/eventstream-serde-node@npm:4.3.0" dependencies: - "@smithy/eventstream-serde-universal": "npm:^4.2.14" - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/c9dda5011ef3e6565dfa483811567b942fd822dfefb41768213fc9322b5298075c4a9228f8aa745af5bcebb23a2a61e24f091ce2be263da1ca3713aafa89c243 - languageName: node - linkType: hard - -"@smithy/eventstream-serde-universal@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/eventstream-serde-universal@npm:4.2.14" - dependencies: - "@smithy/eventstream-codec": "npm:^4.2.14" - "@smithy/types": "npm:^4.14.1" - tslib: "npm:^2.6.2" - checksum: 10c0/82cf563ff67b6543f0981dc3b1ef7fc3b507d5322e611a4f7fde4ae90d1d0eae172298c5bdda3837c5dd5c4d8839d59b72189797e178709be7b1996b3ea71a64 + checksum: 10c0/6bd23367e6d797f1f1fb56ad59ffdb1bdce0e37bca178ad26a11c7fbefb28b2e0dd9ace39343afb374beb0e20c63185179c0463dd764ffb9a7085fd38247538e languageName: node linkType: hard "@smithy/fetch-http-handler@npm:^5.3.17": - version: 5.3.17 - resolution: "@smithy/fetch-http-handler@npm:5.3.17" + version: 5.4.0 + resolution: "@smithy/fetch-http-handler@npm:5.4.0" dependencies: - "@smithy/protocol-http": "npm:^5.3.14" - "@smithy/querystring-builder": "npm:^4.2.14" + "@smithy/core": "npm:^3.24.0" "@smithy/types": "npm:^4.14.1" - "@smithy/util-base64": "npm:^4.3.2" tslib: "npm:^2.6.2" - checksum: 10c0/8adac6bf9d5735f8ddbe9e59dd268f985881b64b03cba7e83401471b3094139189476324fe640bbd19d75f5cd8e098d9a2a7f4925bc9fadecd1ccbe937773c12 + checksum: 10c0/d043f9cf9055eb44da8ac932bc5f04f313a69a43c60bdc5daaf0777ce724fab6f760717c62f385f1d44dc183e1211e333ea673d10c1b5c1afe56a912e17dd800 languageName: node linkType: hard "@smithy/hash-blob-browser@npm:^4.2.15": - version: 4.2.15 - resolution: "@smithy/hash-blob-browser@npm:4.2.15" + version: 4.3.0 + resolution: "@smithy/hash-blob-browser@npm:4.3.0" dependencies: - "@smithy/chunked-blob-reader": "npm:^5.2.2" - "@smithy/chunked-blob-reader-native": "npm:^4.2.3" - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/0f1ee2fd786306c604c4c08bb3b6ba00bf99fd2ba36743ca5d2695c8d37e02bb84435d5d55ebde6483a506ebcc20c42203750671cdd73618255499ef8cd0852b + checksum: 10c0/a5cfc9e1b46e63f89d54f90acdcebff350b67a179fa6299ab2b95d6486f1f6c5da84817ed4ea50eeb93fae3399f9d8d79c7fa91dc8546eb5901cf20ca5c6a8e2 languageName: node linkType: hard "@smithy/hash-node@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/hash-node@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/hash-node@npm:4.3.0" dependencies: - "@smithy/types": "npm:^4.14.1" - "@smithy/util-buffer-from": "npm:^4.2.2" - "@smithy/util-utf8": "npm:^4.2.2" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/c2cfc5dac4f2b996c4c5c503d3c26e52fcd0a647e71541182b24a48925801659828c9d378dad6857444b9d997c1655bdb23f6db5994ad8b7c814b2d0a09655b7 + checksum: 10c0/2b0618a249fb659d5ecb926a1dfa80c91af167e8f762c78704a5cc4150a722c41b0fb7ff8e5f506fcfc7798606fea0f73e80f62e428666c4d03d4be86a6c7ec6 languageName: node linkType: hard "@smithy/hash-stream-node@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/hash-stream-node@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/hash-stream-node@npm:4.3.0" dependencies: - "@smithy/types": "npm:^4.14.1" - "@smithy/util-utf8": "npm:^4.2.2" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/db14c0879527dfc0a184a4958e8f229e1e3f15d0e612ad4596ee92de8f9d672a5ead4a325e517f235129530a8d8472c2bb628d0620af6662abc65adfe71f573e + checksum: 10c0/11dbab69600f54618e95baaf510c462146a8ba08a79351b8328125f11b430bc1792c42c3502c6fe88c36fca4e0fda364dd1a0e265261a6f04e9ef2d95016e6a2 languageName: node linkType: hard "@smithy/invalid-dependency@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/invalid-dependency@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/invalid-dependency@npm:4.3.0" dependencies: - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/deba2c21232050de87e7893b86a27a8f080ace391a3cccf22d7aee183bc3b392247f3bb1a0e4f0b6ea6f928c18ba035fd2ed3af257178ba84f3564d47c635d16 + checksum: 10c0/26a9ffba4d89a05aa71837d4f869d39920de24765542699b980269bf2a4e296d7c02f97ca4c8a3fb8e939100d0209c9d41d5d05c4ff5ea3d0d8f4855437437ef languageName: node linkType: hard @@ -6726,204 +6662,155 @@ __metadata: linkType: hard "@smithy/is-array-buffer@npm:^4.2.2": - version: 4.2.2 - resolution: "@smithy/is-array-buffer@npm:4.2.2" + version: 4.3.0 + resolution: "@smithy/is-array-buffer@npm:4.3.0" dependencies: + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/ab5bf2cad0f3bc6c1d882e15de436b80fa1504739ab9facc3d7006003870855480a6b15367e516fd803b3859c298b1fcc9212c854374b7e756cda01180bab0a6 + checksum: 10c0/091afb4f7b240f30f3a169bc2ceec8d4f398d0a56a6125e1a5df540531f888bcff280d965069464ddc20fafb3bfe14d2f3a66703ae3980abf52ca9a31bdac8b6 languageName: node linkType: hard "@smithy/md5-js@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/md5-js@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/md5-js@npm:4.3.0" dependencies: - "@smithy/types": "npm:^4.14.1" - "@smithy/util-utf8": "npm:^4.2.2" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/d0bd6f08a3e37e83685fc1eba6621ee0f19576466ba23e7239dc201c97f83d46f6b9f7ddecc48ab74535b1f498d9facc43f18018f168dc26d2b5d0f9c72af761 + checksum: 10c0/24d33ac9a7b64af46c643602f79756221043e7b982f8db29ddccdf66ca917d0bd44f0a7b83423680eefea0d8a8bea8f11560e317337e92a4a498b7cb52f9a247 languageName: node linkType: hard "@smithy/middleware-content-length@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/middleware-content-length@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/middleware-content-length@npm:4.3.0" dependencies: - "@smithy/protocol-http": "npm:^5.3.14" - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/c26cc36504ad9a720e42f009bcc185b16e35ff64644bbec710a998c071d3366face29bb6fa68744adc7312356803666922459e416f3a7ae5c804841957832c3d + checksum: 10c0/a21bacf4ea019abdb9aee6d57df8be6f3943ca8f0208a1bb13ba9d77dae60f56391b0201028a5e89b75172c6b01f304ff22c6eb7cb99d62ff52e3c765fde6e51 languageName: node linkType: hard "@smithy/middleware-endpoint@npm:^4.4.32": - version: 4.4.32 - resolution: "@smithy/middleware-endpoint@npm:4.4.32" + version: 4.5.0 + resolution: "@smithy/middleware-endpoint@npm:4.5.0" dependencies: - "@smithy/core": "npm:^3.23.17" - "@smithy/middleware-serde": "npm:^4.2.20" - "@smithy/node-config-provider": "npm:^4.3.14" - "@smithy/shared-ini-file-loader": "npm:^4.4.9" - "@smithy/types": "npm:^4.14.1" - "@smithy/url-parser": "npm:^4.2.14" - "@smithy/util-middleware": "npm:^4.2.14" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/a77ba3f56956b872a533754c71f75d2a9d6df9af8d58a059d07caedd1af3ffdcae5b667a0b96b6d067555a777510f6fc5847f699a2dd705e9b5837d21510daa4 + checksum: 10c0/7e2dced85eda01cc4590a63d9c9f7c157a8a4bd6c08e6229a37cd68830e8ab0e902e9769fddf2f052bca640c70c911d9ea46300a9de3afbfd3b30f5ba21bec41 languageName: node linkType: hard "@smithy/middleware-retry@npm:^4.5.7": - version: 4.5.7 - resolution: "@smithy/middleware-retry@npm:4.5.7" + version: 4.6.0 + resolution: "@smithy/middleware-retry@npm:4.6.0" dependencies: - "@smithy/core": "npm:^3.23.17" - "@smithy/node-config-provider": "npm:^4.3.14" - "@smithy/protocol-http": "npm:^5.3.14" - "@smithy/service-error-classification": "npm:^4.3.1" - "@smithy/smithy-client": "npm:^4.12.13" - "@smithy/types": "npm:^4.14.1" - "@smithy/util-middleware": "npm:^4.2.14" - "@smithy/util-retry": "npm:^4.3.6" - "@smithy/uuid": "npm:^1.1.2" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/44ba622d961f83935aa13210fc92edfbf017f655ad4f4fb2024f7e880a70867d547b358be2089966689ed92c5deedcdf4723177c015babaf332ba3b55a40fe9b + checksum: 10c0/fb94a3643833802e7c54c3e125b381a3445f1f6550355d8283480f13097bfaa9cd79fc77d6cd58dd42a4173430e244a892efbe8118a3aaec3f6fca504c04a5c4 languageName: node linkType: hard "@smithy/middleware-serde@npm:^4.2.20": - version: 4.2.20 - resolution: "@smithy/middleware-serde@npm:4.2.20" + version: 4.3.0 + resolution: "@smithy/middleware-serde@npm:4.3.0" dependencies: - "@smithy/core": "npm:^3.23.17" - "@smithy/protocol-http": "npm:^5.3.14" - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/2db736d58c0d628a33febad86259eedd6d025135fc403458e4469a077a6adbb909587408acb62c982fa1b2d8b1376507da90661a4315a544c7d73a62179a3453 + checksum: 10c0/e493ac0119d833eb1095ed3fda81e64a68397f64fc978da1390f7fb46d2d027b1caf5278fb47c18c3779ba46a05f6187a6678b204789f8dd9df044b5abbaa410 languageName: node linkType: hard "@smithy/middleware-stack@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/middleware-stack@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/middleware-stack@npm:4.3.0" dependencies: - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/5a0323c50b399c4a2ff0d91b219c7c285b006f905f66b291b6013a4e94f14c036ff5a74c565989afc3c6f0fafc6c266bca6746b79e242403713b7d18dc266a92 + checksum: 10c0/af46e3955eea01fb16110c0b27984dc59eb927980b4b0273f11c7e77523500308fb3d0ad080e3befe5f0df1d4b3ca669dad800dc0cc3f1b7875bbf8a7742d980 languageName: node linkType: hard "@smithy/node-config-provider@npm:^4.3.14": - version: 4.3.14 - resolution: "@smithy/node-config-provider@npm:4.3.14" + version: 4.4.0 + resolution: "@smithy/node-config-provider@npm:4.4.0" dependencies: - "@smithy/property-provider": "npm:^4.2.14" - "@smithy/shared-ini-file-loader": "npm:^4.4.9" - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/59033a2fde5327d9ae1a0304a0eb2c3145141024cb4dcb58c5cd3c48371cd3a55d7228a3d2f33a5785b2d6a0a35475cbd0d1b2435d964c824683ac89a664e926 + checksum: 10c0/30842eedc192179cf7ceb93ae7d1fa28d2fe40417b713e1dcdec927256937d635f9c3167a4e9252f8c6c24450a361be1234e86b30d216477a09f2598bd1caf65 languageName: node linkType: hard "@smithy/node-http-handler@npm:^4.6.1": - version: 4.6.1 - resolution: "@smithy/node-http-handler@npm:4.6.1" + version: 4.7.0 + resolution: "@smithy/node-http-handler@npm:4.7.0" dependencies: - "@smithy/protocol-http": "npm:^5.3.14" - "@smithy/querystring-builder": "npm:^4.2.14" + "@smithy/core": "npm:^3.24.0" "@smithy/types": "npm:^4.14.1" tslib: "npm:^2.6.2" - checksum: 10c0/6c07fbfd8326cd5369283bd19c1af923ee49b7dcf42fdd199bb7132e4c25eaa6db8573e3b669fb60be0d8f9757a3f2482cd0cf6d6631494255f08239d3036ed2 + checksum: 10c0/bc0fc552675d0d74137b43cf6c7b1ec2a7f9a4d12e506b00aa1ab204d9e31fa5fe0904eb949c3a0be8d8209f8fd2ca658fe7b3561f979b3ed376e212c5122e10 languageName: node linkType: hard "@smithy/property-provider@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/property-provider@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/property-provider@npm:4.3.0" dependencies: - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/8d3da90e228b53885d1e82f28b33c095b72f3b1cb5dcd7f7edcd96292d90848e9cdfed85cef00040e198343e850acef61540e4de24bd10b07fbc8e1e8f4a1fdd + checksum: 10c0/2d4cc5fd3fd3279a99ff4cbb9d3eba673b83122231a4cb1826254023d7d0fc4cf6973c7c3c57561538c95093822a19bd94e5a55852a6a8e0ca29ae3921832f3a languageName: node linkType: hard "@smithy/protocol-http@npm:^5.3.14": - version: 5.3.14 - resolution: "@smithy/protocol-http@npm:5.3.14" + version: 5.4.0 + resolution: "@smithy/protocol-http@npm:5.4.0" dependencies: - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/ed7306e7e4b919fdacf60d1928f003ac4c4679cffd7472b078547fbed7b6cb9ba524f105b1871c2cd79222871169d3183257fd0a1a81c172fa7cf0a9abaf35f9 + checksum: 10c0/92b0a4d6e9fc58523e410c93aef9903e1a384fa54d794b4cab069e11df68366494204d248a9cc88070d7d73a4d8b562406cc48091fff1219edcc8348750a42ce languageName: node linkType: hard "@smithy/querystring-builder@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/querystring-builder@npm:4.2.14" - dependencies: - "@smithy/types": "npm:^4.14.1" - "@smithy/util-uri-escape": "npm:^4.2.2" - tslib: "npm:^2.6.2" - checksum: 10c0/ae2ceec55b4f32b73fe2eca710563b9dbe65c81157ed58c12c282bad6445998f1fe99d723591f9bac4ae6106d84213b57e960176865fb2dec78fc3bdf024b14b - languageName: node - linkType: hard - -"@smithy/querystring-parser@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/querystring-parser@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/querystring-builder@npm:4.3.0" dependencies: - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/245923618197bbae5eb38b72807368f397fafccee8e98cd98ee7d8961a34acb57b5176fa5fe8083b796229043f135ea775060f17e14aa12080a5d7cdfbf56333 - languageName: node - linkType: hard - -"@smithy/service-error-classification@npm:^4.3.1": - version: 4.3.1 - resolution: "@smithy/service-error-classification@npm:4.3.1" - dependencies: - "@smithy/types": "npm:^4.14.1" - checksum: 10c0/1bc927f53693035165f4b15232c644be579cdd432cf2d3e026faa746d26693e6b1e0f35bcd5d812dd89445695fd1eb7188e415e2523d6d8991e8db900cecdf36 + checksum: 10c0/2e44e85254cd1d911882a95438021cda42b296ae92ee5e8e458cef77662d794ecd8d0cd7f2f17a244bd2f4ac594bb488f3aa31cacad3ef37708edb9ed0a3fb08 languageName: node linkType: hard "@smithy/shared-ini-file-loader@npm:^4.4.9": - version: 4.4.9 - resolution: "@smithy/shared-ini-file-loader@npm:4.4.9" + version: 4.5.0 + resolution: "@smithy/shared-ini-file-loader@npm:4.5.0" dependencies: - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/9bf96ea80cedff027373c5547ffa53b35d272292b7d142a86211726343061953a34610f3dbd02153bb285c7c0f3802c5a25b4e27870e8068d777abdcaa9c1748 + checksum: 10c0/722c47bf11e99531d6bc1fa2f8c3179f6f0269d95a0b2d49a16e74eadb42c0c78f0bbef27f23665fde4d7760b8a75e1412d00502174ec501927324cff43f74e8 languageName: node linkType: hard "@smithy/signature-v4@npm:^5.3.14": - version: 5.3.14 - resolution: "@smithy/signature-v4@npm:5.3.14" + version: 5.4.0 + resolution: "@smithy/signature-v4@npm:5.4.0" dependencies: - "@smithy/is-array-buffer": "npm:^4.2.2" - "@smithy/protocol-http": "npm:^5.3.14" + "@smithy/core": "npm:^3.24.0" "@smithy/types": "npm:^4.14.1" - "@smithy/util-hex-encoding": "npm:^4.2.2" - "@smithy/util-middleware": "npm:^4.2.14" - "@smithy/util-uri-escape": "npm:^4.2.2" - "@smithy/util-utf8": "npm:^4.2.2" tslib: "npm:^2.6.2" - checksum: 10c0/ccc788992e281a681984e079b7194a219af40cf4f7087988eba69c4947264b9a83ac00a806164b9074c7e2f0697e492fa2b377b56b783316d247be02aaccbdb3 + checksum: 10c0/55763bf28e0a5f6b75b4598713db8abfa3b2588c71d1ca4518fa58321e2f91ae4193fa5a1e2cbe5e0d1c0bfc05d37f9da8b11528af186b27078f3910d5c11ab2 languageName: node linkType: hard "@smithy/smithy-client@npm:^4.12.13": - version: 4.12.13 - resolution: "@smithy/smithy-client@npm:4.12.13" + version: 4.13.0 + resolution: "@smithy/smithy-client@npm:4.13.0" dependencies: - "@smithy/core": "npm:^3.23.17" - "@smithy/middleware-endpoint": "npm:^4.4.32" - "@smithy/middleware-stack": "npm:^4.2.14" - "@smithy/protocol-http": "npm:^5.3.14" + "@smithy/core": "npm:^3.24.0" "@smithy/types": "npm:^4.14.1" - "@smithy/util-stream": "npm:^4.5.25" tslib: "npm:^2.6.2" - checksum: 10c0/0680d4d0720d110a637fabb8bdc85175e1471191925f5c55f08636f5327de1bc29c8db75318aac1396491fa83c8a319dd4cd26c5e9fff619207c9a96b9dbd9fe + checksum: 10c0/2f9aadfcfea48d3892a9ccac66bfb5b335231376d0c9f99d0fa0628ca126a3c6d9aa375db0ce0d28f347a15be49d74c75a8991c097664ebefa630fdd77a20c77 languageName: node linkType: hard @@ -6937,42 +6824,42 @@ __metadata: linkType: hard "@smithy/url-parser@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/url-parser@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/url-parser@npm:4.3.0" dependencies: - "@smithy/querystring-parser": "npm:^4.2.14" - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/360463fe1fb51b8a1c5b80f4a16df993ee4e87221e60ce51c428b0737d6f1325434ec572bb7afca7d65bb9a09c750f307822a23e42be675706ee71fedd7c6c06 + checksum: 10c0/41c0a784fb4a7cb2d5ee371900ea543e28d6b144af382c6f5d69cef24f042d67ec9a713a8551ac81c38664f0767fe7feebaf2520121a4ffeca1d0741c80d88cf languageName: node linkType: hard "@smithy/util-base64@npm:^4.3.2": - version: 4.3.2 - resolution: "@smithy/util-base64@npm:4.3.2" + version: 4.4.0 + resolution: "@smithy/util-base64@npm:4.4.0" dependencies: - "@smithy/util-buffer-from": "npm:^4.2.2" - "@smithy/util-utf8": "npm:^4.2.2" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/acc08ff0b482ef4473289be655e0adc21c33555a837bbbc1cc7121d70e3ad595807bcaaec7456d92e93d83c2e8773729d42f78d716ac7d91552845b50cd87d89 + checksum: 10c0/b22850ad39a38b8081a30ea33bee84c384d41eb6424f5b3068f8d05deed5cc8f4f645fe9644867b0a16cd7378cde4a6f01352b74481a127b9e629c1455ca31a7 languageName: node linkType: hard "@smithy/util-body-length-browser@npm:^4.2.2": - version: 4.2.2 - resolution: "@smithy/util-body-length-browser@npm:4.2.2" + version: 4.3.0 + resolution: "@smithy/util-body-length-browser@npm:4.3.0" dependencies: + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/4444039b995068eeda3dd0b143eb22cf86c7ef7632a590559dad12b0e681a728a7d82f8ed4f4019cdc09a72e4b5f14281262b64db75514dbcc08d170d9e8f1db + checksum: 10c0/88f0c18db8d8574ce955b697c5335d587680ee30673c01aae7fa96c2c87d09257d967ca734fdf5cea5788a8b4ef9786810dec5c69722a29407877ebeb96846d4 languageName: node linkType: hard "@smithy/util-body-length-node@npm:^4.2.3": - version: 4.2.3 - resolution: "@smithy/util-body-length-node@npm:4.2.3" + version: 4.3.0 + resolution: "@smithy/util-body-length-node@npm:4.3.0" dependencies: + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/5345d75e8c3e0a726ed6e2fe604dfe97b0bcc37e940b30b045e3e116fced9555d8a9fa684d9f898111773eeef548bcb5f0bb03ee67c206ee498064842d6173b5 + checksum: 10c0/47056e78a72d7d16b681489b9404e45deeb75fa86b684eee1bc0735cad8ee5487b25e7438eb7d3b28df597e98400ef4878e39274fd606d1765ffa5591ca46c5f languageName: node linkType: hard @@ -6986,115 +6873,83 @@ __metadata: languageName: node linkType: hard -"@smithy/util-buffer-from@npm:^4.2.2": - version: 4.2.2 - resolution: "@smithy/util-buffer-from@npm:4.2.2" - dependencies: - "@smithy/is-array-buffer": "npm:^4.2.2" - tslib: "npm:^2.6.2" - checksum: 10c0/d9acea42ee035e494da0373de43a25fa14f81d11e3605a2c6c5f56efef9a4f901289ec2ba343ebb3ad32ae4e0cfe517e8b6b3449a4297d1c060889c83cd1c94f - languageName: node - linkType: hard - "@smithy/util-config-provider@npm:^4.2.2": - version: 4.2.2 - resolution: "@smithy/util-config-provider@npm:4.2.2" + version: 4.3.0 + resolution: "@smithy/util-config-provider@npm:4.3.0" dependencies: + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/cfd3350607ec00b6294724033aa3e469f8d9d258a7a70772e67d80c301f2eae62b17850ea0c8d8a20208b3f4f1ea5aa0019f45545a6c0577a94a47a05c81d8e8 + checksum: 10c0/7a88a7dedd3a68564c0aec73bdc1e1a7b385fd3aab61188a997294435f7514ff49b4710c01dd515474fe7ffadb1b0d444319d9144c6023760fd935dfe3df0321 languageName: node linkType: hard "@smithy/util-defaults-mode-browser@npm:^4.3.49": - version: 4.3.49 - resolution: "@smithy/util-defaults-mode-browser@npm:4.3.49" + version: 4.4.0 + resolution: "@smithy/util-defaults-mode-browser@npm:4.4.0" dependencies: - "@smithy/property-provider": "npm:^4.2.14" - "@smithy/smithy-client": "npm:^4.12.13" - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/0979b09f1108aa41f5bf623e32fa138e129fbffe3adc23c46308afa310b925635428f9d7e36e3e2a312cafe9af325cb5f01667791ee672418d4f302c1961623b + checksum: 10c0/d85bae5eeeda85db720426490a079802f9432dd06b4bf8b0475deca1b3d8c724e8eee947fab9d6aa2a048a321a7ed393405c9bab7a992bf86b603f473de718da languageName: node linkType: hard "@smithy/util-defaults-mode-node@npm:^4.2.54": - version: 4.2.54 - resolution: "@smithy/util-defaults-mode-node@npm:4.2.54" + version: 4.3.0 + resolution: "@smithy/util-defaults-mode-node@npm:4.3.0" dependencies: - "@smithy/config-resolver": "npm:^4.4.17" - "@smithy/credential-provider-imds": "npm:^4.2.14" - "@smithy/node-config-provider": "npm:^4.3.14" - "@smithy/property-provider": "npm:^4.2.14" - "@smithy/smithy-client": "npm:^4.12.13" - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/7146087fa1d7758b37da456719d589f63300843ff5610891de02a0434c5abbd49fd257712c2d9e0bede904f4ec62c9d3c9eddcd6ec0372134f998e4f9c0bce09 + checksum: 10c0/a22cd4a14e79a9eb8ac30d59a78f165977ac755adc7851cec084efdafc29918af1075fb5c7398577871d26bdab4c53650182fc887eb48e07b7b0da896d7efa64 languageName: node linkType: hard "@smithy/util-endpoints@npm:^3.4.2": - version: 3.4.2 - resolution: "@smithy/util-endpoints@npm:3.4.2" + version: 3.5.0 + resolution: "@smithy/util-endpoints@npm:3.5.0" dependencies: - "@smithy/node-config-provider": "npm:^4.3.14" - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/602e05c8dc43d8902604bac74b717d09da5b4f1994dcdd5025bffeced6002eaa0612ae1ccbe7a0e636a3d92a60747715e22cbacf8feeb672394d15b6ae211b13 + checksum: 10c0/b1598ad5424b4648be5df8f3ddea99c02b301cac56445ffd7fff66dca7f3a851578c86c9bb0b3e3332589103fc823a7f0a14157683de19b02d76fd299241cb04 languageName: node linkType: hard "@smithy/util-hex-encoding@npm:^4.2.2": - version: 4.2.2 - resolution: "@smithy/util-hex-encoding@npm:4.2.2" + version: 4.3.0 + resolution: "@smithy/util-hex-encoding@npm:4.3.0" dependencies: + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/b2f2bca85475cd599b998e169b7026db40edc2a0a338ad7988b9c94d9f313c5f7e08451aced4f8e62dbeaa54e15d1300d76c572b83ffa36f9f8ca22b6fc84bd7 + checksum: 10c0/8c3846ff3d3d801f1bcb2f371dcd37c639891b7421ca77d829115358b58fcabf4a453080386a7bb06f5bcb87758ef9dd9109900bdc7cee599fd72dbf1f78b3bd languageName: node linkType: hard "@smithy/util-middleware@npm:^4.2.14": - version: 4.2.14 - resolution: "@smithy/util-middleware@npm:4.2.14" + version: 4.3.0 + resolution: "@smithy/util-middleware@npm:4.3.0" dependencies: - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/6ba5026b70ad7b58fac89d7428c0b1679be2a97a8fb47811a39e0183901a3c6ca8732ee225ddfda28e7b86223d49983ff709421905da571bc00b82c660e4fc27 + checksum: 10c0/9fbd24674413a275768c323cb03f83f82c24a127026c31c58e1448b61ed9179a0473279f63c73a158b424ae11b0c268892b2a0546db7ce290d2d3c40af56a6eb languageName: node linkType: hard "@smithy/util-retry@npm:^4.3.6": - version: 4.3.8 - resolution: "@smithy/util-retry@npm:4.3.8" + version: 4.4.0 + resolution: "@smithy/util-retry@npm:4.4.0" dependencies: - "@smithy/service-error-classification": "npm:^4.3.1" - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/23432bb592baa2d9c89741d8e3603e0d6d2e7ca0764d4a7ab3eda8f5411540bb8b8fa7b8e959c8eb24cb496d9dc37c71e8e9130f92c9d34e3dc9335a893ae59e + checksum: 10c0/c4e3a31331234f55f7473ad6c7165ff6f6a456b53f3a641e934bd8fa4591cf0fc2bc6a1c14a0ba44a5a9bb4565e7de61eccb2e5b5fb10dcf6dd61bdf71e2e8a8 languageName: node linkType: hard -"@smithy/util-stream@npm:^4.5.25": - version: 4.5.25 - resolution: "@smithy/util-stream@npm:4.5.25" +"@smithy/util-stream@npm:^4.5.25, @smithy/util-stream@npm:^4.6.0": + version: 4.6.0 + resolution: "@smithy/util-stream@npm:4.6.0" dependencies: - "@smithy/fetch-http-handler": "npm:^5.3.17" - "@smithy/node-http-handler": "npm:^4.6.1" - "@smithy/types": "npm:^4.14.1" - "@smithy/util-base64": "npm:^4.3.2" - "@smithy/util-buffer-from": "npm:^4.2.2" - "@smithy/util-hex-encoding": "npm:^4.2.2" - "@smithy/util-utf8": "npm:^4.2.2" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/4b222c975eca2018831f1ab4067f122f4ef5e68f3abb71776003309f1a27f67d509a2d86f5f1f81a91656236db0e29c38314c005b17dbf63f053de4d97be7b59 - languageName: node - linkType: hard - -"@smithy/util-uri-escape@npm:^4.2.2": - version: 4.2.2 - resolution: "@smithy/util-uri-escape@npm:4.2.2" - dependencies: - tslib: "npm:^2.6.2" - checksum: 10c0/33b6546086c975278d16b5029e6555df551b4bd1e3a84042544d1ef956a287fe033b317954b1737b2773e82b6f27ebde542956ff79ef0e8a813dc0dbf9d34a58 + checksum: 10c0/283ae66729690496a191bc92726f8423acbad6773622fa93293d14263d90845b32a9bff68369b08fe35a5875a06d9c8828621a125126738bc59a2834d6fd4f4e languageName: node linkType: hard @@ -7109,31 +6964,22 @@ __metadata: linkType: hard "@smithy/util-utf8@npm:^4.2.2": - version: 4.2.2 - resolution: "@smithy/util-utf8@npm:4.2.2" - dependencies: - "@smithy/util-buffer-from": "npm:^4.2.2" - tslib: "npm:^2.6.2" - checksum: 10c0/55b5119873237519a9175491c74fd0a14acd4f9c54c7eec9ae547de6c554098912d46572edb12d5b52a0b9675c0577e2e63d1f7cb8e022ca342f5bf80b56a466 - languageName: node - linkType: hard - -"@smithy/util-waiter@npm:^4.3.0": version: 4.3.0 - resolution: "@smithy/util-waiter@npm:4.3.0" + resolution: "@smithy/util-utf8@npm:4.3.0" dependencies: - "@smithy/types": "npm:^4.14.1" + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/44e18c946c731740801787714a513e6c119f9997f29191d3bababe9781e0935d08a927caa475d70f7dcb02d70b532639be94317ac3c6305496f1ebe90fcfcf9b + checksum: 10c0/80df3505b2e3e2d974ab2496dc3ad3a512cb0129dce0ab59cd4d44d89b69aba82a8a9188dc4614e528ed385c49de10a32bf856a779761e1b50a845e994e2beaf languageName: node linkType: hard -"@smithy/uuid@npm:^1.1.2": - version: 1.1.2 - resolution: "@smithy/uuid@npm:1.1.2" +"@smithy/util-waiter@npm:^4.3.0": + version: 4.4.0 + resolution: "@smithy/util-waiter@npm:4.4.0" dependencies: + "@smithy/core": "npm:^3.24.0" tslib: "npm:^2.6.2" - checksum: 10c0/cbedfe5e2c1ec5ee05ae0cd6cc3c9f6f5e600207362d62470278827488794e19148a05a61ee9b6a2359bb460985af1a528c48d54f365891fe1c4913504250667 + checksum: 10c0/f04d481d880ffaf18014ccaae6e776c0f39b0bd85f74620b5059d73183e81296ded8c8466033f1fb10d28f70b21f00edff8f25b270da06a5eecb9ac2a45a513a languageName: node linkType: hard @@ -8085,10 +7931,10 @@ __metadata: "@aws-sdk/credential-providers": "npm:^3.1045.0" "@aws-sdk/lib-dynamodb": "npm:^3.1045.0" "@faker-js/faker": "npm:^10.4.0" - "@inquirer/core": "npm:^11.1.9" - "@inquirer/prompts": "npm:^8.4.2" + "@inquirer/core": "npm:^11.1.10" + "@inquirer/prompts": "npm:^8.4.3" "@opensearch-project/opensearch": "npm:^3.6.0" - "@smithy/util-stream": "npm:^4.5.25" + "@smithy/util-stream": "npm:^4.6.0" "@types/jsdom": "npm:^28.0.1" "@types/node": "npm:^24.12.3" "@types/yargs": "npm:^17.0.35" @@ -8855,11 +8701,11 @@ __metadata: linkType: hard "baseline-browser-mapping@npm:^2.10.12": - version: 2.10.27 - resolution: "baseline-browser-mapping@npm:2.10.27" + version: 2.10.29 + resolution: "baseline-browser-mapping@npm:2.10.29" bin: baseline-browser-mapping: dist/cli.cjs - checksum: 10c0/8ebadaeeddc99367a1522661476d6c895b452a96023c6cef0576aecf8f5f6d37c91e6bde8fa8904cc11a32c5c6bc9df2030481504e50a5aed17e6c7f8bac3b5a + checksum: 10c0/d629a6d87f1aa0585b85ec8bf686bf58d706836fe7827d7bd11f646db2292f2089d8dff33fdd9a1ffe4c07424ae5b08743f57d9405d45cd8ceeb145e8dfb58e5 languageName: node linkType: hard @@ -9999,9 +9845,9 @@ __metadata: linkType: hard "electron-to-chromium@npm:^1.5.328": - version: 1.5.352 - resolution: "electron-to-chromium@npm:1.5.352" - checksum: 10c0/d1672a327420ea0c5dffbd604c448e5bacf9238953f7ce464f3d31c98309bb5ebe82443fb2425de0a78db3ef89261356ee59967dbac866e16262b8bbc0a03209 + version: 1.5.353 + resolution: "electron-to-chromium@npm:1.5.353" + checksum: 10c0/a5481023e4056d8773b5ccd646906123d82f2821466b26b521b96c9645d0714c5be6a5af792ec8477b904a5fcfe3794bfc45d2acbf8b306f4693842fb85a7cd4 languageName: node linkType: hard @@ -13706,8 +13552,8 @@ __metadata: linkType: hard "protobufjs@npm:^7.3.0, protobufjs@npm:^7.5.5": - version: 7.5.6 - resolution: "protobufjs@npm:7.5.6" + version: 7.5.7 + resolution: "protobufjs@npm:7.5.7" dependencies: "@protobufjs/aspromise": "npm:^1.1.2" "@protobufjs/base64": "npm:^1.1.2" @@ -13721,7 +13567,7 @@ __metadata: "@protobufjs/utf8": "npm:^1.1.1" "@types/node": "npm:>=13.7.0" long: "npm:^5.0.0" - checksum: 10c0/220df6c3cf6d2346748639a9b0b688fecc994bff9fee7018a93167e8cd45ab0ee3b4270d9eaa6be33a11adb46514ef9dce7e8217fd578c36726a9e70b96327cd + checksum: 10c0/432b30edf06c689ca591372812b57a7cda3d5325311b69369e3bb15afef80ccc5dd021cc7e92d72e55e8e0f5665abf87c32e26601319ab9576707f846777842e languageName: node linkType: hard @@ -13892,6 +13738,13 @@ __metadata: languageName: node linkType: hard +"real-require@npm:^1.0.0": + version: 1.0.0 + resolution: "real-require@npm:1.0.0" + checksum: 10c0/74d44b360b8b662d29c646f4688da462c4b10243b11d22ff9635395e81951a9c1ed349393411c4911ca6f3474c3e10e812bf4767052706abb863ba59bde0813b + languageName: node + linkType: hard + "reduce-configs@npm:^1.1.1, reduce-configs@npm:^1.1.2": version: 1.1.2 resolution: "reduce-configs@npm:1.1.2" @@ -14512,7 +14365,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:7.7.4, semver@npm:^7.1.1, semver@npm:^7.3.5, semver@npm:^7.3.7, semver@npm:^7.5.2, semver@npm:^7.5.3, semver@npm:^7.5.4, semver@npm:^7.6.0, semver@npm:^7.6.2, semver@npm:^7.6.3, semver@npm:^7.7.3": +"semver@npm:7.7.4": version: 7.7.4 resolution: "semver@npm:7.7.4" bin: @@ -14530,6 +14383,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:^7.1.1, semver@npm:^7.3.5, semver@npm:^7.3.7, semver@npm:^7.5.2, semver@npm:^7.5.3, semver@npm:^7.5.4, semver@npm:^7.6.0, semver@npm:^7.6.2, semver@npm:^7.6.3, semver@npm:^7.7.3": + version: 7.8.0 + resolution: "semver@npm:7.8.0" + bin: + semver: bin/semver.js + checksum: 10c0/8f096ca9b80ffd47b308d03f9ce8c873e27e2983f36023c559cdc92c51e8433fc23ebbfe57ec9623fc155636a6961ee989501099841ae4bb1babc8d2b3f048cd + languageName: node + linkType: hard + "serialize-error@npm:13.0.1": version: 13.0.1 resolution: "serialize-error@npm:13.0.1" @@ -14820,12 +14682,12 @@ __metadata: linkType: hard "socks@npm:^2.8.3, socks@npm:^2.8.6": - version: 2.8.8 - resolution: "socks@npm:2.8.8" + version: 2.8.9 + resolution: "socks@npm:2.8.9" dependencies: ip-address: "npm:^10.1.1" smart-buffer: "npm:^4.2.0" - checksum: 10c0/4777edf8b554182ddf472524a1855c0fc684c7a3778466851dca687ae81cfa19ea9261856765789e51f7cec12e575e2652d5bd69d98fdbbbc947eabd12e9891d + checksum: 10c0/2d4350c31142b0931eb1758825b426bcbf4bfb5eed682ca48bc46dc9e7d1930ec366ea574ad49fc6c1fd9e9e17ce243be0ef13e31fc4b0319d9093f1fb19743c languageName: node linkType: hard @@ -15241,11 +15103,11 @@ __metadata: linkType: hard "thread-stream@npm:^4.0.0": - version: 4.0.0 - resolution: "thread-stream@npm:4.0.0" + version: 4.1.0 + resolution: "thread-stream@npm:4.1.0" dependencies: - real-require: "npm:^0.2.0" - checksum: 10c0/f0a47a673af574062df20140ec3e857d679365253fcaa98a76c167c9a053ee03291f4b25bd89b078c7f6a48f07f49d5a49e4f5598bb1c8a263ec15955a018fbd + real-require: "npm:^1.0.0" + checksum: 10c0/9ffe41f4befdf11a965b80b288bada63926a3de90f05c2b55ed30660da7ce150c8de80cc69a239832b90c83fbdac762457cbe0f4457a28dc5c15c9f70866de43 languageName: node linkType: hard