diff --git a/.changeset/adr-0104-deployment-migration-gate.md b/.changeset/adr-0104-deployment-migration-gate.md new file mode 100644 index 0000000000..0164d7fe11 --- /dev/null +++ b/.changeset/adr-0104-deployment-migration-gate.md @@ -0,0 +1,58 @@ +--- +"@objectstack/spec": minor +"@objectstack/platform-objects": minor +"@objectstack/service-storage": minor +"@objectstack/objectql": minor +"@objectstack/cli": minor +--- + +feat(migrate): `os migrate files-to-references` — a data migration with a self-check, gated per deployment (#3617) + +The ADR-0104 file-as-reference migration ships as a command a deployment runs +against its own database, and the deployment-level flag it records is what may +later authorise irreversible behaviour — never the platform version. + +```bash +os migrate files-to-references # dry run: reports, writes nothing +os migrate files-to-references --apply # converts, verifies, records the flag +``` + +The run backfills legacy file-field values (inline metadata blobs, own-resolver +URLs, `data:` URIs) into owned `sys_file` references, reconciles the ownership +ledger against what records actually hold, and — only on an `--apply` run whose +reconciliation reports **zero blocking discrepancies** — records +`sys_migration { id: 'adr-0104-file-references', verified_at, blocking: 0 }`. + +**Why a flag rather than a release note.** ObjectStack is a development +platform: third-party deployments upgrade on their own schedule and their data +is not observable by anyone else, so no release-side soak can vouch for them. +The evidence has to be produced where the data is. Consequences: + +- Installing a new version never starts deleting bytes. Running the migration + and passing its self-check is the consent. +- Not run, or not passed → files are retained forever. Wasted storage, zero + data loss. +- A later failing run **clears** `verified_at`: a deployment whose data has + drifted closes its own gate. +- A dry run writes nothing at all — not the conversions, and not the flag, + even when the self-check would pass. +- External URLs stay advisory. They are not `sys_file`s, so they can never + enter collection; whether to remodel them as a `url` field is the app + author's decision (ADR-0104 R7), not a gate. + +Ships alongside: + +- `@objectstack/spec` — `DataMigrationFlagSchema`, `FILE_REFERENCES_MIGRATION_ID`, + and the single `isDataMigrationFlagVerified` predicate both future consumers + (collection #3459, strict value-shape #3438) read, so the two gates cannot + disagree about the same fact. +- `@objectstack/platform-objects` — the `sys_migration` object plus + `readDataMigrationFlag` / `isDataMigrationVerified` / `recordDataMigrationRun`. + Reads fail toward "not verified": a gate that cannot read its evidence stays + closed. +- `@objectstack/objectql` — a read may now opt out of file-reference expansion + via the spec's `RAW_FILE_VALUES_CONTEXT_KEY`, and the storage service's + bookkeeping/scan reads do. Without it the read resolver rewrites stored ids to + their expanded form before the reconciliation sees them, which reports held + references as absent — noisy `stale_owner` findings, and a missed + `unowned_reference` would have been a false pass of the collection gate. diff --git a/content/docs/deployment/cli.mdx b/content/docs/deployment/cli.mdx index d924dfde3b..0cb1d75251 100644 --- a/content/docs/deployment/cli.mdx +++ b/content/docs/deployment/cli.mdx @@ -517,6 +517,48 @@ first. It never drops a table that is absent from your metadata, and on SQLite it reconciles via a table rebuild (copy → swap) that preserves your data. +#### Data migrations + +The commands above reconcile **schema**. A *data* migration rewrites rows, and +whether it is done is a fact about **your** database, not about the platform +version you installed — so each deployment runs it, and its result is recorded +where the data lives. + +| Command | Description | +|---------|-------------| +| `os migrate files-to-references` | Convert legacy file-field values to `sys_file` references, verify the ownership ledger, and record the deployment's migration flag | + +```bash +os migrate files-to-references # Dry run: full report, writes nothing +os migrate files-to-references --apply # Convert, verify, record the flag (prompts) +os migrate files-to-references --apply --yes --json # CI / scripts +os migrate files-to-references --object product # Restrict to one object (repeatable) +``` + +A `file` / `image` / `avatar` / `video` / `audio` field value is an opaque +`sys_file` id that the platform owns. Values written before that (an inline +`{url, name, …}` blob, or a URL naming this platform's own +`…/storage/files/:id` resolver) are converted in place; **external URLs are +reported, never re-hosted** — re-hosting third-party content is a licensing and +privacy decision, and the right fix is usually to model the field as a `url` +field instead. + +The run then reconciles what records actually hold against what `sys_file` +records as each file's owner. Zero blocking discrepancies is what records the +flag — and that flag, not the version number, is what later enables features +that act irreversibly on migrated data (reclaiming the bytes of released files; +rejecting rather than warning about legacy media values). Never running it is +safe: files are simply retained forever. + +Exit status is `0` only when the self-check passes, so CI can gate on it. + + +A dry run writes **nothing** — not the conversions, and not the flag either, +even when the self-check would pass. `--apply` is the only writing mode. A +later run that *fails* its self-check clears the flag's verified state, so a +database that has drifted closes its own gate. + + ### Scaffolding | Command | Alias | Description | diff --git a/content/docs/references/system/migration.mdx b/content/docs/references/system/migration.mdx index b33ea9ca8a..62c994e656 100644 --- a/content/docs/references/system/migration.mdx +++ b/content/docs/references/system/migration.mdx @@ -5,6 +5,28 @@ description: Migration protocol schemas {/* ⚠️ AUTO-GENERATED — DO NOT EDIT. Run build-docs.ts to regenerate. Hand-written docs live in the module folders under content/docs/. */} +Migration protocol — the two kinds of migration, kept apart on purpose. + +A **schema migration** (`ChangeSet` + its atomic operations) reshapes the + +physical database to match metadata: add a field, change a type, create an + +object, run SQL. It is derivable from the metadata and applied by + +`os migrate plan` / `os migrate apply`. + +A **data migration** rewrites the rows themselves, and whether it is done is + +a property of ONE DEPLOYMENT's database rather than of the installed code + +version — so it cannot be expressed as a ChangeSet, and its completion cannot + +be inferred from a release. `DataMigrationFlag` is the per-deployment record + +that one ran here and its self-check passed; consumers that would act + +irreversibly on migrated data gate on the flag instead of the version. + **Source:** `packages/spec/src/system/migration.zod.ts` @@ -12,8 +34,8 @@ description: Migration protocol schemas ## TypeScript Usage ```typescript -import { AddFieldOperation, ChangeSet, CreateObjectOperation, DeleteObjectOperation, ExecuteSqlOperation, MigrationDependency, MigrationOperation, ModifyFieldOperation, RemoveFieldOperation, RenameObjectOperation } from '@objectstack/spec/system'; -import type { AddFieldOperation, ChangeSet, CreateObjectOperation, DeleteObjectOperation, ExecuteSqlOperation, MigrationDependency, MigrationOperation, ModifyFieldOperation, RemoveFieldOperation, RenameObjectOperation } from '@objectstack/spec/system'; +import { AddFieldOperation, ChangeSet, CreateObjectOperation, DataMigrationFlag, DeleteObjectOperation, ExecuteSqlOperation, MigrationDependency, MigrationOperation, ModifyFieldOperation, RemoveFieldOperation, RenameObjectOperation } from '@objectstack/spec/system'; +import type { AddFieldOperation, ChangeSet, CreateObjectOperation, DataMigrationFlag, DeleteObjectOperation, ExecuteSqlOperation, MigrationDependency, MigrationOperation, ModifyFieldOperation, RemoveFieldOperation, RenameObjectOperation } from '@objectstack/spec/system'; // Validate data const result = AddFieldOperation.parse(data); @@ -69,6 +91,25 @@ Create a new object | **object** | `{ name: string; label?: string; pluralLabel?: string; description?: string; … }` | ✅ | Full object definition to create | +--- + +## DataMigrationFlag + +Deployment-level record that a data migration ran here and its self-check passed — the evidence gate consumers read instead of the platform version + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **id** | `string` | ✅ | Migration id (e.g. adr-0104-file-references) — one row per data migration | +| **last_run_at** | `string` | ✅ | When this migration last completed a gated (apply-mode) run on this deployment | +| **verified_at** | `string \| null` | optional | When the self-check last PASSED. Null/absent until it does — and cleared again by a later failing run, so a regression closes the gate | +| **applied_at** | `string \| null` | optional | When the backfill last ran in apply mode (writes enabled) | +| **blocking** | `integer` | ✅ | Blocking discrepancies reported by the last self-check. The gate requires 0 | +| **advisory** | `integer` | optional | Advisory findings from the last run (external URLs, stale owners, …) — cost storage or need a modelling decision, never block the gate | +| **details** | `string` | optional | JSON-encoded counts from the last run, for diagnostics | + + --- ## DeleteObjectOperation diff --git a/docs/adr/0104-field-runtime-value-shape-contract.md b/docs/adr/0104-field-runtime-value-shape-contract.md index 46495ce473..3f6f3ea1f9 100644 --- a/docs/adr/0104-field-runtime-value-shape-contract.md +++ b/docs/adr/0104-field-runtime-value-shape-contract.md @@ -591,10 +591,13 @@ legacy values would reject any update that rewrites such a field, breaking working apps until the backfill catches up. Backfilling first leaves the cutover nothing legacy to reject. -R4's acceptance gate is now executable rather than aspirational: step 6 may not -merge until step 5 reports **zero blocking discrepancies for ≥7 consecutive -days** on real tenant data. R5 (public-posture inventory) and R6 (sub-key read -scan) stand. +R4's acceptance gate is now executable rather than aspirational: step 5's +reconciliation must report **zero blocking discrepancies** before step 6 may +act. (The original wording — "for ≥7 consecutive days on real tenant data" — +assumed a tenant we operate and observe; see the 2026-07-27 platform-form +addendum below, which relocates the same evidence requirement to the one place +that can satisfy it.) R5 (public-posture inventory) and R6 (sub-key read scan) +stand. Steps 4 and 6 are the two that can break something. Step 4 is a declared protocol major and breaks *loudly* — a rejected write, recoverable. Step 6 @@ -604,15 +607,86 @@ adoption that consumes the new form, while step 6 additionally requires the reconciliation evidence above, and neither the enable nor the migration run should be performed without an explicit human decision. +## Addendum (2026-07-27, later) — the evidence gate is per deployment, not per release + +The gate above was written as an operations runbook: *run the backfill on the +tenant, watch `verifyFileReferences` stay clean for a week, then flip the +switch.* Every clause of that sentence assumes **we** run the tenant, **we** +observe it, and **we** flip. + +ObjectStack is a development platform. Third-party developers build metadata +apps on it and ship versions onward; each deployment decides for itself when to +upgrade, and its data is not visible to us — nor should it be. **There is no +observation window of ours that anyone else's data can wait inside.** A gate +phrased as one is not strict, merely unlocatable: it can be satisfied by no +deployment at all, including the deployments whose bytes are at stake. + +### What survives, and what moves + +R4's substance is unchanged: **a ledger may not be given the power to delete +bytes until it has been shown to agree with reality.** What changes is *who +produces the evidence* — from us, once, to each deployment, for itself. + +That rules out a one-shot script (a script whose output nobody must read is not +a gate) and rules in a **data-migration step with a self-check, whose passing +result is recorded on the deployment**: + +``` +os migrate files-to-references + 1. backfill (dry run by default; --apply writes) + 2. verifyFileReferences (reconcile the ledger against what records hold) + 3. zero blocking findings → record sys_migration { id: 'adr-0104-file-references', + verified_at, blocking: 0 } + 4. collection + strict enforcement read THAT ROW, never the version number +``` + +### The properties this buys + +- **Upgrading does not start deleting.** Installing a new version is not + consent; running the migration and passing its self-check is. +- **A third-party developer need not understand R4.** They run one command. + Green means done; red names the record holding a file nobody owns. +- **Not run, or not passed → files live forever.** Wasted storage, zero data + loss: "fail toward retention" holds on the *deployment* axis too. The same + rule applies to a regression — a later failing run clears `verified_at`, so a + deployment whose data has drifted closes its own gate. +- **Each deployment gets its own 30-day tombstone window.** Nobody waits inside + anyone else's soak. The irreversible moment is day 30 after *that* deployment + enabled collection, not release day. +- **A dry run changes nothing** — not the data, and not the flag either, even + when the self-check would pass. `--apply` is the only writing mode, so + "did this run change my posture" never depends on what the run found. + +### D1/D2 carry the identical error, and the identical fix + +The strict-by-default flip for value-shape and action-param validation (#3438) +was scheduled as "flip in a later minor once telemetry is quiet" — again +*our* telemetry, deciding for *their* deployments. A deployment that has not +finished backfilling would have every media-field update rejected the moment it +upgraded: a working app broken by a decision made on its behalf. + +Both therefore read the **same** flag. Strict enforcement becomes effective for +a deployment when that deployment has completed and verified its migration — +not when a version number arrives. One flag, not two gates that can disagree: +they are gating on the same fact. + +### What cannot be automated, and does not block + +Whether an **external URL** (`https://cdn…`) should become an explicit `url` +field is a modelling decision only the app's author can make (R7). It also +cannot block: an external URL is not a `sys_file` and can never enter +collection. It is reported as advisory, never as a gate failure. + ### Why this stays inside ADR-0104 rather than a new ADR D3 is already this ADR's third phase; the two-wave split and the enforcement-point principle are a **refinement of the D4 rollout**, not a new decision, so they live here. Wave 2's migration mechanics (dual-read window, `os migrate` backfill, the R4/R5/R6 gates) are specified in §D3 above and need -no separate record. Wave 2's implementation did surface one genuinely new -decision — the exclusive-ownership model — and it is recorded as the 2026-07-27 -addendum above rather than a separate ADR, because it settles *how* D3's -already-accepted "field values point into `sys_file`" behaves rather than -revisiting whether to do it. A future choice that stands on its own (say, a -chunked-migration protocol, or byte-layer content dedup) would earn its own ADR. +no separate record. Wave 2's implementation did surface two genuinely new +decisions — the exclusive-ownership model, and relocating the evidence gate to +the deployment — and both are recorded as 2026-07-27 addenda above rather than +separate ADRs, because each settles *how* D3's already-accepted "field values +point into `sys_file`" reaches production rather than revisiting whether to do +it. A future choice that stands on its own (say, a chunked-migration protocol, +or byte-layer content dedup) would earn its own ADR. diff --git a/packages/cli/src/commands/migrate/files-to-references.ts b/packages/cli/src/commands/migrate/files-to-references.ts new file mode 100644 index 0000000000..568a01ee8a --- /dev/null +++ b/packages/cli/src/commands/migrate/files-to-references.ts @@ -0,0 +1,289 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { Command, Flags } from '@oclif/core'; +import chalk from 'chalk'; +import { createInterface } from 'node:readline'; +import { + printHeader, + printSuccess, + printWarning, + printError, + printInfo, + printStep, + createTimer, +} from '../../utils/format.js'; +import { bootSchemaStack } from '../../utils/schema-migrate.js'; +import { loadConfig } from '../../utils/config.js'; + +async function confirm(question: string): Promise { + if (!process.stdin.isTTY) return false; // non-interactive → require --yes + const rl = createInterface({ input: process.stdin, output: process.stdout }); + try { + const answer: string = await new Promise((resolve) => rl.question(question, resolve)); + return /^y(es)?$/i.test(answer.trim()); + } finally { + rl.close(); + } +} + +/** + * The settings + storage service plugins, so the booted kernel carries + * `sys_file`/`sys_migration` and the deployment's REAL storage adapter. + * Settings first: the storage plugin re-resolves its adapter from persisted + * settings when a settings service is present, which is how an S3-configured + * deployment's backfill uploads land in S3 rather than on this machine. + * Storage config mirrors `os serve`'s resolution (config.storage, else the + * local default) so the CLI materialises bytes exactly where the server would. + */ +async function buildDataMigrationPlugins(): Promise { + const plugins: unknown[] = []; + try { + const { SettingsServicePlugin } = await import('@objectstack/service-settings'); + plugins.push(new SettingsServicePlugin({ registerRoutes: false })); + } catch { + // optional — without it, constructor/env-driven storage config still applies + } + const { StorageServicePlugin } = await import('@objectstack/service-storage'); + let arg: Record | undefined; + try { + const { config } = await loadConfig(); + const cfgStorage = (config as any)?.storage; + if (cfgStorage && (cfgStorage.driver || cfgStorage.adapter)) arg = cfgStorage; + } catch { + // artifact-only project (no objectstack.config.ts) — use the default below + } + if (arg === undefined) { + const root = process.env.OS_STORAGE_ROOT || '.objectstack/data/uploads'; + arg = { driver: 'local', root }; + } + plugins.push(new StorageServicePlugin({ ...(arg as any), registerRoutes: false })); + return plugins; +} + +/** + * `os migrate files-to-references` — the ADR-0104 D3 data migration, with its + * self-check gate (#3617). + * + * Converts legacy file-field values (inline metadata blobs, resolver URLs, + * `data:` URIs) to owned `sys_file` references, reconciles the ownership + * ledger against what records actually hold, and — on an `--apply` run whose + * reconciliation reports zero blocking discrepancies — records the + * deployment-level `adr-0104-file-references` flag. That flag (never the + * platform version) is what later opens released-file collection (#3459) and + * strict media value-shape enforcement (#3438) on this deployment. + * + * Dry run by default, and a dry run writes NOTHING — not conversions, not the + * flag. Not run / not passed → files keep being retained forever: storage + * cost, zero data loss. + */ +export default class MigrateFilesToReferences extends Command { + static override description = + 'Migrate legacy file-field values to sys_file references and verify the ownership ledger (ADR-0104). ' + + 'Dry-run by default; --apply also records the deployment-level migration flag when the self-check passes.'; + + static override examples = [ + '$ os migrate files-to-references', + '$ os migrate files-to-references --apply', + '$ os migrate files-to-references --apply --yes --json', + '$ os migrate files-to-references --object product --object article', + ]; + + static override flags = { + 'database-url': Flags.string({ + description: 'Database URL to migrate (defaults to $OS_DATABASE_URL / the project DB)', + env: 'OS_DATABASE_URL', + }), + apply: Flags.boolean({ + description: + 'Write the conversions and record the deployment migration flag (default is a read-only dry run)', + default: false, + }), + yes: Flags.boolean({ char: 'y', description: 'Skip the --apply confirmation prompt', default: false }), + object: Flags.string({ + description: 'Restrict to this object (repeatable; default: every object with a file field)', + multiple: true, + }), + 'max-records': Flags.integer({ + description: + 'Safety bound on records scanned per object — exceeding it truncates the scan and fails the gate', + }), + 'include-unreferenced': Flags.boolean({ + description: 'Also sweep for committed files nothing references (advisory; extra full sys_file read)', + default: false, + }), + json: Flags.boolean({ description: 'Output as JSON (implies non-interactive; requires --yes to apply)' }), + }; + + async run(): Promise { + const { flags } = await this.parse(MigrateFilesToReferences); + const timer = createTimer(); + const apply = flags.apply; + + if (!flags.json) { + printHeader('Migrate · files-to-references'); + } + + // Confirmation gate — before boot, since an apply run starts writing as + // it scans. The documented workflow is: dry-run first, then --apply. + if (apply && !flags.yes) { + if (flags.json || !process.stdin.isTTY) { + if (flags.json) { + console.log(JSON.stringify({ error: 'confirmation_required', hint: 'pass --yes' })); + this.exit(1); + } + printWarning('Apply mode rewrites record data. Re-run with --yes to confirm, or run without --apply to preview.'); + this.exit(1); + return; + } + const ok = await confirm( + chalk.bold('\nConvert legacy file values and record the migration flag on this database? [y/N] '), + ); + if (!ok) { + printInfo('Aborted — no changes made.'); + return; + } + } + + if (!flags.json) { + printStep(apply ? 'Booting data stack (APPLY mode)…' : 'Booting data stack (dry run)…'); + } + + let stack; + try { + stack = await bootSchemaStack({ + databaseUrl: flags['database-url'], + extraPlugins: await buildDataMigrationPlugins(), + }); + } catch (error: any) { + if (flags.json) { console.log(JSON.stringify({ error: error.message })); this.exit(1); } + printError(error.message || String(error)); + this.exit(1); + return; + } + + try { + const engine: any = stack.kernel.getService('objectql'); + if (typeof engine?.getObject !== 'function' || !engine.getObject('sys_file')) { + throw new Error( + 'sys_file is not registered on this stack — the storage service objects are required. ' + + 'Ensure @objectstack/service-storage is installed, then re-run.', + ); + } + // An empty scan is indistinguishable from a clean one, and this command's + // verdict is what later authorises irreversible behaviour — so refuse to + // run when no app metadata is loaded (missing artifact / wrong directory) + // rather than "verify" a database the scan never actually looked at. + const loadedObjects: string[] = + typeof engine.getConfigs === 'function' ? Object.keys(engine.getConfigs()) : []; + if (!loadedObjects.some((name) => !name.startsWith('sys_'))) { + throw new Error( + 'No app objects are loaded, so the scan would examine nothing. ' + + 'Run "os build" in your project root first (the migration reads dist/objectstack.json), then re-run.', + ); + } + const getStorage = () => { + try { + return stack.kernel.getService('file-storage'); + } catch { + return null; + } + }; + + const { + runFilesToReferencesMigration, + formatBackfillReport, + formatFileReferenceReport, + } = await import('@objectstack/service-storage'); + + // In JSON mode keep stdout parseable — route migration warnings to stderr. + const logger = flags.json + ? { info: (m: string) => console.error(m), warn: (m: string) => console.error(m) } + : { info: (m: string) => printInfo(m), warn: (m: string) => printWarning(m) }; + + const result = await runFilesToReferencesMigration(engine, getStorage, logger, { + apply, + objects: flags.object, + maxRecordsPerObject: flags['max-records'], + includeUnreferenced: flags['include-unreferenced'], + }); + + if (flags.json) { + console.log(JSON.stringify({ + database: stack.dbLabel, + apply, + backfill: { + scannedObjects: result.backfill.scannedObjects, + scannedRecords: result.backfill.scannedRecords, + converted: result.backfill.converted, + alreadyReferences: result.backfill.alreadyReferences, + externalUrls: result.backfill.externalUrls, + unresolvable: result.backfill.unresolvable, + truncated: result.backfill.truncated, + // already_id rows are the bulk and carry no action — report the rest + actions: result.backfill.actions.filter((a) => a.kind !== 'already_id'), + }, + verify: { + scannedObjects: result.verify.scannedObjects, + scannedRecords: result.verify.scannedRecords, + heldReferences: result.verify.heldReferences, + ownedFiles: result.verify.ownedFiles, + counts: result.verify.counts, + blocking: result.verify.blocking, + ok: result.verify.ok, + truncated: result.verify.truncated, + issues: result.verify.issues, + }, + gatePassed: result.gatePassed, + gateFailures: result.gateFailures, + flag: result.flag, + duration: timer.elapsed(), + }, null, 2)); + if (!result.gatePassed) this.exit(1); + return; + } + + printInfo(`Database: ${chalk.white(stack.dbLabel)}`); + console.log(''); + console.log(formatBackfillReport(result.backfill)); + console.log(''); + console.log(formatFileReferenceReport(result.verify)); + console.log(''); + + if (result.gatePassed) { + if (apply) { + printSuccess( + 'Self-check passed — deployment flag recorded (adr-0104-file-references). ' + + 'This deployment is now eligible for strict media enforcement and, once shipped, file collection.', + ); + } else if (result.backfill.converted > 0) { + printInfo( + `Dry run only — ${result.backfill.converted} value(s) would be converted. ` + + 'Re-run with --apply to convert and record the deployment flag.', + ); + } else { + printInfo( + 'Data is already in reference form. Re-run with --apply to record the deployment flag.', + ); + } + } else { + for (const failure of result.gateFailures) { + printError(`Gate not passed: ${failure}`); + } + printWarning( + apply + ? 'The migration flag was recorded as NOT verified — collection and strict enforcement stay closed. Fix the records listed above and re-run.' + : 'Fix the records listed above, then re-run (and finally with --apply).', + ); + } + console.log(chalk.dim(` ${timer.display()}`)); + console.log(''); + if (!result.gatePassed) this.exit(1); + } catch (error: any) { + if (flags.json) { console.log(JSON.stringify({ error: error.message })); this.exit(1); } + printError(error.message || String(error)); + this.exit(1); + } finally { + await stack.shutdown(); + } + } +} diff --git a/packages/cli/src/utils/schema-migrate.ts b/packages/cli/src/utils/schema-migrate.ts index 4b01e25bac..121f690984 100644 --- a/packages/cli/src/utils/schema-migrate.ts +++ b/packages/cli/src/utils/schema-migrate.ts @@ -75,7 +75,18 @@ function redactUrl(url: string): string { } /** Boot the schema stack. Caller MUST call `shutdown()` when done. */ -export async function bootSchemaStack(opts: { databaseUrl?: string } = {}): Promise { +export async function bootSchemaStack( + opts: { + databaseUrl?: string; + /** + * Service plugins to register after the data stack (driver/metadata/ + * objectql/app) and before start — e.g. `os migrate files-to-references` + * adds settings + storage so `sys_file` and the deployment's real storage + * adapter are present. Plain schema commands pass nothing. + */ + extraPlugins?: unknown[]; + } = {}, +): Promise { const { createStandaloneStack, Runtime } = await import('@objectstack/runtime'); const stack = await createStandaloneStack({ @@ -89,6 +100,9 @@ export async function bootSchemaStack(opts: { databaseUrl?: string } = {}): Prom for (const plugin of stack.plugins) { await kernel.use(plugin); } + for (const plugin of opts.extraPlugins ?? []) { + await kernel.use(plugin as any); + } await runtime.start(); const driver = findSqlDriver(kernel); diff --git a/packages/objectql/src/engine.test.ts b/packages/objectql/src/engine.test.ts index 200cf837f5..4d05da81dd 100644 --- a/packages/objectql/src/engine.test.ts +++ b/packages/objectql/src/engine.test.ts @@ -1321,6 +1321,23 @@ describe('ObjectQL Engine', () => { expect(mockDriver.find).toHaveBeenCalledTimes(1); // primary read only }); + it('[#3617] the raw-file-values context marker skips resolution: stored ids come back verbatim', async () => { + // The ADR-0104 backfill/reconciliation audits the STORED form; if + // this marker stopped being honoured, the reconciliation would see + // expanded objects, report every held reference as absent, and a + // missed unowned_reference would falsely pass the collection gate. + withSchema(); + vi.mocked(mockDriver.find).mockResolvedValueOnce([ + { id: 'd1', cover: 'file_a', attachments: ['file_b'] }, + ]); + + const result = await engine.find('doc', { context: { __rawFileValues: true } } as any); + + expect(result[0].cover).toBe('file_a'); + expect(result[0].attachments).toEqual(['file_b']); + expect(mockDriver.find).toHaveBeenCalledTimes(1); // no sys_file sub-read either + }); + it('does NOT fire a sys_file query for url-shaped values (external url, /api path, data: URI)', async () => { // Regression: a file field legitimately holds a URL string in the // legacy/dual-mode world; only an opaque id token is a reference. diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 6cafb91e96..01d62a82b3 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -12,7 +12,7 @@ import { type DroppedFieldsEvent } from '@objectstack/spec/data'; import type { WriteObservabilityOptions } from '@objectstack/spec/contracts'; -import { parseAutonumberFormat, renderAutonumber, missingFieldValues, isTenancyDisabled, FILE_REFERENCE_TYPES, isFileIdToken } from '@objectstack/spec/data'; +import { parseAutonumberFormat, renderAutonumber, missingFieldValues, isTenancyDisabled, FILE_REFERENCE_TYPES, isFileIdToken, RAW_FILE_VALUES_CONTEXT_KEY } from '@objectstack/spec/data'; import { ExecutionContext, ExecutionContextSchema } from '@objectstack/spec/kernel'; import { IDataDriver, IDataEngine, Logger, createLogger, withTransientRetry, type RetryOptions } from '@objectstack/core'; import { SummaryRecomputeError, type SummaryRecomputeFailure } from './summary-errors.js'; @@ -2199,6 +2199,10 @@ export class ObjectQL implements IDataEngine { execCtx?: ExecutionContext, ): Promise { if (!records || records.length === 0) return records; + // A caller whose subject is the STORED form — the ADR-0104 backfill / + // reconciliation (#3617) — opts out of resolution entirely; expanding + // ids before that scan would hide the very state it exists to audit. + if ((execCtx as any)?.[RAW_FILE_VALUES_CONTEXT_KEY] === true) return records; const objectSchema = this._registry.getObject(objectName); if (!objectSchema || !objectSchema.fields) return records; // Nothing to resolve against if the file object is not even registered diff --git a/packages/platform-objects/scripts/i18n-extract.config.ts b/packages/platform-objects/scripts/i18n-extract.config.ts index 2e51807fa2..c5ec3d181a 100644 --- a/packages/platform-objects/scripts/i18n-extract.config.ts +++ b/packages/platform-objects/scripts/i18n-extract.config.ts @@ -99,7 +99,7 @@ import { } from '../src/metadata/index.js'; // ── System ──────────────────────────────────────────────────────────────── -import { SysSetting, SysSecret, SysSettingAudit } from '../src/system/index.js'; +import { SysSetting, SysSecret, SysSettingAudit, SysMigration } from '../src/system/index.js'; // ── Existing Setup app + dashboards + translations ──────────────────────── import { SETUP_APP } from '../src/apps/setup.app.js'; @@ -187,6 +187,7 @@ export default defineStack({ SysSetting, SysSecret, SysSettingAudit, + SysMigration, ] as any, apps: [SETUP_APP, STUDIO_APP, ACCOUNT_APP] as any, diff --git a/packages/platform-objects/src/apps/translations/bundle-ownership.test.ts b/packages/platform-objects/src/apps/translations/bundle-ownership.test.ts index 5764158a2d..638ee68bea 100644 --- a/packages/platform-objects/src/apps/translations/bundle-ownership.test.ts +++ b/packages/platform-objects/src/apps/translations/bundle-ownership.test.ts @@ -33,7 +33,7 @@ const OWNED_OBJECTS = new Set([ // metadata 'sys_metadata', 'sys_metadata_history', 'sys_view_definition', 'sys_metadata_audit', // system - 'sys_setting', 'sys_secret', 'sys_setting_audit', + 'sys_setting', 'sys_secret', 'sys_setting_audit', 'sys_migration', ]); describe('objects translation bundle ownership (ADR-0029 D8)', () => { diff --git a/packages/platform-objects/src/apps/translations/en.objects.generated.ts b/packages/platform-objects/src/apps/translations/en.objects.generated.ts index ca18bf16a8..53a38fc146 100644 --- a/packages/platform-objects/src/apps/translations/en.objects.generated.ts +++ b/packages/platform-objects/src/apps/translations/en.objects.generated.ts @@ -3046,5 +3046,46 @@ export const enObjects: NonNullable = { label: "Recent" } } + }, + sys_migration: { + label: "Data Migration", + pluralLabel: "Data Migrations", + description: "Deployment-level data-migration flags: which gated data migrations ran here and whether their self-check passed.", + fields: { + id: { + label: "Migration ID", + help: "Well-known migration id (e.g. adr-0104-file-references). One row per migration." + }, + last_run_at: { + label: "Last Run At", + help: "When this migration last completed a gated (apply-mode) run on this deployment." + }, + verified_at: { + label: "Verified At", + help: "When the self-check last PASSED. Null until it does, and cleared again by a later failing run — a regression closes the gate. Consumers require this to be set AND blocking = 0." + }, + applied_at: { + label: "Applied At", + help: "When the backfill last ran in apply mode (writes enabled)." + }, + blocking: { + label: "Blocking Discrepancies", + help: "Blocking discrepancies reported by the last self-check. The gate requires 0." + }, + advisory: { + label: "Advisory Findings", + help: "Advisory findings from the last run (external URLs, stale owners, …) — cost storage or need a modelling decision, never block the gate." + }, + details: { + label: "Details (JSON)", + help: "JSON-encoded counts from the last run, for diagnostics." + }, + created_at: { + label: "Created At" + }, + updated_at: { + label: "Updated At" + } + } } }; diff --git a/packages/platform-objects/src/apps/translations/es-ES.objects.generated.ts b/packages/platform-objects/src/apps/translations/es-ES.objects.generated.ts index cd6fbd2920..4cff4c77ed 100644 --- a/packages/platform-objects/src/apps/translations/es-ES.objects.generated.ts +++ b/packages/platform-objects/src/apps/translations/es-ES.objects.generated.ts @@ -3046,5 +3046,46 @@ export const esESObjects: NonNullable = { label: "Recent" } } + }, + sys_migration: { + label: "", + pluralLabel: "", + description: "", + fields: { + id: { + label: "", + help: "" + }, + last_run_at: { + label: "", + help: "" + }, + verified_at: { + label: "", + help: "" + }, + applied_at: { + label: "", + help: "" + }, + blocking: { + label: "", + help: "" + }, + advisory: { + label: "", + help: "" + }, + details: { + label: "", + help: "" + }, + created_at: { + label: "" + }, + updated_at: { + label: "" + } + } } }; diff --git a/packages/platform-objects/src/apps/translations/ja-JP.objects.generated.ts b/packages/platform-objects/src/apps/translations/ja-JP.objects.generated.ts index 3706f39e16..b10661d276 100644 --- a/packages/platform-objects/src/apps/translations/ja-JP.objects.generated.ts +++ b/packages/platform-objects/src/apps/translations/ja-JP.objects.generated.ts @@ -3046,5 +3046,46 @@ export const jaJPObjects: NonNullable = { label: "Recent" } } + }, + sys_migration: { + label: "", + pluralLabel: "", + description: "", + fields: { + id: { + label: "", + help: "" + }, + last_run_at: { + label: "", + help: "" + }, + verified_at: { + label: "", + help: "" + }, + applied_at: { + label: "", + help: "" + }, + blocking: { + label: "", + help: "" + }, + advisory: { + label: "", + help: "" + }, + details: { + label: "", + help: "" + }, + created_at: { + label: "" + }, + updated_at: { + label: "" + } + } } }; diff --git a/packages/platform-objects/src/apps/translations/zh-CN.objects.generated.ts b/packages/platform-objects/src/apps/translations/zh-CN.objects.generated.ts index be36a5eb3c..b3b8458f3f 100644 --- a/packages/platform-objects/src/apps/translations/zh-CN.objects.generated.ts +++ b/packages/platform-objects/src/apps/translations/zh-CN.objects.generated.ts @@ -3046,5 +3046,46 @@ export const zhCNObjects: NonNullable = { label: "Recent" } } + }, + sys_migration: { + label: "数据迁移", + pluralLabel: "数据迁移", + description: "部署级数据迁移标记:本部署跑过哪些带门禁的数据迁移,以及各自的自检是否通过。", + fields: { + id: { + label: "迁移 ID", + help: "约定的迁移标识(如 adr-0104-file-references)。每个迁移一行。" + }, + last_run_at: { + label: "最近运行时间", + help: "本部署最近一次完成带门禁运行(apply 模式)的时间。" + }, + verified_at: { + label: "通过自检时间", + help: "自检最近一次通过的时间。未通过前为空;之后若有运行未通过会被清空——数据回退即自行关闭门禁。消费方要求本字段有值且阻断项为 0。" + }, + applied_at: { + label: "写入执行时间", + help: "回填最近一次以 apply 模式(允许写入)运行的时间。" + }, + blocking: { + label: "阻断项", + help: "最近一次自检报告的阻断项数量。门禁要求为 0。" + }, + advisory: { + label: "提示项", + help: "最近一次运行的提示项(外部 URL、失效所有者等)——只涉及存储成本或需要建模决定,绝不阻断门禁。" + }, + details: { + label: "详情(JSON)", + help: "最近一次运行的计数,JSON 编码,供诊断用。" + }, + created_at: { + label: "创建时间" + }, + updated_at: { + label: "更新时间" + } + } } }; diff --git a/packages/platform-objects/src/system/index.ts b/packages/platform-objects/src/system/index.ts index 46050d0141..a44e4dceae 100644 --- a/packages/platform-objects/src/system/index.ts +++ b/packages/platform-objects/src/system/index.ts @@ -4,11 +4,19 @@ * platform-objects/system — Platform System Objects * * Cross-cutting system-level objects that don't belong to identity, - * security, audit, or integration. Currently hosts the generic - * settings K/V store backing ADR-0007 (Settings Manifest + K/V Store + - * Resolver). + * security, audit, or integration. Hosts the generic settings K/V store + * backing ADR-0007 (Settings Manifest + K/V Store + Resolver) and the + * deployment-level data-migration flags (#3617). */ export { SysSetting } from './sys-setting.object.js'; export { SysSecret } from './sys-secret.object.js'; export { SysSettingAudit } from './sys-setting-audit.object.js'; +export { SysMigration } from './sys-migration.object.js'; +export { + readDataMigrationFlag, + isDataMigrationVerified, + recordDataMigrationRun, + type MigrationFlagEngine, + type DataMigrationRunOutcome, +} from './migration-flag.js'; diff --git a/packages/platform-objects/src/system/migration-flag.test.ts b/packages/platform-objects/src/system/migration-flag.test.ts new file mode 100644 index 0000000000..60d9307e22 --- /dev/null +++ b/packages/platform-objects/src/system/migration-flag.test.ts @@ -0,0 +1,114 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect } from 'vitest'; +import { + readDataMigrationFlag, + isDataMigrationVerified, + recordDataMigrationRun, + type MigrationFlagEngine, +} from './migration-flag.js'; + +const MIGRATION = 'adr-0104-file-references'; + +function fakeEngine(rows: Array> = [], opts: { registered?: boolean } = {}) { + const tables: Record>> = { sys_migration: rows }; + const engine: MigrationFlagEngine & { tables: typeof tables } = { + getObject: (name) => + name === 'sys_migration' && opts.registered !== false ? { name: 'sys_migration' } : undefined, + async find(object, options: any) { + const id = options?.where?.id; + return (tables[object] ?? []).filter((r) => id === undefined || r.id === id); + }, + async insert(object, data: any) { + (tables[object] ??= []).push({ ...data }); + return data; + }, + async update(object, data: any) { + const row = (tables[object] ?? []).find((r) => r.id === data.id); + if (row) Object.assign(row, data); + return row; + }, + tables, + }; + return engine; +} + +describe('deployment-level data-migration flags (#3617)', () => { + it('reads null when no row exists, and the gate stays closed', async () => { + const engine = fakeEngine(); + expect(await readDataMigrationFlag(engine, MIGRATION)).toBeNull(); + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(false); + }); + + it('reads null when sys_migration is not registered (bare kernel)', async () => { + const engine = fakeEngine([{ id: MIGRATION, blocking: 0, verified_at: 'x' }], { registered: false }); + expect(await readDataMigrationFlag(engine, MIGRATION)).toBeNull(); + }); + + it('a failing read closes the gate rather than opening it', async () => { + const engine = fakeEngine(); + engine.find = async () => { + throw new Error('db down'); + }; + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(false); + }); + + it('records a passing run: verified_at set, blocking 0 — gate opens', async () => { + const engine = fakeEngine(); + const flag = await recordDataMigrationRun(engine, { + migrationId: MIGRATION, + passed: true, + blocking: 0, + advisory: 3, + applied: true, + details: { converted: 5 }, + }); + + expect(flag.verified_at).toBeTruthy(); + expect(engine.tables.sys_migration).toHaveLength(1); + expect(engine.tables.sys_migration[0]).toMatchObject({ id: MIGRATION, blocking: 0, advisory: 3 }); + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(true); + }); + + it('records a failing run: verified_at stays null — gate stays closed', async () => { + const engine = fakeEngine(); + await recordDataMigrationRun(engine, { migrationId: MIGRATION, passed: false, blocking: 4, applied: true }); + + expect(engine.tables.sys_migration[0].verified_at).toBeNull(); + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(false); + }); + + /** + * A deployment whose data regressed since it last verified must close its + * own gate — a later failing run CLEARS verified_at, it does not coast on + * the earlier pass. + */ + it('a failing run after a passing one closes the gate again', async () => { + const engine = fakeEngine(); + await recordDataMigrationRun(engine, { migrationId: MIGRATION, passed: true, blocking: 0, applied: true }); + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(true); + + await recordDataMigrationRun(engine, { migrationId: MIGRATION, passed: false, blocking: 2 }); + + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(false); + expect(engine.tables.sys_migration).toHaveLength(1); // upsert, not a second row + }); + + it('preserves applied_at across a verify-only re-run', async () => { + const engine = fakeEngine(); + await recordDataMigrationRun(engine, { migrationId: MIGRATION, passed: true, blocking: 0, applied: true }); + const appliedAt = engine.tables.sys_migration[0].applied_at; + expect(appliedAt).toBeTruthy(); + + await recordDataMigrationRun(engine, { migrationId: MIGRATION, passed: true, blocking: 0, applied: false }); + + expect(engine.tables.sys_migration[0].applied_at).toBe(appliedAt); + }); + + it('a malformed blocking count reads as not-verified, never as zero', async () => { + const engine = fakeEngine([ + { id: MIGRATION, last_run_at: 'x', verified_at: 'x', blocking: 'garbage' }, + ]); + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(false); + }); +}); diff --git a/packages/platform-objects/src/system/migration-flag.ts b/packages/platform-objects/src/system/migration-flag.ts new file mode 100644 index 0000000000..336a38f7ce --- /dev/null +++ b/packages/platform-objects/src/system/migration-flag.ts @@ -0,0 +1,134 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { + DATA_MIGRATION_FLAG_OBJECT, + isDataMigrationFlagVerified, + type DataMigrationFlag, +} from '@objectstack/spec/system'; + +/** + * Deployment-level data-migration flag persistence (#3617). + * + * Reads and writes the `sys_migration` rows defined by {@link SysMigration}, + * against the duck-typed engine surface the other platform helpers use. The + * asymmetry between the two directions is deliberate: + * + * - **Reads fail toward "not verified".** A missing table, an unreadable row, + * a malformed row — every failure mode answers `null` / `false`, because a + * gate that cannot read its evidence must stay closed, never open. + * - **Writes fail loudly.** They run inside a migration command whose whole + * output is this record; a swallowed write error would report a gate as + * passed that no consumer will ever see. + */ + +const SYSTEM_CTX = { isSystem: true } as const; + +/** Engine surface the flag helpers need — duck-typed like the storage seams. */ +export interface MigrationFlagEngine { + getObject(name: string): unknown | undefined; + find(object: string, options: Record): Promise>>; + insert(object: string, data: Record, options?: Record): Promise; + update(object: string, data: Record, options: Record): Promise; +} + +/** + * The flag row for `migrationId`, or `null` — where `null` covers every way + * of not having trustworthy evidence: no `sys_migration` object registered, + * no row, or a read failure. + */ +export async function readDataMigrationFlag( + engine: MigrationFlagEngine, + migrationId: string, +): Promise { + if (!engine.getObject(DATA_MIGRATION_FLAG_OBJECT)) return null; + try { + const rows = await engine.find(DATA_MIGRATION_FLAG_OBJECT, { + where: { id: migrationId }, + limit: 1, + context: { ...SYSTEM_CTX }, + }); + const row = rows?.[0]; + if (!row || row.id !== migrationId) return null; + return { + id: migrationId, + last_run_at: String(row.last_run_at ?? ''), + verified_at: row.verified_at == null ? null : String(row.verified_at), + applied_at: row.applied_at == null ? null : String(row.applied_at), + // A non-numeric count must read as "not zero", not as 0 — coerce + // failures land on NaN, which fails the === 0 gate. + blocking: typeof row.blocking === 'number' ? row.blocking : Number(row.blocking ?? Number.NaN), + advisory: typeof row.advisory === 'number' ? row.advisory : undefined, + details: typeof row.details === 'string' ? row.details : undefined, + }; + } catch { + return null; + } +} + +/** + * Has `migrationId` been run AND self-check-verified on this deployment? + * The read side of the gate — `verified_at` set and `blocking === 0`, via + * the spec's single `isDataMigrationFlagVerified` predicate. + */ +export async function isDataMigrationVerified( + engine: MigrationFlagEngine, + migrationId: string, +): Promise { + return isDataMigrationFlagVerified(await readDataMigrationFlag(engine, migrationId)); +} + +/** Outcome of one gated (apply-mode) migration run. */ +export interface DataMigrationRunOutcome { + migrationId: string; + /** Did the self-check pass (zero blocking findings, nothing truncated)? */ + passed: boolean; + /** Blocking findings from the self-check. */ + blocking: number; + /** Advisory findings (never gate). */ + advisory?: number; + /** Whether this run applied writes (as opposed to verifying existing state). */ + applied?: boolean; + /** JSON-serialisable counts for the `details` column. */ + details?: unknown; +} + +/** + * Upsert the flag row for a completed apply-mode run. `verified_at` is set on + * a passing run and CLEARED on a failing one — a deployment whose data has + * regressed since it last verified closes its own gate. Dry runs must not + * call this: recording is what distinguishes a gated migration from a script + * whose output can be ignored. + */ +export async function recordDataMigrationRun( + engine: MigrationFlagEngine, + outcome: DataMigrationRunOutcome, +): Promise { + const now = new Date().toISOString(); + const existing = await readDataMigrationFlag(engine, outcome.migrationId); + const flag: DataMigrationFlag = { + id: outcome.migrationId, + last_run_at: now, + verified_at: outcome.passed ? now : null, + applied_at: outcome.applied === true ? now : (existing?.applied_at ?? null), + blocking: outcome.blocking, + advisory: outcome.advisory, + details: outcome.details === undefined ? undefined : JSON.stringify(outcome.details), + }; + + const row: Record = { + ...flag, + advisory: flag.advisory ?? null, + details: flag.details ?? null, + updated_at: now, + }; + if (existing) { + await engine.update(DATA_MIGRATION_FLAG_OBJECT, row, { context: { ...SYSTEM_CTX } }); + } else { + await engine.insert( + DATA_MIGRATION_FLAG_OBJECT, + { ...row, created_at: now }, + { context: { ...SYSTEM_CTX } }, + ); + } + return flag; +} diff --git a/packages/platform-objects/src/system/sys-migration.object.ts b/packages/platform-objects/src/system/sys-migration.object.ts new file mode 100644 index 0000000000..bc9af0e0c9 --- /dev/null +++ b/packages/platform-objects/src/system/sys-migration.object.ts @@ -0,0 +1,115 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { ObjectSchema, Field } from '@objectstack/spec/data'; + +/** + * sys_migration — Deployment-level data-migration flags (#3617) + * + * One row per DATA migration (row rewrites — distinct from the DDL ChangeSet + * surface `os migrate plan/apply` covers), recording whether THIS deployment + * has run it and whether its self-check passed. ObjectStack is a development + * platform: every deployment upgrades on its own schedule and its data is + * observable only by itself, so "the migration is done" can never be inferred + * from the installed version — it is evidence each deployment produces by + * running `os migrate ` against its own database. + * + * Consumers that would act irreversibly on migrated data gate on + * `isDataMigrationFlagVerified` (verified_at set AND blocking = 0) — e.g. the + * ADR-0104 file-as-reference row (`adr-0104-file-references`) gates released- + * file collection (#3459 PR-5b) and the strict media value-shape default + * (#3438). No row / failed self-check → consumers stay in their safe legacy + * posture: files are retained forever, lax values keep warning. Fail toward + * retention, per deployment. + * + * Writes flow through `recordDataMigrationRun` (system context, from the + * migration commands) — the API surface is read-only diagnostics. + * + * Registered by the first consuming service (`@objectstack/service-storage`); + * the row contract lives in `@objectstack/spec/system` (`DataMigrationFlagSchema`) + * so any package can read flags without depending on this one. + * + * @namespace sys + */ +export const SysMigration = ObjectSchema.create({ + name: 'sys_migration', + label: 'Data Migration', + pluralLabel: 'Data Migrations', + icon: 'database', + isSystem: true, + managedBy: 'engine-owned', + description: 'Deployment-level data-migration flags: which gated data migrations ran here and whether their self-check passed.', + nameField: 'id', // [ADR-0079] canonical primary-title pointer + titleFormat: '{id}', + highlightFields: ['id', 'blocking', 'verified_at', 'last_run_at'], + + fields: { + id: Field.text({ + label: 'Migration ID', + required: true, + readonly: true, + maxLength: 128, + description: 'Well-known migration id (e.g. adr-0104-file-references). One row per migration.', + }), + + last_run_at: Field.datetime({ + label: 'Last Run At', + required: true, + readonly: true, + description: 'When this migration last completed a gated (apply-mode) run on this deployment.', + }), + + verified_at: Field.datetime({ + label: 'Verified At', + readonly: true, + description: + 'When the self-check last PASSED. Null until it does, and cleared again by a later failing ' + + 'run — a regression closes the gate. Consumers require this to be set AND blocking = 0.', + }), + + applied_at: Field.datetime({ + label: 'Applied At', + readonly: true, + description: 'When the backfill last ran in apply mode (writes enabled).', + }), + + blocking: Field.number({ + label: 'Blocking Discrepancies', + required: true, + readonly: true, + description: 'Blocking discrepancies reported by the last self-check. The gate requires 0.', + }), + + advisory: Field.number({ + label: 'Advisory Findings', + readonly: true, + description: + 'Advisory findings from the last run (external URLs, stale owners, …) — cost storage or ' + + 'need a modelling decision, never block the gate.', + }), + + details: Field.text({ + label: 'Details (JSON)', + readonly: true, + description: 'JSON-encoded counts from the last run, for diagnostics.', + }), + + created_at: Field.datetime({ + label: 'Created At', + readonly: true, + }), + + updated_at: Field.datetime({ + label: 'Updated At', + readonly: true, + }), + }, + + enable: { + trackHistory: false, + searchable: false, + apiEnabled: true, + // Diagnostic read surface only — writes go through recordDataMigrationRun + // (system context) so the gate cannot be flipped by hand from the API. + apiMethods: ['get', 'list'], + }, +}); diff --git a/packages/services/service-storage/src/backfill-file-references.ts b/packages/services/service-storage/src/backfill-file-references.ts index f4b0f98ba6..31aa6c45be 100644 --- a/packages/services/service-storage/src/backfill-file-references.ts +++ b/packages/services/service-storage/src/backfill-file-references.ts @@ -1,7 +1,7 @@ // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. import { randomUUID } from 'node:crypto'; -import { FILE_REFERENCE_TYPES, isFileIdToken } from '@objectstack/spec/data'; +import { FILE_REFERENCE_TYPES, isFileIdToken, RAW_FILE_VALUES_CONTEXT_KEY } from '@objectstack/spec/data'; import type { IStorageService } from '@objectstack/spec/contracts'; /** @@ -38,7 +38,12 @@ import type { IStorageService } from '@objectstack/spec/contracts'; * and left alone, so a partially-completed run is safe to repeat. */ -const SYSTEM_CTX = { isSystem: true } as const; +// Raw-form reads: the backfill's subject is the STORED value. Without the +// raw marker, a live kernel's read resolver would re-expand every id this +// run (or a previous one) already wrote, so each pass would re-report the +// same values as freshly converted instead of `already_id` — the run would +// never converge to zero. +const SYSTEM_CTX = { isSystem: true, [RAW_FILE_VALUES_CONTEXT_KEY]: true } as const; const SCAN_PAGE_SIZE = 200; /** `data:;base64,` — the only inline form carrying real bytes. */ diff --git a/packages/services/service-storage/src/file-reference-lifecycle.ts b/packages/services/service-storage/src/file-reference-lifecycle.ts index 23ad98b715..6811417602 100644 --- a/packages/services/service-storage/src/file-reference-lifecycle.ts +++ b/packages/services/service-storage/src/file-reference-lifecycle.ts @@ -1,7 +1,7 @@ // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. import { randomUUID } from 'node:crypto'; -import { FILE_REFERENCE_TYPES, isFileIdToken } from '@objectstack/spec/data'; +import { FILE_REFERENCE_TYPES, isFileIdToken, RAW_FILE_VALUES_CONTEXT_KEY } from '@objectstack/spec/data'; import type { IStorageService } from '@objectstack/spec/contracts'; /** @@ -53,7 +53,10 @@ import type { IStorageService } from '@objectstack/spec/contracts'; */ const PACKAGE_ID = 'com.objectstack.service.storage'; -const SYSTEM_CTX = { isSystem: true } as const; +// Bookkeeping reads want the STORED form — never the read resolver's expanded +// shape (and skipping resolution also saves a batched sys_file sub-read on +// every internal page). Inert on write contexts. +const SYSTEM_CTX = { isSystem: true, [RAW_FILE_VALUES_CONTEXT_KEY]: true } as const; /** Bound on records resolved per where-shaped multi-delete — matches the * attachment lifecycle's posture: bound one pass, converge across sweeps. */ diff --git a/packages/services/service-storage/src/files-to-references-migration.test.ts b/packages/services/service-storage/src/files-to-references-migration.test.ts new file mode 100644 index 0000000000..319919ef84 --- /dev/null +++ b/packages/services/service-storage/src/files-to-references-migration.test.ts @@ -0,0 +1,160 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect, vi } from 'vitest'; +import { isDataMigrationVerified } from '@objectstack/platform-objects/system'; +import { + runFilesToReferencesMigration, + type FilesToReferencesEngine, +} from './files-to-references-migration.js'; + +const silentLogger = () => ({ info: vi.fn(), warn: vi.fn(), debug: vi.fn() }); + +const MIGRATION = 'adr-0104-file-references'; + +const REGISTRY: Record = { + sys_file: { fields: { id: { type: 'text' } } }, + sys_migration: { fields: { id: { type: 'text' } } }, + product: { + fields: { + id: { type: 'text' }, + name: { type: 'text' }, + image: { type: 'image' }, + }, + }, +}; + +function fakeEngine(tables: Record>>) { + tables.sys_migration ??= []; + const engine: FilesToReferencesEngine & { tables: typeof tables } = { + getObject: (name) => REGISTRY[name], + getConfigs: () => REGISTRY, + async find(object, options: any) { + let rows = tables[object] ?? []; + const id = options?.where?.id; + if (typeof id === 'string') rows = rows.filter((r) => r.id === id); + const start = typeof options?.offset === 'number' ? options.offset : 0; + const end = typeof options?.limit === 'number' ? start + options.limit : undefined; + return rows.slice(start, end); + }, + async insert(object, data: any) { + (tables[object] ??= []).push({ ...data }); + return data; + }, + async update(object, data: any) { + const row = (tables[object] ?? []).find((r) => String(r.id) === String(data.id)); + if (row) Object.assign(row, data); + return row; + }, + tables, + }; + return engine; +} + +const run = (engine: any, opts: any = {}) => + runFilesToReferencesMigration(engine, () => null, silentLogger(), opts); + +describe('runFilesToReferencesMigration (#3617)', () => { + it('dry run writes NOTHING — no conversions, no flag — even when the gate would pass', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1', image: 'f1' }], + sys_file: [{ id: 'f1', ref_object: 'product', ref_id: 'p1', ref_field: 'image', status: 'committed' }], + }); + + const result = await run(engine); + + expect(result.gatePassed).toBe(true); + expect(result.flag).toBeNull(); + expect(engine.tables.sys_migration).toHaveLength(0); + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(false); + }); + + it('apply + clean reconciliation records a verified flag — the gate opens', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1', image: '/api/v1/storage/files/f1' }], + sys_file: [{ id: 'f1', status: 'committed' }], + }); + // The live claim hooks are what record ownership after a rewrite; this + // fake engine has none, so pre-claim the file the way the hook would. + engine.tables.sys_file[0] = { + ...engine.tables.sys_file[0], + ref_object: 'product', + ref_id: 'p1', + ref_field: 'image', + }; + + const result = await run(engine, { apply: true }); + + expect(engine.tables.product[0].image).toBe('f1'); // converted + expect(result.backfill.converted).toBe(1); + expect(result.gatePassed).toBe(true); + expect(result.flag?.verified_at).toBeTruthy(); + expect(result.flag?.blocking).toBe(0); + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(true); + }); + + it('apply with a blocking discrepancy records the failure and the gate stays closed', async () => { + // p1 holds f1, but no sys_file owner is recorded → unowned_reference. + const engine = fakeEngine({ + product: [{ id: 'p1', image: 'f1' }], + sys_file: [{ id: 'f1', status: 'committed' }], + }); + + const result = await run(engine, { apply: true }); + + expect(result.gatePassed).toBe(false); + expect(result.gateFailures.join(' ')).toMatch(/blocking/); + expect(result.verify.counts.unowned_reference).toBe(1); + expect(result.flag?.verified_at).toBeNull(); + expect(result.flag?.blocking).toBe(1); + expect(engine.tables.sys_migration).toHaveLength(1); + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(false); + }); + + it('a regression after a verified run closes the gate again', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1', image: 'f1' }], + sys_file: [{ id: 'f1', ref_object: 'product', ref_id: 'p1', ref_field: 'image', status: 'committed' }], + }); + await run(engine, { apply: true }); + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(true); + + // Something later strips the ownership row — reality and ledger diverge. + delete engine.tables.sys_file[0].ref_object; + delete engine.tables.sys_file[0].ref_id; + await run(engine, { apply: true }); + + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(false); + }); + + it('a truncated scan fails the gate even with zero blocking findings', async () => { + // The backfill reads 200-row pages and only re-checks its bound between + // FULL pages, so >200 rows are needed for the bound to actually bite. + const engine = fakeEngine({ + product: Array.from({ length: 201 }, (_, i) => ({ id: `p${i}`, image: null })), + sys_file: [], + }); + + const result = await run(engine, { apply: true, maxRecordsPerObject: 100 }); + + expect(result.verify.blocking).toBe(0); + expect(result.gatePassed).toBe(false); + expect(result.gateFailures.join(' ')).toMatch(/truncated/); + expect(result.flag?.verified_at).toBeNull(); + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(false); + }); + + it('external URLs are advisory: reported on the flag, never blocking the gate', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1', image: 'https://cdn.example.com/logo.png' }], + sys_file: [], + }); + + const result = await run(engine, { apply: true }); + + expect(result.backfill.externalUrls).toBe(1); + expect(engine.tables.product[0].image).toBe('https://cdn.example.com/logo.png'); // untouched + expect(result.gatePassed).toBe(true); + expect(result.flag?.advisory).toBe(1); + expect(await isDataMigrationVerified(engine, MIGRATION)).toBe(true); + }); +}); diff --git a/packages/services/service-storage/src/files-to-references-migration.ts b/packages/services/service-storage/src/files-to-references-migration.ts new file mode 100644 index 0000000000..87c44ef4b9 --- /dev/null +++ b/packages/services/service-storage/src/files-to-references-migration.ts @@ -0,0 +1,144 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { FILE_REFERENCES_MIGRATION_ID } from '@objectstack/spec/system'; +import type { DataMigrationFlag } from '@objectstack/spec/system'; +import { recordDataMigrationRun } from '@objectstack/platform-objects/system'; +import type { MigrationFlagEngine } from '@objectstack/platform-objects/system'; +import type { IStorageService } from '@objectstack/spec/contracts'; +import { + backfillFileReferences, + type BackfillEngine, + type BackfillLogger, + type BackfillReport, +} from './backfill-file-references.js'; +import { verifyFileReferences, type FileReferenceReport } from './verify-file-references.js'; + +/** + * The gated file-as-reference data migration (#3617) — ADR-0104 D3 wave 2's + * acceptance evidence, produced per deployment. + * + * Composes the two existing halves into the migration step a deployment + * actually runs (`os migrate files-to-references`): + * + * 1. {@link backfillFileReferences} — convert legacy file-field values to + * `sys_file` ids (dry run unless `apply`), + * 2. {@link verifyFileReferences} — reconcile the ownership ledger against + * what records actually hold, + * 3. on an APPLY run, record the outcome on the deployment-level + * `sys_migration` flag (`adr-0104-file-references`). + * + * The flag — not the platform version — is what may later open released-file + * collection (#3459 PR-5b) and strict media value-shape enforcement (#3438) + * on this deployment. That ordering is the point of the design: a ledger must + * be shown to agree with reality *here* before anything may delete bytes + * *here*, and no release-side observation window can vouch for a database + * only its own operator can see. + * + * ## The self-check gate + * + * A run passes only when the reconciliation reports ZERO blocking + * discrepancies AND neither scan was truncated (`ok` covers only what was + * read). Advisory findings — external URLs awaiting a modelling decision, + * stale owners, unreferenced files — never gate: they cost storage, not data. + * + * ## Dry run writes NOTHING + * + * Not the conversions, and not the flag either — even when the self-check + * would pass. `--apply` is the single writing mode, so "did this run change + * my deployment's posture" never depends on what the run happened to find. + * A failing APPLY run, by contrast, deliberately records `blocking` and + * clears `verified_at`: data that has regressed closes its own gate. + */ + +/** Engine surface the migration needs — the union of its three halves'. */ +export type FilesToReferencesEngine = BackfillEngine & MigrationFlagEngine; + +export interface FilesToReferencesOptions { + /** Write conversions and record the deployment flag. Omit for a dry run. */ + apply?: boolean; + /** Restrict to these objects (default: every object with a file field). */ + objects?: string[]; + /** Safety bound on records read per object; exceeding it fails the gate. */ + maxRecordsPerObject?: number; + /** Include the advisory unreferenced-file sweep in the reconciliation. */ + includeUnreferenced?: boolean; +} + +export interface FilesToReferencesResult { + backfill: BackfillReport; + verify: FileReferenceReport; + /** Zero blocking discrepancies and nothing truncated. */ + gatePassed: boolean; + /** Why the gate did not pass (empty when it did). */ + gateFailures: string[]; + /** The flag row as recorded (apply runs only; null on a dry run). */ + flag: DataMigrationFlag | null; +} + +export async function runFilesToReferencesMigration( + engine: FilesToReferencesEngine, + getStorage: () => IStorageService | null | undefined, + logger: BackfillLogger, + options: FilesToReferencesOptions = {}, +): Promise { + const apply = options.apply === true; + const shared = { + objects: options.objects, + maxRecordsPerObject: options.maxRecordsPerObject, + }; + + const backfill = await backfillFileReferences(engine, getStorage, logger, { + ...shared, + apply, + }); + + // The reconciliation runs on BOTH modes: a dry run still reports, against + // current data, exactly what stands between this deployment and the gate. + const verify = await verifyFileReferences(engine, { + ...shared, + includeUnreferenced: options.includeUnreferenced, + }); + + const gateFailures: string[] = []; + if (verify.blocking > 0) { + gateFailures.push( + `${verify.blocking} blocking discrepancy(ies) between records and recorded ownership`, + ); + } + if (backfill.truncated || verify.truncated) { + gateFailures.push( + 'scan truncated by maxRecordsPerObject — the verdict covers only what was read; ' + + 'raise the bound and re-run', + ); + } + const gatePassed = gateFailures.length === 0; + + let flag: DataMigrationFlag | null = null; + if (apply) { + const advisory = + (verify.issues.length - verify.blocking) + backfill.externalUrls + backfill.unresolvable; + flag = await recordDataMigrationRun(engine, { + migrationId: FILE_REFERENCES_MIGRATION_ID, + passed: gatePassed, + blocking: verify.blocking, + advisory, + applied: true, + details: { + scanned_objects: verify.scannedObjects.length, + scanned_records: verify.scannedRecords, + held_references: verify.heldReferences, + owned_files: verify.ownedFiles, + issue_counts: verify.counts, + backfill: { + converted: backfill.converted, + already_references: backfill.alreadyReferences, + external_urls: backfill.externalUrls, + unresolvable: backfill.unresolvable, + }, + truncated: backfill.truncated || verify.truncated, + }, + }); + } + + return { backfill, verify, gatePassed, gateFailures, flag }; +} diff --git a/packages/services/service-storage/src/index.ts b/packages/services/service-storage/src/index.ts index b2ffa76f89..b8937070ab 100644 --- a/packages/services/service-storage/src/index.ts +++ b/packages/services/service-storage/src/index.ts @@ -43,3 +43,9 @@ export type { } from './backfill-file-references.js'; export { installAttachmentAccessHooks, installAttachmentReadVisibility } from './attachment-access-hooks.js'; export type { AttachmentSharingLike } from './attachment-access-hooks.js'; +export { runFilesToReferencesMigration } from './files-to-references-migration.js'; +export type { + FilesToReferencesEngine, + FilesToReferencesOptions, + FilesToReferencesResult, +} from './files-to-references-migration.js'; diff --git a/packages/services/service-storage/src/storage-service-plugin.ts b/packages/services/service-storage/src/storage-service-plugin.ts index 373069b876..a4f50afd2f 100644 --- a/packages/services/service-storage/src/storage-service-plugin.ts +++ b/packages/services/service-storage/src/storage-service-plugin.ts @@ -29,6 +29,12 @@ import { SystemFile, SystemUploadSession } from './objects/index.js'; // storage domain, not the audit/compliance ledger. Definition stays in // platform-objects; storage now contributes (registers) it instead of audit. import { SysAttachment } from '@objectstack/platform-objects/audit'; +// Deployment-level data-migration flags (#3617). The table is platform-generic +// (defined in platform-objects, contract in spec/system), registered here by +// its first consuming domain: the ADR-0104 file-as-reference row gates this +// service's released-file collection (#3459 PR-5b) and the strict media +// value-shape default (#3438). +import { SysMigration } from '@objectstack/platform-objects/system'; import { SwappableStorageService } from './swappable-storage-service.js'; /** @@ -185,7 +191,7 @@ export class StorageServicePlugin implements Plugin { version: '1.0.0', type: 'plugin', scope: 'system', - objects: [SystemFile, SystemUploadSession, SysAttachment], + objects: [SystemFile, SystemUploadSession, SysAttachment, SysMigration], }); } catch { // manifest service may not be available in all environments diff --git a/packages/services/service-storage/src/verify-file-references.ts b/packages/services/service-storage/src/verify-file-references.ts index a56a416595..7b70b41097 100644 --- a/packages/services/service-storage/src/verify-file-references.ts +++ b/packages/services/service-storage/src/verify-file-references.ts @@ -1,6 +1,6 @@ // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. -import { FILE_REFERENCE_TYPES, isFileIdToken } from '@objectstack/spec/data'; +import { FILE_REFERENCE_TYPES, isFileIdToken, RAW_FILE_VALUES_CONTEXT_KEY } from '@objectstack/spec/data'; /** * File-reference reconciliation (ADR-0104 D3 wave 2) — the executable form of @@ -33,7 +33,11 @@ import { FILE_REFERENCE_TYPES, isFileIdToken } from '@objectstack/spec/data'; * The scan is read-only. It never writes, never tombstones, and never deletes. */ -const SYSTEM_CTX = { isSystem: true } as const; +// Raw-form reads: on a live kernel the engine's read resolver expands stored +// ids to `{ id, url, … }` in place, which would hide every held reference +// from this scan (false stale_owner noise — and a missed unowned_reference +// would falsely pass the gate). The scan's subject is the stored form. +const SYSTEM_CTX = { isSystem: true, [RAW_FILE_VALUES_CONTEXT_KEY]: true } as const; /** Records read per page while scanning an object. */ const SCAN_PAGE_SIZE = 500; diff --git a/packages/spec/api-surface.json b/packages/spec/api-surface.json index 29ddafa93b..8a2e1fa766 100644 --- a/packages/spec/api-surface.json +++ b/packages/spec/api-surface.json @@ -447,6 +447,7 @@ "QueryFilterSchema (const)", "QueryInput (type)", "QuerySchema (const)", + "RAW_FILE_VALUES_CONTEXT_KEY (const)", "RECORD_SURFACE_PAGE_THRESHOLD (const)", "REFERENCE_VALUE_TYPES (const)", "RangeOperatorSchema (const)", @@ -714,11 +715,14 @@ "CursorStyleSchema (const)", "CursorUpdate (type)", "CursorUpdateSchema (const)", + "DATA_MIGRATION_FLAG_OBJECT (const)", "DashboardLike (interface)", "DataClassification (type)", "DataClassificationPolicy (type)", "DataClassificationPolicySchema (const)", "DataClassificationSchema (const)", + "DataMigrationFlag (type)", + "DataMigrationFlagSchema (const)", "DatabaseLevelIsolationStrategy (type)", "DatabaseLevelIsolationStrategyInput (type)", "DatabaseLevelIsolationStrategySchema (const)", @@ -796,6 +800,7 @@ "ExtendedLogLevel (type)", "ExternalServiceDestinationConfig (type)", "ExternalServiceDestinationConfigSchema (const)", + "FILE_REFERENCES_MIGRATION_ID (const)", "FacetConfig (type)", "FacetConfigSchema (const)", "FailoverConfig (type)", @@ -1260,6 +1265,7 @@ "docAudienceAllows (function)", "emailTemplateForm (const)", "gcsStorageExample (const)", + "isDataMigrationFlagVerified (function)", "isPublicAudience (function)", "minioStorageExample (const)", "resolveActionConfirm (function)", diff --git a/packages/spec/json-schema.manifest.json b/packages/spec/json-schema.manifest.json index 3c4ddd66b6..de6aed0044 100644 --- a/packages/spec/json-schema.manifest.json +++ b/packages/spec/json-schema.manifest.json @@ -1317,6 +1317,7 @@ "system/CursorUpdate", "system/DataClassification", "system/DataClassificationPolicy", + "system/DataMigrationFlag", "system/DatabaseLevelIsolationStrategy", "system/DatabaseProvider", "system/DeadLetterQueue", diff --git a/packages/spec/src/data/field-value.zod.ts b/packages/spec/src/data/field-value.zod.ts index f640887cf6..edf155452d 100644 --- a/packages/spec/src/data/field-value.zod.ts +++ b/packages/spec/src/data/field-value.zod.ts @@ -122,6 +122,25 @@ export function isFileIdToken(value: unknown): value is string { return typeof value === 'string' && /^[A-Za-z0-9_-]{1,64}$/.test(value); } +/** + * ExecutionContext key that makes a read return file-field values in their + * STORED form (the bare `sys_file` id) instead of the expanded + * `{ id, name, url, … }` shape the engine's read resolver derives in place. + * + * Exists for the callers whose subject is the stored form itself — the + * ADR-0104 backfill and `verifyFileReferences` reconciliation (#3617). On a + * live kernel the resolver otherwise rewrites every resolvable id before the + * scan sees it, so the reconciliation would report held references as absent: + * false `stale_owner` noise, and — worse — a missed `unowned_reference` is a + * false pass of the very gate that authorises irreversible behaviour. + * + * A spec-level constant for the same reason `isFileIdToken` is one: the + * engine (which honours the key) and the storage service (which sends it) + * must agree on the exact string, and two hand-copied literals drifting by a + * character would silently re-enable expansion mid-scan. + */ +export const RAW_FILE_VALUES_CONTEXT_KEY = '__rawFileValues'; + /** Structured JSON payloads persisted in JSON columns. */ export const STRUCTURED_JSON_TYPES: ReadonlySet = new Set([ 'json', 'composite', 'repeater', 'record', 'location', 'address', 'vector', diff --git a/packages/spec/src/system/migration.zod.ts b/packages/spec/src/system/migration.zod.ts index 049d373a53..5f1845af94 100644 --- a/packages/spec/src/system/migration.zod.ts +++ b/packages/spec/src/system/migration.zod.ts @@ -1,5 +1,21 @@ // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. +/** + * Migration protocol — the two kinds of migration, kept apart on purpose. + * + * A **schema migration** (`ChangeSet` + its atomic operations) reshapes the + * physical database to match metadata: add a field, change a type, create an + * object, run SQL. It is derivable from the metadata and applied by + * `os migrate plan` / `os migrate apply`. + * + * A **data migration** rewrites the rows themselves, and whether it is done is + * a property of ONE DEPLOYMENT's database rather than of the installed code + * version — so it cannot be expressed as a ChangeSet, and its completion cannot + * be inferred from a release. `DataMigrationFlag` is the per-deployment record + * that one ran here and its self-check passed; consumers that would act + * irreversibly on migrated data gate on the flag instead of the version. + */ + import { z } from 'zod'; import { FieldSchema } from '../data/field.zod'; import { ObjectSchema } from '../data/object.zod'; @@ -86,3 +102,65 @@ export const ChangeSetSchema = lazySchema(() => z.object({ export type ChangeSet = z.infer; export type MigrationOperation = z.infer; + +// --- Deployment-level data-migration flags (#3617) --- +// +// A ChangeSet above describes SCHEMA work (DDL). A data migration is different: +// it rewrites rows, and whether it has been completed — and shown correct — is +// a property of ONE DEPLOYMENT's database, not of the installed code version. +// ObjectStack is a development platform: third-party deployments upgrade on +// their own schedule and their data is not observable by anyone else, so no +// release-side observation window can vouch for them. The evidence has to be +// produced by each deployment itself. +// +// The flag row below is that evidence. `os migrate ` runs the +// migration's backfill, runs its self-check, and only a passing self-check +// records `verified_at`. Consumers that would act irreversibly on migrated +// data (e.g. the sys_file reap for ADR-0104 D3, or a strict value-shape +// default) gate on the flag — NOT on the platform version — so upgrading the +// code never flips destructive behaviour on by itself. No flag / failed run → +// consumers stay in their safe legacy posture ("fail toward retention"). + +/** + * Object (table) name under which deployment-level data-migration flags + * persist. The object definition lives in + * `@objectstack/platform-objects/system` (`SysMigration`). + */ +export const DATA_MIGRATION_FLAG_OBJECT = 'sys_migration'; + +/** + * Well-known migration id: ADR-0104 D3 wave 2 file-as-reference — legacy + * file-field values backfilled to `sys_file` ids and the ownership ledger + * reconciled (`verifyFileReferences` reports zero blocking discrepancies). + * Gates BOTH ADR-0104 follow-ups on this deployment: enabling released-file + * collection (#3459 PR-5b) and strict media value-shape enforcement (#3438). + */ +export const FILE_REFERENCES_MIGRATION_ID = 'adr-0104-file-references'; + +export const DataMigrationFlagSchema = lazySchema(() => z.object({ + id: z.string().describe('Migration id (e.g. adr-0104-file-references) — one row per data migration'), + last_run_at: z.string().datetime().describe('When this migration last completed a gated (apply-mode) run on this deployment'), + verified_at: z.string().datetime().nullable().optional() + .describe('When the self-check last PASSED. Null/absent until it does — and cleared again by a later failing run, so a regression closes the gate'), + applied_at: z.string().datetime().nullable().optional() + .describe('When the backfill last ran in apply mode (writes enabled)'), + blocking: z.number().int().min(0) + .describe('Blocking discrepancies reported by the last self-check. The gate requires 0'), + advisory: z.number().int().min(0).optional() + .describe('Advisory findings from the last run (external URLs, stale owners, …) — cost storage or need a modelling decision, never block the gate'), + details: z.string().optional() + .describe('JSON-encoded counts from the last run, for diagnostics'), +}).describe('Deployment-level record that a data migration ran here and its self-check passed — the evidence gate consumers read instead of the platform version')); +export type DataMigrationFlag = z.infer; + +/** + * Does a flag row authorise its consumers? The ONE arbiter for both current + * consumers (reap gating #3459, strict flip #3438) — a hand-copied predicate + * drifting by one clause would let one consumer act on evidence the other + * rejects. Pure derivation (Prime Directive #2): reading the row from the + * engine lives in `@objectstack/platform-objects/system`. + */ +export function isDataMigrationFlagVerified(flag: DataMigrationFlag | null | undefined): boolean { + if (!flag) return false; + return flag.verified_at != null && flag.verified_at !== '' && flag.blocking === 0; +}