diff --git a/apps/cli/docs/go-cli-porting-status.md b/apps/cli/docs/go-cli-porting-status.md index c1ac3b2c96..c104f116bf 100644 --- a/apps/cli/docs/go-cli-porting-status.md +++ b/apps/cli/docs/go-cli-porting-status.md @@ -86,7 +86,7 @@ These commands exist in the TS CLI today but have no direct top-level equivalent | `db dump` | `ported` | `legacy/commands/db/dump/` | `n/a` | `n/a` | Native TS port. Streams `pg_dump`/`pg_dumpall` via a Docker container (`LegacyDockerRun`); schema/data/role modes, `--dry-run` script print, IPv4 transaction-pooler fallback. | | `db lint` | `ported` | `legacy/commands/db/lint/` | `n/a` | `n/a` | Native TS port. Runs `plpgsql_check` in a rolled-back transaction via LegacyDbConnection; emits Go-parity pretty JSON. | | `db pull` | `ported` | `legacy/commands/db/pull/` | `n/a` | `n/a` | Native TS port. Native pg-delta / migra migration + `--declarative` pg-delta export; reconciles `schema_migrations`. The initial-migra pull dumps the remote schema natively (`pg_dump`) then appends the migra diff. Only `--experimental` (structured dump) still delegates to Go, pending a TS PostgreSQL DDL parser for `format.WriteStructuredSchemas`. | -| `db push` | `ported` | `legacy/commands/db/push/` | `n/a` | `n/a` | Native TS port. Connects local/linked/`--db-url`; pushes pending migrations, `--include-seed` seeds (`seed_files` hash tracking), `--include-roles`, `[db.vault]` secrets; `--dry-run`. `encrypted:` vault secrets + best-effort pg-delta catalog cache not ported (no output impact). | +| `db push` | `ported` | `legacy/commands/db/push/` | `n/a` | `n/a` | Native TS port. Connects local/linked/`--db-url`; pushes pending migrations, `--include-seed` seeds (`seed_files` hash tracking), `--include-roles`, `[db.vault]` secrets including decrypted `encrypted:` values; `--dry-run`; best-effort pg-delta migrations-catalog cache (warning-only on failure). | | `db reset` | `ported` | `legacy/commands/db/reset/` | `n/a` | `n/a` | Remote path native (drop user schemas, vault upsert, MigrateAndSeed, `--version`/`--last`, `--sql-paths` seed override). Local path native: running check, recreate + migrate + seed via the hidden Go `db __db-bootstrap` seam, storage-gated bucket seeding (reuses `seed buckets`), git-branch `Finished…` line. Only the niche `--experimental` remote schema-files path still delegates to the Go binary (telemetry-disabled). | | `db start` | `ported` | `legacy/commands/db/start/` | `n/a` | `n/a` | Native TS port. Validates config, checks "already running" (prints Go's line), else delegates the container bootstrap (create + health + initial schema/roles/migrations/seed + `_current_branch`) to the hidden Go `db __db-bootstrap --mode start` seam. No status table / `cli_stack_started` (those are `supabase start`). `--from-backup` supported. | | `inspect report` | `ported` | `legacy/commands/inspect/report/` | `n/a` | `n/a` | Native TS port. Runs every inspect query via server-side `COPY ... CSV`, writes 14 CSVs under `//`, then renders a Go-parity Glamour rules summary (bounded csvq-subset evaluator; custom `[experimental.inspect.rules]` supported). | diff --git a/apps/cli/src/legacy/commands/db/diff/diff.handler.ts b/apps/cli/src/legacy/commands/db/diff/diff.handler.ts index 72bc7c2e58..61fb818dbc 100644 --- a/apps/cli/src/legacy/commands/db/diff/diff.handler.ts +++ b/apps/cli/src/legacy/commands/db/diff/diff.handler.ts @@ -8,7 +8,6 @@ import { LegacyCliConfig } from "../../../config/legacy-cli-config.service.ts"; import { legacyAqua, legacyYellow } from "../../../shared/legacy-colors.ts"; import { legacyReadDbToml } from "../../../shared/legacy-db-config.toml-read.ts"; import { LegacyDbConfigResolver } from "../../../shared/legacy-db-config.service.ts"; -import type { LegacyPgConnInput } from "../../../shared/legacy-db-connection.service.ts"; import type { LegacyDbConnType } from "../../../shared/legacy-db-target-flags.ts"; import { legacyGetHostname } from "../../../shared/legacy-hostname.ts"; import { legacyMakeDir } from "../../../shared/legacy-make-dir.ts"; @@ -45,23 +44,6 @@ import { const warnDiff = `WARNING: The diff tool is not foolproof, so you may need to manually rearrange and modify the generated migration. Run ${legacyAqua("supabase db reset")} to verify that the new migration does not generate errors.`; -/** Builds a plain Postgres URL from a resolved connection (Go's `ToPostgresURL`). */ -const connToUrl = (conn: LegacyPgConnInput): string => - legacyToPostgresURL({ - host: conn.host, - port: conn.port, - user: conn.user, - password: conn.password, - database: conn.database, - ...(conn.options !== undefined ? { options: conn.options } : {}), - ...(conn.runtimeParams !== undefined ? { runtimeParams: conn.runtimeParams } : {}), - // Preserve a `--db-url` connect_timeout; Go's ToPostgresURL serializes the - // parsed ConnectTimeout (`connect.go`), defaulting to 10 only when unset. - ...(conn.connectTimeoutSeconds !== undefined - ? { connectTimeoutSeconds: conn.connectTimeoutSeconds } - : {}), - }); - /** * Rebuilds the `db diff` argv for the pgAdmin / pg-schema delegate path. Flags * stay flags (the Go-proxy channel-parity rule). The explicit `--from`/`--to` and @@ -235,7 +217,7 @@ export const legacyDbDiff = Effect.fn("legacy.db.diff")(function* (flags: Legacy mergedLinkedRef = ref2; cfg = yield* legacyReadDbToml(fs, path, cliConfig.workdir, ref2); } - return connToUrl(resolved.conn); + return legacyToPostgresURL(resolved.conn); } case "migrations": return yield* seam.exportCatalog({ @@ -362,7 +344,7 @@ export const legacyDbDiff = Effect.fn("legacy.db.diff")(function* (flags: Legacy }); const linkedRef = Option.getOrUndefined(resolved.ref ?? Option.none()); if (linkedRef !== undefined) linkedRefForCache = linkedRef; - const targetUrl = connToUrl(resolved.conn); + const targetUrl = legacyToPostgresURL(resolved.conn); // Read config with the resolved linked ref so a matching `[remotes.]` // block merges before the engine/format/runtime are read — Go loads config diff --git a/apps/cli/src/legacy/commands/db/pull/pull.handler.ts b/apps/cli/src/legacy/commands/db/pull/pull.handler.ts index 858bb50c1a..7334bbfbe1 100644 --- a/apps/cli/src/legacy/commands/db/pull/pull.handler.ts +++ b/apps/cli/src/legacy/commands/db/pull/pull.handler.ts @@ -91,23 +91,6 @@ const DEPRECATION_LINE = /** Migration-file mode for the initial pg_dump seed (Go's `OpenFile(..., 0644)`). */ const MIGRATION_FILE_MODE = 0o644; -/** Builds a plain Postgres URL from a resolved connection (Go's `ToPostgresURL`). */ -const connToUrl = (conn: LegacyPgConnInput): string => - legacyToPostgresURL({ - host: conn.host, - port: conn.port, - user: conn.user, - password: conn.password, - database: conn.database, - ...(conn.options !== undefined ? { options: conn.options } : {}), - ...(conn.runtimeParams !== undefined ? { runtimeParams: conn.runtimeParams } : {}), - // Preserve a `--db-url` connect_timeout; Go's ToPostgresURL serializes the - // parsed ConnectTimeout (`connect.go`), defaulting to 10 only when unset. - ...(conn.connectTimeoutSeconds !== undefined - ? { connectTimeoutSeconds: conn.connectTimeoutSeconds } - : {}), - }); - /** Rebuilds the `db pull` argv for the Go-delegated `--experimental` structured-dump branch. */ const rebuildDelegateArgs = (flags: LegacyDbPullFlags): Array => { const args = ["db", "pull"]; @@ -233,7 +216,7 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy }); const linkedRef = Option.getOrUndefined(resolved.ref ?? Option.none()); if (linkedRef !== undefined) linkedRefForCache = linkedRef; - const targetUrl = connToUrl(resolved.conn); + const targetUrl = legacyToPostgresURL(resolved.conn); // Reload config with the resolved linked ref so a matching `[remotes.]` // block merges before the engine/format/runtime/declarative paths are read — @@ -294,7 +277,7 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy .pipe(Effect.orElseSucceed(() => Option.none())); if (Option.isSome(pooler)) { yield* legacyEmitPoolerFallbackWarning(resolved.conn.host); - return yield* attempt(connToUrl(pooler.value)); + return yield* attempt(legacyToPostgresURL(pooler.value)); } } return yield* Effect.fail(error); diff --git a/apps/cli/src/legacy/commands/db/push/SIDE_EFFECTS.md b/apps/cli/src/legacy/commands/db/push/SIDE_EFFECTS.md index 5216459f76..6b928c8c4b 100644 --- a/apps/cli/src/legacy/commands/db/push/SIDE_EFFECTS.md +++ b/apps/cli/src/legacy/commands/db/push/SIDE_EFFECTS.md @@ -18,12 +18,12 @@ linked/remote Postgres database. ## Files Written -| Path | Format | When | -| ------------------------------------------------ | ------ | ------------------------------------------------------------------------- | -| `~/.supabase//linked-project.json` | JSON | on the `--linked` path (post-run cache, Go's `ensureProjectGroupsCached`) | -| `~/.supabase/telemetry.json` | JSON | always (post-run telemetry flush) | - -No project files are written. All other effects are database mutations (below). +| Path | Format | When | +| ------------------------------------------------------------------------------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `~/.supabase//linked-project.json` | JSON | on the `--linked` path (post-run cache, Go's `ensureProjectGroupsCached`) | +| `~/.supabase/telemetry.json` | JSON | always (post-run telemetry flush) | +| `/supabase/.temp/pgdelta/catalog--migrations--.json` | JSON | best-effort, after a successful migration apply, when pg-delta is enabled (`[experimental.pgdelta] enabled` or `SUPABASE_EXPERIMENTAL_PG_DELTA`); a failure only warns on stderr and never fails the push (Go's `pgcache.TryCacheMigrationsCatalog`) | +| `/supabase/.temp/pgdelta/pgdelta-target-ca.crt` | PEM | same gate as above, when the target requires SSL (`legacyPreparePgDeltaRef`) | ## Database Mutations @@ -43,11 +43,13 @@ No project files are written. All other effects are database mutations (below). ## Environment Variables -| Variable | Purpose | Required? | -| ----------------------- | ------------------------------------------- | ------------------------------------------------------- | -| `SUPABASE_ACCESS_TOKEN` | auth token for the `--linked` resolver path | no (falls back to keyring → `~/.supabase/access-token`) | -| `SUPABASE_DB_PASSWORD` | password for the linked/remote connection | no (`--password`/`-p` takes precedence) | -| `SUPABASE_YES` | auto-confirm prompts (Go's `viper YES`) | no (also `--yes`) | +| Variable | Purpose | Required? | +| ---------------------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------- | +| `SUPABASE_ACCESS_TOKEN` | auth token for the `--linked` resolver path | no (falls back to keyring → `~/.supabase/access-token`) | +| `SUPABASE_DB_PASSWORD` | password for the linked/remote connection | no (`--password`/`-p` takes precedence) | +| `SUPABASE_YES` | auto-confirm prompts (Go's `viper YES`) | no (also `--yes`) | +| `SUPABASE_EXPERIMENTAL_PG_DELTA` | enables the migrations-catalog cache when `[experimental.pgdelta].enabled` is unset | no (project `.env` or shell) | +| `SUPABASE_INTERNAL_IMAGE_REGISTRY` | overrides the pg-delta edge-runtime image registry for the cache export | no (project `.env` or shell) | ## Exit Codes @@ -97,8 +99,14 @@ stdout is payload-only. A single `result` object is emitted: - **`--dry-run`** prints the plan (roles / migrations / seeds) and applies nothing. - **`[db.migrations].enabled = false`** / **`[db.seed].enabled = false`** print a skip notice naming the project ref (empty for local/db-url). -- **Vault**: only non-empty, non-`env()` `[db.vault]` literals are synced (Go syncs - secrets with a non-empty SHA256). **Known gap vs Go**: `encrypted:`-prefixed - vault secrets are currently skipped — dotenvx/ECIES decryption is not yet ported. -- **Migrations catalog cache** (Go's best-effort `pgcache.TryCacheMigrationsCatalog`, - warning-only) is not ported; it produces no output, so parity is preserved. +- **Vault**: non-empty, non-`env()` `[db.vault]` values are synced after config + load, including decrypted `encrypted:` values. +- **Migrations catalog cache**: ported (Go's best-effort `pgcache.TryCacheMigrationsCatalog`). + After a successful migration apply, when pg-delta is enabled, exports the target's + pg-delta catalog via the edge-runtime stack and writes it under + `supabase/.temp/pgdelta/`, pruning older snapshots for the same prefix (retains 2). + A failure only warns on stderr (`Warning: failed to cache migrations catalog: …`) + and never fails the push, matching Go exactly. Reuses `legacyExportCatalogPgDelta` + (the same pg-delta export path `db pull`/`db diff` use, which always mounts the + project root at `/workspace`) rather than a second copy, so the ENOENT bug fixed in + Go's `pgcache/cache.go` (supabase/cli#5921) has no TS equivalent. diff --git a/apps/cli/src/legacy/commands/db/push/push.handler.ts b/apps/cli/src/legacy/commands/db/push/push.handler.ts index 8bb4fdcd76..47ea574459 100644 --- a/apps/cli/src/legacy/commands/db/push/push.handler.ts +++ b/apps/cli/src/legacy/commands/db/push/push.handler.ts @@ -1,4 +1,4 @@ -import { Effect, FileSystem, Option, Path } from "effect"; +import { Clock, Effect, FileSystem, Option, Path } from "effect"; import { CliArgs } from "../../../../shared/cli/cli-args.service.ts"; import { LegacyDnsResolverFlag } from "../../../../shared/legacy/global-flags.ts"; @@ -9,6 +9,7 @@ import { LegacyProjectRefResolver } from "../../../config/legacy-project-ref.ser import { legacyAqua, legacyBold } from "../../../shared/legacy-colors.ts"; import { LegacyDbConfigResolver } from "../../../shared/legacy-db-config.service.ts"; import { + legacyApplyProjectEnv, legacyCheckDbToml, legacyLoadProjectEnv, } from "../../../shared/legacy-db-config.toml-read.ts"; @@ -18,10 +19,17 @@ import { legacySeedGlobals, } from "../../../shared/legacy-migration-apply.ts"; import { legacyPromptYesNo } from "../../../shared/legacy-prompt-yes-no.ts"; +import { legacyToPostgresURL } from "../../../shared/legacy-postgres-url.ts"; import { resolveLegacyDbTargetFlags } from "../../../shared/legacy-db-target-flags.ts"; import { LegacyLinkedProjectCache } from "../../../telemetry/legacy-linked-project-cache.service.ts"; import { LegacyTelemetryState } from "../../../telemetry/legacy-telemetry-state.service.ts"; -import { legacyListLocalMigrations } from "../shared/legacy-pgdelta.cache.ts"; +import { redactLegacyConnectionString } from "../../../shared/legacy-db-config.parse.ts"; +import { legacyParseBoolEnv } from "../shared/legacy-diff-engine.ts"; +import { + legacyListLocalMigrations, + legacyTryCacheMigrationsCatalog, +} from "../shared/legacy-pgdelta.cache.ts"; +import { type LegacyPgDeltaContext } from "../shared/legacy-pgdelta.ts"; import { LEGACY_ERR_MISSING_LOCAL, LEGACY_ERR_MISSING_REMOTE, @@ -92,6 +100,7 @@ export const legacyDbPush = Effect.fn("legacy.db.push")(function* (flags: Legacy let linkedRefForCache: string | undefined; const body = Effect.gen(function* () { + yield* legacyApplyProjectEnv(projectEnv); const target = resolveLegacyDbTargetFlags(cliArgs.args); // cobra MarkFlagsMutuallyExclusive("db-url", "linked", "local"), keyed off the // explicitly-set flags (cobra's `Changed`), not the `--linked` default value. @@ -286,9 +295,30 @@ export const legacyDbPush = Effect.fn("legacy.db.push")(function* (flags: Legacy } yield* legacyUpsertVaultSecrets(session, vaultSecrets); yield* legacyApplyMigrations(session, fs, path, pending, applyError); - // Go best-effort caches the migrations catalog for pg-delta; a failure - // only warns (`push.go:99-101`). The catalog cache is not yet ported, so - // there is nothing to warn about — parity is preserved (no extra output). + const cacheEnabled = + toml.pgDelta.enabled || + legacyParseBoolEnv(toml.envLookup("SUPABASE_EXPERIMENTAL_PG_DELTA")); + const pgDeltaCtx: LegacyPgDeltaContext = { + projectId: Option.getOrElse(cliConfig.projectId, () => ""), + cwd: workdir, + npmVersion: Option.getOrUndefined(toml.pgDelta.npmVersion), + denoVersion: toml.denoVersion, + }; + yield* legacyTryCacheMigrationsCatalog(fs, path, pgDeltaCtx, { + enabled: cacheEnabled, + targetUrl: legacyToPostgresURL(cfg.conn), + conn: cfg.conn, + isLocal: cfg.isLocal, + migrationsDir: path.join(workdir, "supabase", "migrations"), + nowMillis: yield* Clock.currentTimeMillis, + }).pipe( + Effect.catch((error) => + output.raw( + `Warning: failed to cache migrations catalog: ${redactLegacyConnectionString(error.message)}\n`, + "stderr", + ), + ), + ); } else { yield* output.raw("Schema migrations are up to date.\n", "stderr"); } @@ -336,5 +366,6 @@ export const legacyDbPush = Effect.fn("legacy.db.push")(function* (flags: Legacy ), ), Effect.ensuring(telemetryState.flush), + Effect.scoped, ); }); diff --git a/apps/cli/src/legacy/commands/db/push/push.integration.test.ts b/apps/cli/src/legacy/commands/db/push/push.integration.test.ts index 9a4b21db68..8ebd64c289 100644 --- a/apps/cli/src/legacy/commands/db/push/push.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/push/push.integration.test.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { BunServices } from "@effect/platform-bun"; @@ -29,6 +29,12 @@ import { LegacyDbConnection, type LegacyPgConnInput, } from "../../../shared/legacy-db-connection.service.ts"; +import { LegacyEdgeRuntimeScriptError } from "../../../shared/legacy-edge-runtime-script.errors.ts"; +import { + LegacyEdgeRuntimeScript, + type LegacyEdgeRuntimeRunOpts, +} from "../../../shared/legacy-edge-runtime-script.service.ts"; +import { LegacyPgDeltaSslProbe } from "../../../shared/legacy-pgdelta-ssl-probe.service.ts"; import { legacyDbPush } from "./push.handler.ts"; import type { LegacyDbPushFlags } from "./push.command.ts"; @@ -153,6 +159,9 @@ function setup( vaultRows?: ReadonlyArray<{ id: string; name: string }>; noSeedTable?: boolean; failExec?: string; + catalogStdout?: string; + catalogExportFailWith?: string; + noProjectId?: boolean; }, ) { if (opts.toml !== undefined) { @@ -169,6 +178,25 @@ function setup( const conn = mockConnection(opts); const telemetry = mockLegacyTelemetryStateTracked(); const linkedCache = mockLegacyLinkedProjectCacheTracked(); + + const edgeRunCalls: Array = []; + const registryEnvAtRunTime: Array = []; + const edge = Layer.succeed(LegacyEdgeRuntimeScript, { + run: (runOpts: LegacyEdgeRuntimeRunOpts) => { + edgeRunCalls.push(runOpts); + registryEnvAtRunTime.push(process.env["SUPABASE_INTERNAL_IMAGE_REGISTRY"]); + if (opts.catalogExportFailWith !== undefined) { + return Effect.fail( + new LegacyEdgeRuntimeScriptError({ message: opts.catalogExportFailWith }), + ); + } + return Effect.succeed({ stdout: opts.catalogStdout ?? '{"version":1}', stderr: "" }); + }, + }); + const sslProbe = Layer.succeed(LegacyPgDeltaSslProbe, { + requireSsl: () => Effect.succeed(false), + requireSslForHost: () => Effect.succeed(false), + }); const projectRefLayer = Layer.succeed(LegacyProjectRefResolver, { resolve: () => Effect.succeed(opts.projectRef ?? LEGACY_VALID_REF), resolveForLink: () => Effect.succeed(opts.projectRef ?? LEGACY_VALID_REF), @@ -188,7 +216,10 @@ function setup( out.layer, conn.layer, mockResolver({ isLocal: opts.isLocal ?? true }), - mockLegacyCliConfig({ workdir }), + mockLegacyCliConfig({ + workdir, + ...(opts.noProjectId === true ? { projectId: Option.none() } : {}), + }), BunServices.layer, // Prompts (migration/seed confirmation) are answered through mockOutput's // `promptConfirmResponses` (the TTY/clack path), so mark stdin a TTY. Stdin is @@ -201,8 +232,10 @@ function setup( projectRefLayer, telemetry.layer, linkedCache.layer, + edge, + sslProbe, ); - return { layer, out, conn, telemetry, linkedCache }; + return { layer, out, conn, telemetry, linkedCache, edgeRunCalls, registryEnvAtRunTime }; } const MIGRATION_DIR = "supabase/migrations"; @@ -266,6 +299,123 @@ describe("legacy db push", () => { }); }); + it.live("does not attempt to cache the migrations catalog when pg-delta is disabled", () => { + const { layer, out, edgeRunCalls } = setup(tmp.current, { + toml: 'project_id = "test"\n', + files: migrationFile("20240101000000"), + confirm: [true], + }); + return Effect.gen(function* () { + yield* legacyDbPush(DEFAULT_FLAGS).pipe(Effect.provide(layer)); + expect(edgeRunCalls).toHaveLength(0); + expect(out.stderrText).not.toContain("failed to cache migrations catalog"); + expect(existsSync(join(tmp.current, "supabase", ".temp", "pgdelta"))).toBe(false); + }); + }); + + it.live("caches the migrations catalog when project .env enables pg-delta", () => { + const { layer, out, edgeRunCalls } = setup(tmp.current, { + toml: 'project_id = "test"\n', + files: { + ...migrationFile("20240101000000"), + "supabase/.env": "SUPABASE_EXPERIMENTAL_PG_DELTA=true\n", + }, + confirm: [true], + catalogStdout: '{"snapshot":"ok"}', + }); + return Effect.gen(function* () { + yield* legacyDbPush(DEFAULT_FLAGS).pipe(Effect.provide(layer)); + expect(out.stderrText).not.toContain("failed to cache migrations catalog"); + expect(edgeRunCalls).toHaveLength(1); + const tempDir = join(tmp.current, "supabase", ".temp", "pgdelta"); + const catalogFiles = readdirSync(tempDir).filter((name) => + name.startsWith("catalog-local-migrations-"), + ); + expect(catalogFiles).toHaveLength(1); + }); + }); + + it.live("caches the migrations catalog after a successful push when pg-delta is enabled", () => { + const { layer, out, edgeRunCalls } = setup(tmp.current, { + toml: 'project_id = "test"\n[experimental.pgdelta]\nenabled = true\n', + files: migrationFile("20240101000000"), + confirm: [true], + catalogStdout: '{"snapshot":"ok"}', + }); + return Effect.gen(function* () { + yield* legacyDbPush(DEFAULT_FLAGS).pipe(Effect.provide(layer)); + expect(out.stderrText).not.toContain("failed to cache migrations catalog"); + expect(edgeRunCalls).toHaveLength(1); + const tempDir = join(tmp.current, "supabase", ".temp", "pgdelta"); + const catalogFiles = readdirSync(tempDir).filter((name) => + name.startsWith("catalog-local-migrations-"), + ); + expect(catalogFiles).toHaveLength(1); + expect(readFileSync(join(tempDir, catalogFiles[0]!), "utf8")).toBe('{"snapshot":"ok"}'); + }); + }); + + it.live("caches the migrations catalog with an empty projectId when none is resolved", () => { + const { layer, out, edgeRunCalls } = setup(tmp.current, { + toml: 'project_id = "test"\n[experimental.pgdelta]\nenabled = true\n', + files: migrationFile("20240101000000"), + confirm: [true], + catalogStdout: '{"snapshot":"ok"}', + noProjectId: true, + }); + return Effect.gen(function* () { + yield* legacyDbPush(DEFAULT_FLAGS).pipe(Effect.provide(layer)); + expect(out.stderrText).not.toContain("failed to cache migrations catalog"); + expect(edgeRunCalls).toHaveLength(1); + }); + }); + + it.live("warns without failing the push when the catalog export fails", () => { + const { layer, out } = setup(tmp.current, { + toml: 'project_id = "test"\n[experimental.pgdelta]\nenabled = true\n', + files: migrationFile("20240101000000"), + confirm: [true], + catalogExportFailWith: "edge-runtime script produced no output", + }); + return Effect.gen(function* () { + const exit = yield* legacyDbPush(DEFAULT_FLAGS).pipe(Effect.provide(layer), Effect.exit); + expect(Exit.isSuccess(exit)).toBe(true); + expect(out.stderrText).toContain( + "Warning: failed to cache migrations catalog: edge-runtime script produced no output", + ); + expect(out.stdoutText).toContain("Finished"); + }); + }); + + it.live( + "resolves the pg-delta cache export image via SUPABASE_INTERNAL_IMAGE_REGISTRY from supabase/.env", + () => { + const prev = process.env["SUPABASE_INTERNAL_IMAGE_REGISTRY"]; + delete process.env["SUPABASE_INTERNAL_IMAGE_REGISTRY"]; + const { layer, registryEnvAtRunTime } = setup(tmp.current, { + toml: 'project_id = "test"\n[experimental.pgdelta]\nenabled = true\n', + files: { + ...migrationFile("20240101000000"), + "supabase/.env": "SUPABASE_INTERNAL_IMAGE_REGISTRY=my-mirror.example.com\n", + }, + confirm: [true], + catalogStdout: '{"snapshot":"ok"}', + }); + return Effect.gen(function* () { + yield* legacyDbPush(DEFAULT_FLAGS).pipe(Effect.provide(layer)); + expect(registryEnvAtRunTime).toEqual(["my-mirror.example.com"]); + expect(process.env["SUPABASE_INTERNAL_IMAGE_REGISTRY"]).toBeUndefined(); + }).pipe( + Effect.ensuring( + Effect.sync(() => { + if (prev === undefined) delete process.env["SUPABASE_INTERNAL_IMAGE_REGISTRY"]; + else process.env["SUPABASE_INTERNAL_IMAGE_REGISTRY"] = prev; + }), + ), + ); + }, + ); + it.live("returns context canceled when the migration prompt is declined", () => { const { layer, conn } = setup(tmp.current, { toml: 'project_id = "test"\n', diff --git a/apps/cli/src/legacy/commands/db/push/push.layers.ts b/apps/cli/src/legacy/commands/db/push/push.layers.ts index dbc5bd5250..cf99bc494c 100644 --- a/apps/cli/src/legacy/commands/db/push/push.layers.ts +++ b/apps/cli/src/legacy/commands/db/push/push.layers.ts @@ -9,8 +9,11 @@ import { legacyProjectRefLayer } from "../../../config/legacy-project-ref.layer. import { legacyDbConfigLayer } from "../../../shared/legacy-db-config.layer.ts"; import { legacyDbConnectionLayer } from "../../../shared/legacy-db-connection.layer.ts"; import { legacyDebugLoggerLayer } from "../../../shared/legacy-debug-logger.layer.ts"; +import { legacyDockerRunLayer } from "../../../shared/legacy-docker-run.layer.ts"; +import { legacyEdgeRuntimeScriptLayer } from "../../../shared/legacy-edge-runtime-script.layer.ts"; import { stdinLayer } from "../../../../shared/runtime/stdin.layer.ts"; import { legacyIdentityStitchLayer } from "../../../shared/legacy-identity-stitch.ts"; +import { legacyPgDeltaSslProbeLayer } from "../../../shared/legacy-pgdelta-ssl-probe.layer.ts"; import { legacyLinkedProjectCacheLayer } from "../../../telemetry/legacy-linked-project-cache.layer.ts"; import { legacyTelemetryStateLayer } from "../../../telemetry/legacy-telemetry-state.layer.ts"; @@ -60,9 +63,17 @@ const dbConfig = legacyDbConfigLayer.pipe( Layer.provide(legacyIdentityStitchLayer), ); +const edgeRuntime = legacyEdgeRuntimeScriptLayer.pipe( + Layer.provide(legacyDockerRunLayer), + Layer.provide(cliConfig), +); + export const legacyDbPushRuntimeLayer = Layer.mergeAll( dbConfig, legacyDbConnectionLayer, + legacyDockerRunLayer, + edgeRuntime, + legacyPgDeltaSslProbeLayer, cliConfig, httpClient, credentials, diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.ts index a15f683d02..f06b687000 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.ts @@ -3,6 +3,7 @@ import { Effect, type FileSystem, Option, type Path } from "effect"; import { Output } from "../../../../shared/output/output.service.ts"; import { LegacyMigrationsReadError } from "../../../shared/legacy-migration.errors.ts"; +import { type LegacyPgDeltaContext, legacyExportCatalogPgDelta } from "./legacy-pgdelta.ts"; /** * Declarative catalog-cache key builders + on-disk catalog resolution, ported @@ -19,6 +20,8 @@ const INIT_SCHEMA_PATTERN = /([0-9]{14})_init\.sql/; const INIT_SCHEMA_CUTOFF = 20211209000000; // `pkg/migration/file.go` — valid migration filenames. const MIGRATE_FILE_PATTERN = /^([0-9]+)_(.*)\.sql$/; +// `internal/utils/misc.go` — `ProjectHostPattern`, matches a direct `db..supabase.{co,red}` host. +const PROJECT_HOST_PATTERN = /^(db\.)([a-z]{20})\.supabase\.(co|red)$/; /** Inputs to `setupInputsToken` — everything `start.SetupDatabase` consumes. */ export interface LegacySetupInputs { @@ -43,6 +46,28 @@ export function legacySanitizedCatalogPrefix(prefix: string): string { return trimmed.replace(CATALOG_PREFIX_PATTERN, "-"); } +/** + * Mirrors Go's `pgcache.CatalogPrefixFromConfig` (`pgcache/cache.go`): `"local"` + * for the local dev database, the project ref for a direct `db..supabase.*` + * host, else a stable `url-` derived from the connection. + */ +export function legacyCatalogPrefixFromConfig( + conn: { + readonly host: string; + readonly port: number; + readonly user: string; + readonly database: string; + }, + isLocal: boolean, +): string { + if (isLocal) return "local"; + const match = PROJECT_HOST_PATTERN.exec(conn.host); + if (match?.[2] !== undefined) return match[2]; + const key = `${conn.user}@${conn.host}:${conn.port}/${conn.database}`; + const digest = createHash("sha256").update(key, "utf8").digest("hex"); + return `url-${digest.slice(0, 12)}`; +} + /** Mirrors Go's `baselineVersionToken` (`declarative.go:665`): the image tag, or `pg`. */ export function legacyBaselineVersionToken(image: string, majorVersion: number): string { let tag = image.trim(); @@ -167,18 +192,20 @@ export const legacyListLocalMigrations = Effect.fnUntraced(function* ( /** * Mirrors Go's `pgcache.HashMigrations` (`pgcache/cache.go`): for each local - * migration (in list order), hash its path then its contents. Returns full hex. + * migration (in list order), hash its `workdir`-relative path then its + * contents. Returns full hex. */ export const legacyHashMigrations = Effect.fnUntraced(function* ( fs: FileSystem.FileSystem, path: Path.Path, + workdir: string, migrationsDir: string, ) { const migrations = yield* legacyListLocalMigrations(fs, path, migrationsDir); const hash = createHash("sha256"); for (const filePath of migrations) { const contents = yield* fs.readFile(filePath); - hash.update(filePath, "utf8"); + hash.update(path.relative(workdir, filePath), "utf8"); hash.update(contents); } return hash.digest("hex"); @@ -274,18 +301,13 @@ export const legacyResolveDeclarativeCatalogPath = Effect.fnUntraced(function* ( return latestPath; }); -/** - * Removes all but the newest `catalogRetentionCount` declarative catalogs for a - * prefix family. Mirrors Go's `cleanupOldDeclarativeCatalogs` (`declarative.go:610`). - */ -export const legacyCleanupOldDeclarativeCatalogs = Effect.fnUntraced(function* ( +const cleanupOldCatalogsByFamily = Effect.fnUntraced(function* ( fs: FileSystem.FileSystem, path: Path.Path, tempDir: string, - prefix: string, + familyPrefix: string, ) { const entries = yield* listJsonEntries(fs, tempDir); - const familyPrefix = `catalog-${legacySanitizedCatalogPrefix(prefix)}-declarative-`; const files = entries .filter((name) => name.startsWith(familyPrefix) && name.endsWith(".json")) .map((name) => ({ name, timestamp: Option.getOrElse(parseCatalogTimestamp(name), () => 0) })) @@ -298,3 +320,117 @@ export const legacyCleanupOldDeclarativeCatalogs = Effect.fnUntraced(function* ( .pipe(Effect.orElseSucceed(() => undefined)); } }); + +/** + * Removes all but the newest `catalogRetentionCount` declarative catalogs for a + * prefix family. Mirrors Go's `cleanupOldDeclarativeCatalogs` (`declarative.go:610`). + */ +export const legacyCleanupOldDeclarativeCatalogs = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + tempDir: string, + prefix: string, +) { + yield* cleanupOldCatalogsByFamily( + fs, + path, + tempDir, + `catalog-${legacySanitizedCatalogPrefix(prefix)}-declarative-`, + ); +}); + +/** + * Removes all but the newest `catalogRetentionCount` migrations catalogs for a + * prefix family. Mirrors Go's `pgcache.CleanupOldMigrationCatalogs` (`pgcache/cache.go`). + */ +export const legacyCleanupOldMigrationCatalogs = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + tempDir: string, + prefix: string, +) { + yield* cleanupOldCatalogsByFamily( + fs, + path, + tempDir, + `catalog-${legacySanitizedCatalogPrefix(prefix)}-migrations-`, + ); +}); + +/** `catalog--migrations--.json` (Go's `migrationsCatalogName`, `pgcache/cache.go`). */ +export function legacyMigrationCatalogFileName( + prefix: string, + hash: string, + timestampMillis: number, +): string { + return `catalog-${legacySanitizedCatalogPrefix(prefix)}-migrations-${hash}-${timestampMillis}.json`; +} + +/** + * Writes a migrations-catalog snapshot to `/catalog--migrations--.json` + * and prunes older snapshots for the same `(prefix)` family. Mirrors Go's + * `pgcache.WriteMigrationCatalogSnapshot` (`pgcache/cache.go`). + */ +export const legacyWriteMigrationCatalogSnapshot = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + tempDir: string, + prefix: string, + hash: string, + snapshot: string, + timestampMillis: number, +) { + yield* fs.makeDirectory(tempDir, { recursive: true }).pipe(Effect.ignore); + const filePath = path.join( + tempDir, + legacyMigrationCatalogFileName(prefix, hash, timestampMillis), + ); + yield* fs.writeFileString(filePath, snapshot); + yield* legacyCleanupOldMigrationCatalogs(fs, path, tempDir, prefix); + return filePath; +}); + +/** + * Best-effort caches the migrations catalog for pg-delta after a successful + * `db push` migration apply. Mirrors Go's `pgcache.TryCacheMigrationsCatalog` + * (`pgcache/cache.go`); `enabled` is resolved by the caller since it depends on + * already-loaded config. Reuses `legacyExportCatalogPgDelta` (Go's correct + * `diff/pgdelta.go` `ExportCatalogPgDelta`) rather than porting a second copy, + * so this can't reintroduce the `/workspace` mount bug `pgcache/cache.go` had + * (supabase/cli#5921). + */ +export const legacyTryCacheMigrationsCatalog = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + ctx: LegacyPgDeltaContext, + params: { + readonly enabled: boolean; + readonly targetUrl: string; + readonly conn: { + readonly host: string; + readonly port: number; + readonly user: string; + readonly database: string; + }; + readonly isLocal: boolean; + readonly migrationsDir: string; + readonly nowMillis: number; + }, +) { + if (!params.enabled) return; + const prefix = legacyCatalogPrefixFromConfig(params.conn, params.isLocal); + const hash = yield* legacyHashMigrations(fs, path, ctx.cwd, params.migrationsDir); + const snapshot = yield* legacyExportCatalogPgDelta(ctx, { + targetRef: params.targetUrl, + role: "postgres", + }); + yield* legacyWriteMigrationCatalogSnapshot( + fs, + path, + legacyPgDeltaTempPath(path, ctx.cwd), + prefix, + hash, + snapshot, + params.nowMillis, + ); +}); diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.unit.test.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.unit.test.ts index 9f2e57e3aa..83535b91c0 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.unit.test.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.unit.test.ts @@ -13,15 +13,19 @@ import { legacyBaselineCatalogFileName, legacyBaselineCatalogKey, legacyBaselineVersionToken, + legacyCatalogPrefixFromConfig, legacyCleanupOldDeclarativeCatalogs, + legacyCleanupOldMigrationCatalogs, legacyDeclarativeCatalogCacheKey, legacyDeclarativeCatalogFileName, legacyHashDeclarativeSchemas, legacyHashMigrations, legacyListLocalMigrations, + legacyMigrationCatalogFileName, legacyResolveDeclarativeCatalogPath, legacySanitizedCatalogPrefix, legacySetupInputsToken, + legacyWriteMigrationCatalogSnapshot, } from "./legacy-pgdelta.cache.ts"; const BASE: LegacySetupInputs = { @@ -216,25 +220,57 @@ describe("legacyListLocalMigrations", () => { }); describe("legacyHashMigrations", () => { - it.effect("hashes path + contents in list order (stable, content-sensitive)", () => { - const dir = withTemp(); - const migrationsDir = join(dir, "supabase", "migrations"); - mkdirSync(migrationsDir, { recursive: true }); - const file = join(migrationsDir, "20240101120000_create.sql"); - writeFileSync(file, "create table x();"); - const expected = createHash("sha256") - .update(file, "utf8") - .update(Buffer.from("create table x();")) - .digest("hex"); - return withServices((fs, path) => legacyHashMigrations(fs, path, migrationsDir)).pipe( - Effect.tap((hash) => - Effect.sync(() => { - expect(hash).toBe(expected); - rmSync(dir, { recursive: true, force: true }); + it.effect( + "hashes the workdir-relative path + contents in list order (stable, content-sensitive)", + () => { + const dir = withTemp(); + const migrationsDir = join(dir, "supabase", "migrations"); + mkdirSync(migrationsDir, { recursive: true }); + const file = join(migrationsDir, "20240101120000_create.sql"); + writeFileSync(file, "create table x();"); + const relPath = join("supabase", "migrations", "20240101120000_create.sql"); + const expected = createHash("sha256") + .update(relPath, "utf8") + .update(Buffer.from("create table x();")) + .digest("hex"); + return withServices((fs, path) => legacyHashMigrations(fs, path, dir, migrationsDir)).pipe( + Effect.tap((hash) => + Effect.sync(() => { + expect(hash).toBe(expected); + rmSync(dir, { recursive: true, force: true }); + }), + ), + ); + }, + ); + + it.effect( + "is unaffected by the absolute location of workdir (Go-parity, not machine-specific)", + () => { + const dirA = withTemp(); + const dirB = withTemp(); + const migrationsA = join(dirA, "supabase", "migrations"); + const migrationsB = join(dirB, "supabase", "migrations"); + mkdirSync(migrationsA, { recursive: true }); + mkdirSync(migrationsB, { recursive: true }); + writeFileSync(join(migrationsA, "20240101120000_create.sql"), "create table x();"); + writeFileSync(join(migrationsB, "20240101120000_create.sql"), "create table x();"); + return withServices((fs, path) => + Effect.gen(function* () { + const hashA = yield* legacyHashMigrations(fs, path, dirA, migrationsA); + const hashB = yield* legacyHashMigrations(fs, path, dirB, migrationsB); + expect(hashA).toBe(hashB); }), - ), - ); - }); + ).pipe( + Effect.tap(() => + Effect.sync(() => { + rmSync(dirA, { recursive: true, force: true }); + rmSync(dirB, { recursive: true, force: true }); + }), + ), + ); + }, + ); }); describe("legacyHashDeclarativeSchemas", () => { @@ -281,7 +317,6 @@ describe("legacyResolveDeclarativeCatalogPath + cleanup", () => { const remaining = (yield* fs.readDirectory(tempDir)).filter((n) => n.startsWith("catalog-local-declarative-"), ); - // Retention keeps the 2 newest of the family (300, 200); 100 + other-50 pruned. expect(remaining.sort()).toEqual([ "catalog-local-declarative-h-200.json", "catalog-local-declarative-h-300.json", @@ -290,3 +325,119 @@ describe("legacyResolveDeclarativeCatalogPath + cleanup", () => { ).pipe(Effect.tap(() => Effect.sync(() => rmSync(dir, { recursive: true, force: true })))); }); }); + +describe("legacyCatalogPrefixFromConfig", () => { + const CONN = { host: "127.0.0.1", port: 5432, user: "postgres", database: "postgres" }; + + it("returns 'local' for a local database regardless of host", () => { + expect(legacyCatalogPrefixFromConfig(CONN, true)).toBe("local"); + }); + + it("returns the project ref for a direct db..supabase.{co,red} host", () => { + const ref = "abcdefghijklmnopqrst"; + expect(legacyCatalogPrefixFromConfig({ ...CONN, host: `db.${ref}.supabase.co` }, false)).toBe( + ref, + ); + expect(legacyCatalogPrefixFromConfig({ ...CONN, host: `db.${ref}.supabase.red` }, false)).toBe( + ref, + ); + }); + + it("falls back to a stable url- hash for anything else", () => { + const conn = { + host: "aws-0-us-east-1.pooler.supabase.com", + port: 6543, + user: "postgres.ref", + database: "postgres", + }; + expect(legacyCatalogPrefixFromConfig(conn, false)).toBe( + `url-${sha12(`${conn.user}@${conn.host}:${conn.port}/${conn.database}`)}`, + ); + }); + + it("does not match a host with the wrong ref length or a different TLD", () => { + const conn = { ...CONN, host: "db.tooshort.supabase.co" }; + const digest = createHash("sha256") + .update(`${conn.user}@${conn.host}:${conn.port}/${conn.database}`, "utf8") + .digest("hex"); + expect(legacyCatalogPrefixFromConfig(conn, false)).toBe(`url-${digest.slice(0, 12)}`); + }); +}); + +describe("legacyMigrationCatalogFileName", () => { + it("formats catalog--migrations--.json", () => { + expect(legacyMigrationCatalogFileName("local", "h", 1700)).toBe( + "catalog-local-migrations-h-1700.json", + ); + }); +}); + +describe("legacyWriteMigrationCatalogSnapshot + cleanup", () => { + it.effect( + "writes the snapshot and prunes older migrations catalogs past the retention count", + () => { + const dir = withTemp(); + const tempDir = join(dir, "pgdelta"); + mkdirSync(tempDir, { recursive: true }); + for (const ts of [100, 300, 200]) { + writeFileSync(join(tempDir, `catalog-local-migrations-h-${ts}.json`), "{}"); + } + return withServices((fs, path) => + Effect.gen(function* () { + const filePath = yield* legacyWriteMigrationCatalogSnapshot( + fs, + path, + tempDir, + "local", + "h", + '{"snapshot":true}', + 400, + ); + expect(filePath.endsWith("catalog-local-migrations-h-400.json")).toBe(true); + expect(yield* fs.readFileString(filePath)).toBe('{"snapshot":true}'); + const remaining = (yield* fs.readDirectory(tempDir)).filter((n) => + n.startsWith("catalog-local-migrations-"), + ); + expect(remaining.sort()).toEqual([ + "catalog-local-migrations-h-300.json", + "catalog-local-migrations-h-400.json", + ]); + }), + ).pipe(Effect.tap(() => Effect.sync(() => rmSync(dir, { recursive: true, force: true })))); + }, + ); + + it.effect("creates the temp dir when it doesn't exist yet", () => { + const dir = withTemp(); + const tempDir = join(dir, "pgdelta"); + return withServices((fs, path) => + Effect.gen(function* () { + yield* legacyWriteMigrationCatalogSnapshot(fs, path, tempDir, "local", "h", "{}", 100); + expect(yield* fs.exists(join(tempDir, "catalog-local-migrations-h-100.json"))).toBe(true); + }), + ).pipe(Effect.tap(() => Effect.sync(() => rmSync(dir, { recursive: true, force: true })))); + }); +}); + +describe("legacyCleanupOldMigrationCatalogs", () => { + it.effect("only prunes files matching the given prefix's family", () => { + const dir = withTemp(); + const tempDir = join(dir, "pgdelta"); + mkdirSync(tempDir, { recursive: true }); + for (const ts of [100, 200, 300]) { + writeFileSync(join(tempDir, `catalog-local-migrations-h-${ts}.json`), "{}"); + } + writeFileSync(join(tempDir, "catalog-other-migrations-h-50.json"), "{}"); + return withServices((fs, path) => + Effect.gen(function* () { + yield* legacyCleanupOldMigrationCatalogs(fs, path, tempDir, "local"); + const remaining = (yield* fs.readDirectory(tempDir)).sort(); + expect(remaining).toEqual([ + "catalog-local-migrations-h-200.json", + "catalog-local-migrations-h-300.json", + "catalog-other-migrations-h-50.json", + ]); + }), + ).pipe(Effect.tap(() => Effect.sync(() => rmSync(dir, { recursive: true, force: true })))); + }); +});