From ec7d79906772a65d881b66b1c2bec3d9d423c36f Mon Sep 17 00:00:00 2001 From: Taylor Date: Wed, 5 Aug 2026 10:05:12 -0400 Subject: [PATCH 1/6] =?UTF-8?q?feat(migrate):=20label=20addition=20tracer?= =?UTF-8?q?=20=E2=80=94=20migrate=5Fupdates=20appends=20missing=20enum=20l?= =?UTF-8?q?abels=20(#330)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An existing ferro-owned Postgres enum type now learns model-declared labels it is missing: the reconciliation pass diffs declared labels against pg_enum (one decision table in ferro-ddl-lowering) and executes ALTER TYPE ... ADD VALUE IF NOT EXISTS per missing label as an autocommit pre-pass before the per-table transactions. Ownership is by derivation; append-only (ADR-0011). --- AGENTS.md | 10 +++++ crates/ferro-ddl-lowering/src/lib.rs | 53 ++++++++++++++++++++++ src/introspect.rs | 29 ++++++++++++ src/migrate.rs | 67 ++++++++++++++++++++++++++-- tests/test_label_addition.py | 52 +++++++++++++++++++++ 5 files changed, 208 insertions(+), 3 deletions(-) create mode 100644 tests/test_label_addition.py diff --git a/AGENTS.md b/AGENTS.md index 2c5a62a..cb7839a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,6 +50,16 @@ For a single model, every emitter must agree on: and `db_check_constraint_name` (Rust). 9. **Default values** — server-side defaults must serialize identically. 10. **Nullability** — must agree. +11. **Enum label additions** — the label-addition decision (which model-declared + labels a live enum type is missing, and which live labels are extra) and + the rendered `ALTER TYPE ... ADD VALUE IF NOT EXISTS` statement are decided + by ONE pair of functions: `ferro_ddl_lowering::missing_enum_labels` / + `extra_enum_labels` + `render_pg_enum_add_value`. The auto-migrate + reconciliation pass consumes them directly; the Alembic autogenerate + comparator consumes them over FFI (`_core._plan_enum_label_addition`) and + executes the byte-identical statements. Pinned by + `tests/test_cross_emitter_parity.py` and the ferro-ddl-lowering unit pins. + See ADR-0011 (append-only; update-gated; warn-never-act for extras). ### Why this invariant exists diff --git a/crates/ferro-ddl-lowering/src/lib.rs b/crates/ferro-ddl-lowering/src/lib.rs index 9d27b80..92e8308 100644 --- a/crates/ferro-ddl-lowering/src/lib.rs +++ b/crates/ferro-ddl-lowering/src/lib.rs @@ -498,6 +498,33 @@ pub fn render_pg_enum_create_type(type_name: &str, labels: &[String]) -> String ) } +/// The label-addition decision (ADR-0011): which model-declared labels a live +/// ferro-owned enum type is missing, in declared order. This is the single +/// decision table for enum label drift — the auto-migrate planner and the +/// Alembic autogenerate comparator both consume it mechanically (AGENTS.md +/// § I-1); neither side re-derives it. Append-only by construction: labels the +/// live type has but the model lacks are not this function's concern. +pub fn missing_enum_labels(declared: &[String], live: &[String]) -> Vec { + declared + .iter() + .filter(|label| !live.contains(label)) + .cloned() + .collect() +} + +/// One `ALTER TYPE ... ADD VALUE IF NOT EXISTS` for a label addition. +/// `IF NOT EXISTS` makes concurrent boots and shared-type replans harmless; +/// executed outside transactions (autocommit) so the statement is legal on +/// every supported Postgres version and the label is committed before any +/// table plan that references it. +pub fn render_pg_enum_add_value(type_name: &str, label: &str) -> String { + format!( + "ALTER TYPE {} ADD VALUE IF NOT EXISTS '{}'", + quote_ident(type_name), + label.replace('\'', "''"), + ) +} + /// Detect a refused conversion from a live column to a resolved storage /// target. Extends [`refused_scalar_conversion`] with the native-enum case: /// a live non-enum (varchar/text) column targeted at a native Postgres enum @@ -1464,6 +1491,32 @@ mod tests { ); } + #[test] + fn missing_enum_labels_returns_additions_in_declared_order() { + let declared = vec!["plaid".to_string(), "mx".to_string(), "teller".to_string()]; + let live = vec!["plaid".to_string()]; + assert_eq!(missing_enum_labels(&declared, &live), vec!["mx", "teller"]); + } + + #[test] + fn missing_enum_labels_empty_when_live_covers_declared() { + let declared = vec!["plaid".to_string()]; + let live = vec!["plaid".to_string(), "legacy".to_string()]; + assert!(missing_enum_labels(&declared, &live).is_empty()); + } + + #[test] + fn render_pg_enum_add_value_is_pinned_and_escapes() { + assert_eq!( + render_pg_enum_add_value("provider", "mx"), + "ALTER TYPE \"provider\" ADD VALUE IF NOT EXISTS 'mx'" + ); + assert_eq!( + render_pg_enum_add_value("od'd", "it's"), + "ALTER TYPE \"od'd\" ADD VALUE IF NOT EXISTS 'it''s'" + ); + } + #[test] fn render_pg_enum_create_type_escapes_quotes() { let sql = render_pg_enum_create_type("od'd", &["it's".to_string()]); diff --git a/src/introspect.rs b/src/introspect.rs index 1faf959..1dfcba8 100644 --- a/src/introspect.rs +++ b/src/introspect.rs @@ -192,6 +192,35 @@ pub async fn live_table_names( .collect()) } +/// Read every native enum type in the connected schema with its labels in +/// enum sort order: `type name → labels`. Postgres-only — SQLite has no native +/// enum types — and taken once per reconciliation run (label addition is +/// per-type, not per-table). Callers must guard on dialect. +pub async fn live_enum_type_labels( + engine: &EngineHandle, +) -> PyResult>> { + let sql = "SELECT t.typname::text AS type_name, e.enumlabel::text AS label \ + FROM pg_type t \ + JOIN pg_namespace n ON n.oid = t.typnamespace \ + JOIN pg_enum e ON e.enumtypid = t.oid \ + WHERE n.nspname = current_schema() \ + ORDER BY t.typname, e.enumsortorder"; + let rows = engine + .fetch_all_sql_unprepared(sql) + .await + .map_err(|e| introspection_error("pg_enum", "*", e))?; + let mut labels_by_type: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for row in &rows { + if let (Some(type_name), Some(label)) = + (row_string(row, "type_name"), row_string(row, "label")) + { + labels_by_type.entry(type_name).or_default().push(label); + } + } + Ok(labels_by_type) +} + /// Read the live single-column foreign-key constraints on `table`. pub async fn live_table_foreign_keys( engine: &EngineHandle, diff --git a/src/migrate.rs b/src/migrate.rs index e6b3576..4bd218c 100644 --- a/src/migrate.rs +++ b/src/migrate.rs @@ -12,15 +12,18 @@ //! matches a freshly created one (AGENTS.md § I-1). use crate::backend::EngineHandle; -use ferro_ddl_lowering::{Dialect, information_schema_to_db_type_token}; +use ferro_ddl_lowering::{ + Dialect, ResolvedStorage, information_schema_to_db_type_token, missing_enum_labels, + render_pg_enum_add_value, resolve_column_storage, +}; use ferro_migrate::{MigrationOp, emit_sql_with_ir, plan_from_ir}; use ferro_schema_ir::{ IrEnvelope, SchemaCheck, SchemaColumn, SchemaForeignKey, SchemaIndex, SchemaIrPayload, SchemaModel, SchemaUnique, }; use crate::introspect::{ - LiveColumn, LiveForeignKey, LiveIndex, live_table_columns, live_table_foreign_keys, - live_table_indexes, quote_ident, + LiveColumn, LiveForeignKey, LiveIndex, live_enum_type_labels, live_table_columns, + live_table_foreign_keys, live_table_indexes, quote_ident, sqlite_indexes_covering_column, }; use crate::schema::{internal_create_tables, order_models_for_migration}; @@ -426,6 +429,17 @@ pub async fn internal_migrate(engine: Arc, opts: MigrateOptions) - let mut warnings = Vec::new(); let mut ddl_ran = false; + // Label addition (ADR-0011): reconcile ferro-owned enum types before any + // table's plan. Per-type, not per-table (a shared StrEnum reconciles + // once), and outside the per-table transactions below — `ALTER TYPE ... + // ADD VALUE` is non-transactional before PG12 and its label is unusable + // until commit on PG12+; autocommit execution here means every label is + // committed before a table plan (e.g. a new column defaulting to it) + // can reference it. + if backend == Dialect::Postgres { + ddl_ran |= add_missing_enum_labels(&engine, &modelset).await?; + } + for (_name, model) in order_models_for_migration(schemas, &modelset) { let table_lower = model.table_name.clone(); let Some(live) = live_table_columns(&engine, &table_lower).await? else { @@ -556,6 +570,53 @@ pub async fn internal_migrate(engine: Arc, opts: MigrateOptions) - Ok(()) } +/// The reconciliation pass's label addition (ADR-0011; CONTEXT.md *label +/// addition*): append model-declared labels missing from live ferro-owned +/// enum types. A type is ferro-owned by *derivation* — its name is the one +/// model resolution produces — so the declared side of the diff is itself the +/// ownership test; live types with no model-derived counterpart are user-owned +/// and never touched. Live types absent entirely are the create pass's / ADD +/// COLUMN guard's concern, not label addition's. Returns whether DDL executed. +async fn add_missing_enum_labels( + engine: &EngineHandle, + modelset: &IrEnvelope, +) -> PyResult { + // Declared native enum types, deduped across models and columns in + // deterministic order (a shared StrEnum reconciles exactly once). + let mut declared: std::collections::BTreeMap> = Default::default(); + for model in &modelset.payload.models { + for col in &model.columns { + if let Ok(ResolvedStorage::PgEnum { type_name, labels }) = + resolve_column_storage(col, Dialect::Postgres) + { + declared.entry(type_name).or_insert(labels); + } + } + } + if declared.is_empty() { + return Ok(false); + } + + let live = live_enum_type_labels(engine).await?; + let mut ran = false; + for (type_name, labels) in &declared { + let Some(live_labels) = live.get(type_name) else { continue }; + for label in missing_enum_labels(labels, live_labels) { + let sql = render_pg_enum_add_value(type_name, &label); + engine.execute_sql_unprepared(&sql).await.map_err(|e| { + crate::errors::map_db_error( + &format!( + "Auto-migrate failed to add enum label '{label}' to type '{type_name}'" + ), + e, + ) + })?; + ran = true; + } + } + Ok(ran) +} + /// Manually run the auto-migrate pass against a connected engine. /// /// Mirrors `connect(auto_migrate=True, migrate_updates=..., migrate_destructive=...)` diff --git a/tests/test_label_addition.py b/tests/test_label_addition.py new file mode 100644 index 0000000..3864c87 --- /dev/null +++ b/tests/test_label_addition.py @@ -0,0 +1,52 @@ +"""Label addition (#329/#330): the reconciliation pass appends model-declared +labels missing from a live ferro-owned enum type. See ADR-0011 and CONTEXT.md +(*enum label*, *label addition*).""" + +from enum import StrEnum + +import pytest + +import ferro +from ferro import Model, connect, engines, reset_engine +from ferro.raw import execute, fetch_all + +pytestmark = [pytest.mark.backend_matrix, pytest.mark.postgres_only] + + +@pytest.mark.asyncio +async def test_migrate_updates_appends_missing_label_and_member_round_trips( + db_url, clean_registry +): + """#328's repro, fixed at the correct gate: a StrEnum grown after the type + was created becomes usable on the next migrate_updates=True boot.""" + # An old deployment: the type was created when the StrEnum had one member. + await connect(db_url) + async with engines.session(): + await execute("CREATE TYPE \"provider\" AS ENUM ('plaid')") + await execute( + 'CREATE TABLE "feed" (' + '"id" serial PRIMARY KEY, "provider" "provider" NOT NULL)' + ) + await execute("INSERT INTO \"feed\" (\"provider\") VALUES ('plaid')") + reset_engine() + + class Provider(StrEnum): + PLAID = "plaid" + MX = "mx" + + class Feed(Model): + id: int | None = ferro.Field(primary_key=True, default=None) + provider: Provider + + await connect(db_url, migrate_updates=True) + async with engines.session(): + created = await Feed.create(provider=Provider.MX) + assert created.provider is Provider.MX + + fetched = await Feed.where(lambda f: f.provider == Provider.MX).all() + assert len(fetched) == 1 + assert fetched[0].provider is Provider.MX + + # The old deployment's row is untouched. + rows = await fetch_all('SELECT "provider" FROM "feed" ORDER BY "id"') + assert [r["provider"] for r in rows] == ["plaid", "mx"] From 6246b63a6368326fdda22f2e3e12cc689688f781 Mon Sep 17 00:00:00 2001 From: Taylor Date: Wed, 5 Aug 2026 10:06:49 -0400 Subject: [PATCH 2/6] feat(migrate): warn-never-act for extra enum labels; create pass stays silent (#331) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live labels the model no longer declares warn loudly — naming the type, the labels, and the reviewed-migration exit — and are never removed (rows may still hold them; old code may still run against the schema mid-deploy). Exactly one warning per drifted type. Plain auto_migrate remains inert and warning-free with drift in either direction, pinned by test (ADR-0011). --- crates/ferro-ddl-lowering/src/lib.rs | 53 ++++++++++++++++++++ src/migrate.rs | 15 ++++-- tests/test_label_addition.py | 73 ++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 3 deletions(-) diff --git a/crates/ferro-ddl-lowering/src/lib.rs b/crates/ferro-ddl-lowering/src/lib.rs index 92e8308..020db14 100644 --- a/crates/ferro-ddl-lowering/src/lib.rs +++ b/crates/ferro-ddl-lowering/src/lib.rs @@ -512,6 +512,36 @@ pub fn missing_enum_labels(declared: &[String], live: &[String]) -> Vec .collect() } +/// The warn-never-act half of the label-addition decision (ADR-0011): labels +/// the live type carries that the model no longer declares, in live (enum +/// sort) order. Rows may still hold these labels and older code may still be +/// running against the schema, so callers warn loudly and never remove — +/// removal and rename are reviewed-migration territory. +pub fn extra_enum_labels(declared: &[String], live: &[String]) -> Vec { + live.iter() + .filter(|label| !declared.contains(label)) + .cloned() + .collect() +} + +/// The warn-never-act message for one drifted enum type, or `None` when +/// nothing is extra. Single-sourced like [`refused_conversion_warning`]: +/// callers emit it verbatim, never re-derive the wording. +pub fn extra_enum_labels_warning(type_name: &str, extra: &[String]) -> Option { + if extra.is_empty() { + return None; + } + let listed: Vec = extra.iter().map(|l| format!("'{l}'")).collect(); + Some(format!( + "Enum type '{}' has label(s) {} that the model no longer declares. \ + Label addition is append-only: ferro never removes enum labels \ + (existing rows may still hold them). Remove or rename labels with \ + a reviewed Alembic migration.", + type_name, + listed.join(", ") + )) +} + /// One `ALTER TYPE ... ADD VALUE IF NOT EXISTS` for a label addition. /// `IF NOT EXISTS` makes concurrent boots and shared-type replans harmless; /// executed outside transactions (autocommit) so the statement is legal on @@ -1505,6 +1535,29 @@ mod tests { assert!(missing_enum_labels(&declared, &live).is_empty()); } + #[test] + fn extra_enum_labels_returns_undeclared_live_labels_in_live_order() { + let declared = vec!["plaid".to_string()]; + let live = vec!["legacy".to_string(), "plaid".to_string(), "old".to_string()]; + assert_eq!(extra_enum_labels(&declared, &live), vec!["legacy", "old"]); + assert!(extra_enum_labels(&live, &live).is_empty()); + } + + #[test] + fn extra_enum_labels_warning_is_pinned_and_names_the_exit() { + assert_eq!( + extra_enum_labels_warning("provider", &["legacy".to_string()]), + Some( + "Enum type 'provider' has label(s) 'legacy' that the model no longer \ + declares. Label addition is append-only: ferro never removes enum \ + labels (existing rows may still hold them). Remove or rename labels \ + with a reviewed Alembic migration." + .to_string() + ) + ); + assert_eq!(extra_enum_labels_warning("provider", &[]), None); + } + #[test] fn render_pg_enum_add_value_is_pinned_and_escapes() { assert_eq!( diff --git a/src/migrate.rs b/src/migrate.rs index 4bd218c..7762249 100644 --- a/src/migrate.rs +++ b/src/migrate.rs @@ -13,8 +13,9 @@ use crate::backend::EngineHandle; use ferro_ddl_lowering::{ - Dialect, ResolvedStorage, information_schema_to_db_type_token, missing_enum_labels, - render_pg_enum_add_value, resolve_column_storage, + Dialect, ResolvedStorage, extra_enum_labels, extra_enum_labels_warning, + information_schema_to_db_type_token, missing_enum_labels, render_pg_enum_add_value, + resolve_column_storage, }; use ferro_migrate::{MigrationOp, emit_sql_with_ir, plan_from_ir}; use ferro_schema_ir::{ @@ -437,7 +438,7 @@ pub async fn internal_migrate(engine: Arc, opts: MigrateOptions) - // committed before a table plan (e.g. a new column defaulting to it) // can reference it. if backend == Dialect::Postgres { - ddl_ran |= add_missing_enum_labels(&engine, &modelset).await?; + ddl_ran |= add_missing_enum_labels(&engine, &modelset, &mut warnings).await?; } for (_name, model) in order_models_for_migration(schemas, &modelset) { @@ -580,6 +581,7 @@ pub async fn internal_migrate(engine: Arc, opts: MigrateOptions) - async fn add_missing_enum_labels( engine: &EngineHandle, modelset: &IrEnvelope, + warnings: &mut Vec, ) -> PyResult { // Declared native enum types, deduped across models and columns in // deterministic order (a shared StrEnum reconciles exactly once). @@ -601,6 +603,13 @@ async fn add_missing_enum_labels( let mut ran = false; for (type_name, labels) in &declared { let Some(live_labels) = live.get(type_name) else { continue }; + // Warn-never-act (ADR-0011): live labels the model no longer declares + // are named loudly — rows may still hold them — but never removed. + // Once per drifted type, not per table referencing it. + let extra = extra_enum_labels(labels, live_labels); + if let Some(warning) = extra_enum_labels_warning(type_name, &extra) { + warnings.push(warning); + } for label in missing_enum_labels(labels, live_labels) { let sql = render_pg_enum_add_value(type_name, &label); engine.execute_sql_unprepared(&sql).await.map_err(|e| { diff --git a/tests/test_label_addition.py b/tests/test_label_addition.py index 3864c87..009a474 100644 --- a/tests/test_label_addition.py +++ b/tests/test_label_addition.py @@ -50,3 +50,76 @@ class Feed(Model): # The old deployment's row is untouched. rows = await fetch_all('SELECT "provider" FROM "feed" ORDER BY "id"') assert [r["provider"] for r in rows] == ["plaid", "mx"] + + +async def _live_labels(type_name: str) -> list[str]: + rows = await fetch_all( + "SELECT e.enumlabel AS label FROM pg_type t " + "JOIN pg_namespace n ON n.oid = t.typnamespace " + "JOIN pg_enum e ON e.enumtypid = t.oid " + f"WHERE n.nspname = current_schema() AND t.typname = '{type_name}' " + "ORDER BY e.enumsortorder" + ) + return [r["label"] for r in rows] + + +@pytest.mark.asyncio +async def test_extra_live_labels_warn_and_are_never_removed(db_url, clean_registry): + """Warn-never-act (ADR-0011): a live label the model no longer declares is + named in a UserWarning — with the reviewed-migration exit — and survives.""" + await connect(db_url) + async with engines.session(): + await execute("CREATE TYPE \"provider\" AS ENUM ('plaid', 'legacy')") + await execute( + 'CREATE TABLE "feed" (' + '"id" serial PRIMARY KEY, "provider" "provider" NOT NULL)' + ) + reset_engine() + + class Provider(StrEnum): + PLAID = "plaid" + + class Feed(Model): + id: int | None = ferro.Field(primary_key=True, default=None) + provider: Provider + + with pytest.warns(UserWarning, match=r"legacy") as record: + await connect(db_url, migrate_updates=True) + enum_warnings = [ + str(w.message) for w in record if "provider" in str(w.message) + ] + assert len(enum_warnings) == 1, "warning fires per drifted type, exactly once" + assert "'legacy'" in enum_warnings[0] + assert "Alembic" in enum_warnings[0], "warning names the reviewed-migration exit" + + async with engines.session(): + assert await _live_labels("provider") == ["plaid", "legacy"] + + +@pytest.mark.asyncio +async def test_plain_auto_migrate_stays_silent_and_inert_with_drift( + db_url, clean_registry, recwarn +): + """The create pass neither acts nor warns on enum drift in either + direction (ADR-0011): drift handling of every kind is migrate_updates'.""" + await connect(db_url) + async with engines.session(): + await execute("CREATE TYPE \"provider\" AS ENUM ('plaid', 'legacy')") + await execute( + 'CREATE TABLE "feed" (' + '"id" serial PRIMARY KEY, "provider" "provider" NOT NULL)' + ) + reset_engine() + + class Provider(StrEnum): + PLAID = "plaid" + MX = "mx" # missing live; 'legacy' is extra live: drift both ways + + class Feed(Model): + id: int | None = ferro.Field(primary_key=True, default=None) + provider: Provider + + await connect(db_url, auto_migrate=True) + assert not [w for w in recwarn if "provider" in str(w.message)] + async with engines.session(): + assert await _live_labels("provider") == ["plaid", "legacy"] From b259ace5b57cc45a31ffc6f75209ac370e45640e Mon Sep 17 00:00:00 2001 From: Taylor Date: Wed, 5 Aug 2026 10:07:54 -0400 Subject: [PATCH 3/6] =?UTF-8?q?test(migrate):=20label=20addition=20edges?= =?UTF-8?q?=20=E2=80=94=20shared=20types,=20default-in-same-run,=20orderin?= =?UTF-8?q?g,=20idempotence=20(#332)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins the production edges of label addition: a shared StrEnum reconciles once (one warning per drifted type, both tables usable); a new label used as a new column's literal-backfill default works in one run because the autocommit pre-pass commits the label before any table plan (the Prisma #8424 trap); appended labels sort last regardless of Python declaration order (documented ORDER BY caveat); a second boot replans to nothing. --- tests/test_label_addition.py | 157 +++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/tests/test_label_addition.py b/tests/test_label_addition.py index 009a474..d85ad05 100644 --- a/tests/test_label_addition.py +++ b/tests/test_label_addition.py @@ -123,3 +123,160 @@ class Feed(Model): assert not [w for w in recwarn if "provider" in str(w.message)] async with engines.session(): assert await _live_labels("provider") == ["plaid", "legacy"] + + +@pytest.mark.asyncio +async def test_shared_type_reconciles_exactly_once(db_url, clean_registry): + """A StrEnum shared by two models is one type and reconciles once: one + warning for its drift, and both tables accept the appended label.""" + await connect(db_url) + async with engines.session(): + await execute("CREATE TYPE \"provider\" AS ENUM ('plaid', 'legacy')") + await execute( + 'CREATE TABLE "feed" ("id" serial PRIMARY KEY, "provider" "provider" NOT NULL)' + ) + await execute( + 'CREATE TABLE "payout" ("id" serial PRIMARY KEY, "provider" "provider" NOT NULL)' + ) + reset_engine() + + class Provider(StrEnum): + PLAID = "plaid" + MX = "mx" + + class Feed(Model): + id: int | None = ferro.Field(primary_key=True, default=None) + provider: Provider + + class Payout(Model): + id: int | None = ferro.Field(primary_key=True, default=None) + provider: Provider + + with pytest.warns(UserWarning) as record: + await connect(db_url, migrate_updates=True) + per_type = [w for w in record if "'legacy'" in str(w.message)] + assert len(per_type) == 1, "one warning per drifted type, not per table" + + async with engines.session(): + assert (await Feed.create(provider=Provider.MX)).provider is Provider.MX + assert (await Payout.create(provider=Provider.MX)).provider is Provider.MX + + +@pytest.mark.asyncio +async def test_new_label_as_default_of_new_column_in_same_run(db_url, clean_registry): + """The single-deploy shape: add a member AND a new column defaulting to it. + The label commits (autocommit pre-pass) before the table plan references + it — the trap Prisma #8424 documents.""" + await connect(db_url) + async with engines.session(): + await execute("CREATE TYPE \"state\" AS ENUM ('open')") + await execute('CREATE TABLE "ticket" ("id" serial PRIMARY KEY)') + await execute('INSERT INTO "ticket" DEFAULT VALUES') + reset_engine() + + class State(StrEnum): + OPEN = "open" + CLOSED = "closed" + + class Ticket(Model): + id: int | None = ferro.Field(primary_key=True, default=None) + state: State = ferro.Field(default=State.CLOSED) + + await connect(db_url, migrate_updates=True) + async with engines.session(): + rows = await fetch_all('SELECT "state" FROM "ticket"') + assert [r["state"] for r in rows] == ["closed"], ( + "existing row backfilled with the appended label" + ) + assert (await Ticket.create()).state is State.CLOSED + + +@pytest.mark.asyncio +async def test_new_table_defaulting_to_new_label_of_existing_stale_type( + db_url, clean_registry +): + """A brand new table whose enum column defaults to a new member of an + existing stale type creates cleanly in one migrate_updates run: fresh + CREATE TABLE never renders server-side defaults (they are client-side), + so no create-pass statement can reference a label before label addition + lands it.""" + await connect(db_url) + async with engines.session(): + # The type exists (older model used it); the new table does not. + await execute("CREATE TYPE \"state\" AS ENUM ('open')") + reset_engine() + + class State(StrEnum): + OPEN = "open" + CLOSED = "closed" + + class Audit(Model): + id: int | None = ferro.Field(primary_key=True, default=None) + state: State = ferro.Field(default=State.CLOSED) + + await connect(db_url, migrate_updates=True) + async with engines.session(): + assert (await Audit.create()).state is State.CLOSED + assert await _live_labels("state") == ["open", "closed"] + + +@pytest.mark.asyncio +async def test_appended_labels_sort_last_regardless_of_declaration_order( + db_url, clean_registry +): + """Documented caveat: ADD VALUE appends, so enum ORDER BY follows database + order, not Python declaration order.""" + await connect(db_url) + async with engines.session(): + await execute("CREATE TYPE \"provider\" AS ENUM ('plaid')") + await execute( + 'CREATE TABLE "feed" ("id" serial PRIMARY KEY, "provider" "provider" NOT NULL)' + ) + reset_engine() + + class Provider(StrEnum): + MX = "mx" # declared first in Python... + PLAID = "plaid" + + class Feed(Model): + id: int | None = ferro.Field(primary_key=True, default=None) + provider: Provider + + await connect(db_url, migrate_updates=True) + async with engines.session(): + # ...but appended last in the database ordering. + assert await _live_labels("provider") == ["plaid", "mx"] + await Feed.create(provider=Provider.MX) + await Feed.create(provider=Provider.PLAID) + rows = await fetch_all('SELECT "provider" FROM "feed" ORDER BY "provider"') + assert [r["provider"] for r in rows] == ["plaid", "mx"] + + +@pytest.mark.asyncio +async def test_second_boot_is_a_noop(db_url, clean_registry, recwarn): + """Label addition is idempotent: a reconciled schema replans to nothing — + no statements, no warnings, labels and order untouched.""" + await connect(db_url) + async with engines.session(): + await execute("CREATE TYPE \"provider\" AS ENUM ('plaid')") + await execute( + 'CREATE TABLE "feed" ("id" serial PRIMARY KEY, "provider" "provider" NOT NULL)' + ) + reset_engine() + + class Provider(StrEnum): + PLAID = "plaid" + MX = "mx" + + class Feed(Model): + id: int | None = ferro.Field(primary_key=True, default=None) + provider: Provider + + await connect(db_url, migrate_updates=True) + reset_engine() + recwarn.clear() + + await connect(db_url, migrate_updates=True) + assert not [w for w in recwarn if "provider" in str(w.message)] + async with engines.session(): + assert await _live_labels("provider") == ["plaid", "mx"] From 4dbd29962751783735990e2fdd83a4721cb10985 Mon Sep 17 00:00:00 2001 From: Taylor Date: Wed, 5 Aug 2026 10:12:28 -0400 Subject: [PATCH 4/6] feat(alembic): autogenerate comparator emits label additions from the shared diff (#333) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bridge registers a schema comparator: named sa.Enum types in the target metadata diff against pg_enum on the connection, through the same Rust decision table auto-migrate consumes (_plan_enum_label_addition over FFI) — the generated revision executes byte-identical statements. Additions render inside op.get_context().autocommit_block(), inserted ahead of table ops so the label is committed before anything references it; extra live labels render as a warn-never-act comment naming the reviewed-migration exit. In-sync models generate nothing (no phantom diffs, AGENTS.md I-1); a cross-language pin holds the rendered statement byte-for-byte on both sides. --- src/ferro/_core.pyi | 12 +++ src/ferro/migrations/alembic.py | 139 ++++++++++++++++++++++++++++- src/lib.rs | 1 + src/naming_ffi.rs | 23 +++++ tests/test_cross_emitter_parity.py | 20 +++++ tests/test_label_addition.py | 106 +++++++++++++++++++++- 6 files changed, 296 insertions(+), 5 deletions(-) diff --git a/src/ferro/_core.pyi b/src/ferro/_core.pyi index 0e44185..3392a6e 100644 --- a/src/ferro/_core.pyi +++ b/src/ferro/_core.pyi @@ -221,3 +221,15 @@ def _resolve_storage_type(column_ir_json: str, dialect: str) -> str: def _render_check_body(column: str, values: list[str]) -> str: """The shared db_check CHECK body, byte-identical to the Rust emitters.""" ... + +def _plan_enum_label_addition( + type_name: str, declared: list[str], live: list[str] +) -> str: + """The label-addition decision (ADR-0011) for one enum type. + + Returns JSON: ``{"statements": [...], "extra_labels": [...]}`` — the + Rust-rendered ``ADD VALUE IF NOT EXISTS`` statements for model-declared + labels the live type is missing, and the live labels the model no longer + declares (warn-never-act). + """ + ... diff --git a/src/ferro/migrations/alembic.py b/src/ferro/migrations/alembic.py index 12b7d16..d72a573 100644 --- a/src/ferro/migrations/alembic.py +++ b/src/ferro/migrations/alembic.py @@ -7,7 +7,12 @@ sa = None from .._annotation_utils import _VARCHAR_RE -from .._core import _ddl_fk_name, _render_check_body, _resolve_storage_type +from .._core import ( + _ddl_fk_name, + _plan_enum_label_addition, + _render_check_body, + _resolve_storage_type, +) #: SQLAlchemy ``naming_convention`` mirroring the Rust emitter's names. IR-backed #: metadata names every artifact explicitly; this convention covers any @@ -271,3 +276,135 @@ def _db_type_to_sa_type(token: str) -> "sa.types.TypeEngine | None": except ValueError: return None return None + + +# --------------------------------------------------------------------------- +# Label addition comparator (ADR-0011; CONTEXT.md *label addition*). +# +# Alembic core is blind to enum label drift: autogenerate against a database +# whose native enum type is missing a model-declared label produces an empty +# revision, and the first use of the new member fails at runtime (#328). This +# comparator is the second mechanical consumer of the label-addition decision +# table (AGENTS.md § I-1): the diff AND the rendered statements come from the +# Rust core over FFI (`_plan_enum_label_addition`), byte-identical to what the +# auto-migrate reconciliation pass executes. +# +# There is no `migrate_updates` gate here — running autogenerate is itself the +# request for a diff; parity is in the decision, not the gate. The generated +# ops render inside `op.get_context().autocommit_block()` so the revision is +# legal on every supported Postgres version and the label is committed before +# any table op that references it; label additions are inserted ahead of the +# revision's table ops for the same reason. Extra live labels render as a +# comment (warn-never-act): removal is reviewed-migration territory. +# --------------------------------------------------------------------------- + +try: + from alembic.autogenerate import comparators as _alembic_comparators + from alembic.autogenerate import renderers as _alembic_renderers + from alembic.operations.ops import MigrateOperation as _MigrateOperation +except ImportError: # pragma: no cover - alembic optional at import time + _alembic_comparators = None + + +if _alembic_comparators is not None: + + class AddEnumLabelsOp(_MigrateOperation): + """Autogenerate carrier for one ferro-owned enum type's label drift. + + Renders to plain ``op.execute`` calls — a generated revision does not + import ferro to run. + """ + + def __init__( + self, + type_name: str, + statements: list[str], + extra_labels: list[str], + ) -> None: + self.type_name = type_name + self.statements = statements + self.extra_labels = extra_labels + + def to_diff_tuple(self): + return ( + "ferro_add_enum_labels", + self.type_name, + tuple(self.statements), + tuple(self.extra_labels), + ) + + def reverse(self): + # Labels cannot be removed in place; the downgrade is a no-op by + # the same warn-never-act contract that governs upgrades. + return AddEnumLabelsOp(self.type_name, [], []) + + @_alembic_comparators.dispatch_for("schema") + def _compare_enum_labels(autogen_context, upgrade_ops, schemas) -> None: + if autogen_context.dialect.name != "postgresql": + return + metadata = autogen_context.metadata + if metadata is None: + return + + # Declared native enum types: the named sa.Enum types the bridge maps + # from the shared storage decision (`_sa_type_from_ir_column`). + declared: dict[str, list[str]] = {} + for table in metadata.tables.values(): + for column in table.columns: + if isinstance(column.type, sa.Enum) and column.type.name: + declared.setdefault(str(column.type.name), list(column.type.enums)) + if not declared: + return + + # Live labels per type in enum sort order — the same catalog read the + # reconciliation pass takes, scoped to the connection's schema. + rows = autogen_context.connection.execute( + sa.text( + "SELECT t.typname AS type_name, e.enumlabel AS label " + "FROM pg_type t " + "JOIN pg_namespace n ON n.oid = t.typnamespace " + "JOIN pg_enum e ON e.enumtypid = t.oid " + "WHERE n.nspname = current_schema() " + "ORDER BY t.typname, e.enumsortorder" + ) + ).fetchall() + live: dict[str, list[str]] = {} + for row in rows: + live.setdefault(row.type_name, []).append(row.label) + + # A live type with no model-derived counterpart is user-owned; a + # declared type absent live belongs to table-creation ops. Insert + # drifted types ahead of the table ops, in deterministic order. + drifted = [] + for type_name in sorted(declared): + if type_name not in live: + continue + plan = json.loads( + _plan_enum_label_addition( + type_name, declared[type_name], live[type_name] + ) + ) + if plan["statements"] or plan["extra_labels"]: + drifted.append( + AddEnumLabelsOp(type_name, plan["statements"], plan["extra_labels"]) + ) + upgrade_ops.ops[:0] = drifted + + @_alembic_renderers.dispatch_for(AddEnumLabelsOp) + def _render_add_enum_labels(autogen_context, op: AddEnumLabelsOp) -> list[str]: + lines: list[str] = [] + if op.extra_labels: + listed = ", ".join(f"'{label}'" for label in op.extra_labels) + lines.append( + f"# ferro: enum type '{op.type_name}' has live label(s) {listed} " + "that the model no longer declares. Label addition is append-only " + "and never removes labels (rows may still hold them); remove or " + "rename them in a reviewed migration." + ) + if op.statements: + # Outside the migration transaction: ALTER TYPE ... ADD VALUE is + # non-transactional before PG12, and the label must be committed + # before any table op below can reference it. + lines.append("with op.get_context().autocommit_block():") + lines.extend(f" op.execute({stmt!r})" for stmt in op.statements) + return lines diff --git a/src/lib.rs b/src/lib.rs index de95302..7076871 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -156,6 +156,7 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(naming_ffi::_ddl_fk_name, m)?)?; m.add_function(wrap_pyfunction!(naming_ffi::_resolve_storage_type, m)?)?; m.add_function(wrap_pyfunction!(naming_ffi::_render_check_body, m)?)?; + m.add_function(wrap_pyfunction!(naming_ffi::_plan_enum_label_addition, m)?)?; m.add_function(wrap_pyfunction!( hydration::_verify_hydration_abi_for_test, m diff --git a/src/naming_ffi.rs b/src/naming_ffi.rs index 99512bc..16e38ba 100644 --- a/src/naming_ffi.rs +++ b/src/naming_ffi.rs @@ -79,6 +79,29 @@ pub fn _resolve_storage_type(column_ir_json: String, dialect: String) -> PyResul Ok(payload.to_string()) } +/// The label-addition decision over FFI (ADR-0011): given one enum type's +/// declared and live labels, return the Rust-rendered `ADD VALUE` statements +/// (in declared order) and the extra warn-never-act labels (in live order). +/// The Alembic autogenerate comparator consumes this instead of re-deriving +/// the diff or re-rendering the SQL (AGENTS.md § I-1) — the auto-migrate +/// planner and the generated revision execute byte-identical statements. +#[pyfunction] +pub fn _plan_enum_label_addition( + type_name: String, + declared: Vec, + live: Vec, +) -> String { + let statements: Vec = ferro_ddl_lowering::missing_enum_labels(&declared, &live) + .iter() + .map(|label| ferro_ddl_lowering::render_pg_enum_add_value(&type_name, label)) + .collect(); + serde_json::json!({ + "statements": statements, + "extra_labels": ferro_ddl_lowering::extra_enum_labels(&declared, &live), + }) + .to_string() +} + /// Render the shared `db_check` CHECK body (`"col" IN (v1, v2, ...)`) — /// byte-identical to the Rust emitters. `values` arrive pre-rendered (quoted) /// from the IR compiler. diff --git a/tests/test_cross_emitter_parity.py b/tests/test_cross_emitter_parity.py index c644d7d..8a1049d 100644 --- a/tests/test_cross_emitter_parity.py +++ b/tests/test_cross_emitter_parity.py @@ -294,3 +294,23 @@ async def test_alembic_autogen_after_migrate_updates_is_idempotent(db_url): "what Alembic expects of the current models.\n\n" f"Diff:\n{significant}" ) + + +def test_label_addition_statement_parity_pin(): + """Cross-language golden pin for label addition (AGENTS.md § I-1 item 11). + + The FFI returns the Rust-rendered ``ADD VALUE IF NOT EXISTS`` statement + byte-for-byte — the same literal is pinned in ferro-ddl-lowering's unit + tests, and the Alembic comparator executes it verbatim. If either side + drifts, the two migration doors would run different SQL for the same + model; this pin fails first. + """ + import json + + from ferro._core import _plan_enum_label_addition + + plan = json.loads( + _plan_enum_label_addition("provider", ["plaid", "mx"], ["plaid", "legacy"]) + ) + assert plan["statements"] == ["ALTER TYPE \"provider\" ADD VALUE IF NOT EXISTS 'mx'"] + assert plan["extra_labels"] == ["legacy"] diff --git a/tests/test_label_addition.py b/tests/test_label_addition.py index d85ad05..c3f914e 100644 --- a/tests/test_label_addition.py +++ b/tests/test_label_addition.py @@ -27,7 +27,7 @@ async def test_migrate_updates_appends_missing_label_and_member_round_trips( 'CREATE TABLE "feed" (' '"id" serial PRIMARY KEY, "provider" "provider" NOT NULL)' ) - await execute("INSERT INTO \"feed\" (\"provider\") VALUES ('plaid')") + await execute('INSERT INTO "feed" ("provider") VALUES (\'plaid\')') reset_engine() class Provider(StrEnum): @@ -85,9 +85,7 @@ class Feed(Model): with pytest.warns(UserWarning, match=r"legacy") as record: await connect(db_url, migrate_updates=True) - enum_warnings = [ - str(w.message) for w in record if "provider" in str(w.message) - ] + enum_warnings = [str(w.message) for w in record if "provider" in str(w.message)] assert len(enum_warnings) == 1, "warning fires per drifted type, exactly once" assert "'legacy'" in enum_warnings[0] assert "Alembic" in enum_warnings[0], "warning names the reviewed-migration exit" @@ -280,3 +278,103 @@ class Feed(Model): assert not [w for w in recwarn if "provider" in str(w.message)] async with engines.session(): assert await _live_labels("provider") == ["plaid", "mx"] + + +# --------------------------------------------------------------------------- +# Alembic comparator (#333): the second consumer of the label-addition +# decision (AGENTS.md § I-1) — autogenerate sees the same drift. +# --------------------------------------------------------------------------- + + +def _autogen_upgrade_code(postgres_base_url, db_schema_name): + """Run real autogenerate against the live per-test schema and render the + upgrade code, mirroring test_cross_emitter_parity.py's connection dance.""" + import sqlalchemy as sa + from alembic.autogenerate import produce_migrations, render_python_code + from alembic.migration import MigrationContext + + from ferro.migrations import get_metadata + + metadata = get_metadata() + for scheme in ("postgresql://", "postgres://"): + if postgres_base_url.startswith(scheme): + sync_url = "postgresql+psycopg://" + postgres_base_url[len(scheme) :] + break + else: + sync_url = postgres_base_url + engine = sa.create_engine(sync_url) + try: + with engine.connect() as conn: + conn.execute(sa.text(f'SET search_path TO "{db_schema_name}"')) + ctx = MigrationContext.configure( + conn, opts={"compare_type": True, "compare_server_default": True} + ) + script = produce_migrations(ctx, metadata) + return render_python_code(script.upgrade_ops) + finally: + engine.dispose() + + +@pytest.mark.asyncio +async def test_autogenerate_emits_label_additions_in_autocommit_block( + db_url, postgres_base_url, db_schema_name, clean_registry +): + """The reviewed-migration door sees the drift: the generated revision + carries the addition inside an autocommit block (runnable on every + supported PG version) and a comment for the warn-never-act direction.""" + await connect(db_url) + async with engines.session(): + await execute("CREATE TYPE \"provider\" AS ENUM ('plaid', 'legacy')") + await execute( + 'CREATE TABLE "feed" ("id" serial PRIMARY KEY, "provider" "provider" NOT NULL)' + ) + reset_engine() + + class Provider(StrEnum): + PLAID = "plaid" + MX = "mx" + + class Feed(Model): + id: int | None = ferro.Field(primary_key=True, default=None) + provider: Provider + + code = _autogen_upgrade_code(postgres_base_url, db_schema_name) + assert "autocommit_block" in code + # The op.execute payload is the Rust-rendered statement (repr-quoted by + # the renderer); assert on its stable substrings. + assert "ADD VALUE IF NOT EXISTS" in code + assert '"provider"' in code + assert "mx" in code + assert "legacy" in code, "extra live label surfaces as a revision comment" + assert "reviewed" in code, "comment names the reviewed-migration exit" + + +@pytest.mark.asyncio +async def test_autogenerate_emits_nothing_for_enums_in_sync( + db_url, postgres_base_url, db_schema_name, clean_registry +): + """No phantom diffs (AGENTS.md § I-1): model and database agree → the + comparator stays silent.""" + await connect(db_url) + async with engines.session(): + await execute("CREATE TYPE \"provider\" AS ENUM ('plaid', 'mx')") + await execute( + 'CREATE TABLE "feed" ("id" serial PRIMARY KEY, "provider" "provider" NOT NULL)' + ) + reset_engine() + + class Provider(StrEnum): + PLAID = "plaid" + MX = "mx" + + class Feed(Model): + id: int | None = ferro.Field(primary_key=True, default=None) + provider: Provider + + code = _autogen_upgrade_code(postgres_base_url, db_schema_name) + assert "ADD VALUE" not in code + assert "autocommit_block" not in code + + +# The cross-language statement parity pin lives in the canonical parity seam: +# tests/test_cross_emitter_parity.py::test_label_addition_statement_parity_pin. From bc7ae059b26a68929a0db2d8293c903317a27625 Mon Sep 17 00:00:00 2001 From: Taylor Date: Wed, 5 Aug 2026 10:13:53 -0400 Subject: [PATCH 5/6] =?UTF-8?q?docs(migrate):=20label=20addition=20?= =?UTF-8?q?=E2=80=94=20the=20auto=5Fmigrate=20trap,=20the=20migrate=5Fupda?= =?UTF-8?q?tes=20contract,=20the=20ordering=20caveat=20(#334)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents enum evolution end to end: the test-invisible trap (fresh schemas always get the complete label set, so only existing databases fail), the migrate_updates contract (append-only, warn-never-act, labels commit before table plans, shared types reconcile once), the ORDER BY caveat for appended labels, SQLite non-applicability, and the bridge's autogenerate comparator. Examples in both declaration styles with lambda predicates. --- docs/pages/guide/migrations.md | 71 ++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/pages/guide/migrations.md b/docs/pages/guide/migrations.md index 66a5ed3..081b28b 100644 --- a/docs/pages/guide/migrations.md +++ b/docs/pages/guide/migrations.md @@ -48,6 +48,8 @@ What it covers is capability-relative per backend: | Change column type | ⚠️ `UserWarning`, no DDL (SQLite type affinity makes drift mostly cosmetic) | ✅ `ALTER COLUMN ... TYPE ... USING` cast | | Change nullability | ⚠️ `UserWarning`, no DDL | ✅ `SET NOT NULL` / `DROP NOT NULL` | | Drop orphaned Ferro-named index (`idx_*` / `uq_*`) | ✅ with `migrate_destructive=True` | ✅ with `migrate_destructive=True` | +| Add a missing enum label (a `StrEnum` grew a member) | ✅ nothing to do — enums store as text | ✅ `ALTER TYPE ... ADD VALUE` *0.18.0+* | +| Remove or rename an enum label | ✅ nothing to do | ⚠️ `UserWarning`, no DDL — Alembic territory | | Inline single-column `UNIQUE` on existing column, index option changes | ❌ never — Alembic territory | ❌ never | | Rename column/table, change primary key, drop table | ❌ never — Alembic territory | ❌ never | @@ -59,6 +61,74 @@ Rules worth knowing: - **Postgres type changes take an exclusive lock** and fail the connect if existing data does not cast cleanly — fine for a development flag, but worth knowing. - **The pool refreshes after any schema change**, so no cached statement or stale identity-mapped instance can observe the pre-migration schema. +### Evolving enums: label addition + +*Added in 0.18.0.* On PostgreSQL, `StrEnum` fields create a **native enum type**, and a type that already exists in the database does not learn new members on its own. When a `StrEnum` grows, `migrate_updates=True` performs **label addition**: it compares the model's members against the live type and appends what's missing with `ALTER TYPE ... ADD VALUE IF NOT EXISTS`. + +!!! danger "This gap is invisible to your tests" + Under plain `auto_migrate=True` (without `migrate_updates`), an existing enum type is **never** updated — like every existing object, it belongs to the update pass. The failure mode is nasty: every test suite that creates its schema fresh gets the complete enum and stays green, while every *existing* database rejects the new member at runtime with `invalid input value for enum`. No app-side test against a throwaway schema can catch this. If your models' enums evolve, run with `migrate_updates=True` (or generate the migration with Alembic — the [autogenerate bridge](#alembic-for-production) sees the same drift). + +=== "Assignment" + + ```python + from enum import StrEnum + + import ferro + from ferro import Model + + + class Provider(StrEnum): + PLAID = "plaid" + MX = "mx" # new member — the live type only has 'plaid' + + + class Feed(Model): + id: int | None = ferro.Field(primary_key=True, default=None) + provider: Provider + + + await ferro.connect("postgres://...", migrate_updates=True) + # → ALTER TYPE "provider" ADD VALUE IF NOT EXISTS 'mx' + + feed = await Feed.create(provider=Provider.MX) + recent = await Feed.where(lambda feed: feed.provider == Provider.MX).all() + ``` + +=== "Annotated" + + ```python + from enum import StrEnum + from typing import Annotated + + import ferro + from ferro import FerroField, Model + + + class Provider(StrEnum): + PLAID = "plaid" + MX = "mx" # new member — the live type only has 'plaid' + + + class Feed(Model): + id: Annotated[int | None, FerroField(primary_key=True)] = None + provider: Provider + + + await ferro.connect("postgres://...", migrate_updates=True) + # → ALTER TYPE "provider" ADD VALUE IF NOT EXISTS 'mx' + + feed = await Feed.create(provider=Provider.MX) + recent = await Feed.where(lambda feed: feed.provider == Provider.MX).all() + ``` + +The contract, precisely: + +- **Append-only, metadata-only.** Label addition adds labels and does nothing else; rows are never touched. A shared `StrEnum` used by several models is one type and reconciles once. +- **Removals and renames are never automatic.** A live label the model no longer declares raises a `UserWarning` naming the type and labels — rows may still hold that label, and older code may still be running against the schema mid-deploy — and the label stays. Remove or rename labels in a reviewed Alembic migration. +- **Labels commit before table changes.** Additions run as their own autocommit statements ahead of the per-table plans, so a new column whose literal default is a brand-new member works in a single deploy, on every supported PostgreSQL version. +- **Appended labels sort last.** `ADD VALUE` appends: a member inserted mid-enum in Python lands at the end of the database ordering, and `ORDER BY` on an enum column follows *database* order, not declaration order. +- **SQLite is unaffected.** Enums store as text there; a new member needs no DDL. + ### Destructive drops with `migrate_destructive` *Added in 0.11.0.* Also **drop** live columns that no longer exist on the model (never whole tables): @@ -134,6 +204,7 @@ target_metadata = get_metadata() - **Composite constraints** (`__ferro_composite_uniques__`, `__ferro_composite_indexes__`) emit matching `UniqueConstraint` / `Index` objects, including the automatic constraints on many-to-many join tables. - **One-to-one** relations (`ForeignKey(unique=True)`) emit the same `UNIQUE` on the shadow column that `auto_migrate` creates at runtime. - **Enums** map to named `sqlalchemy.Enum` types (class name lowercased, e.g. `UserRole` → `userrole`) so revisions compile on PostgreSQL, which rejects anonymous enum types. +- **Enum label drift is diffed.** *0.18.0+.* Alembic core is blind to enum value changes; ferro's bridge registers an autogenerate comparator that diffs each named enum type against the live PostgreSQL catalog — the same decision (and the same rendered SQL) the auto-migrate pass uses. A grown `StrEnum` generates `ALTER TYPE ... ADD VALUE IF NOT EXISTS` inside an `autocommit_block()` (placed before table operations, runnable on every supported PostgreSQL version); a live label the model no longer declares generates a comment in the revision telling you removal needs a hand-written step. Models in sync generate nothing. ### Autogenerate From 1b37b1b4289072257348ac86438822a1052126bf Mon Sep 17 00:00:00 2001 From: Taylor Date: Wed, 5 Aug 2026 10:14:08 -0400 Subject: [PATCH 6/6] docs(adr): ADR-0011 label addition + glossary terms (enum label, label addition) Records the grilling decisions behind #329: update-gated (the ADR-0010 line extended to enum types), derived-name ownership, append-only with warn-never-act, one decision table for both migration doors, and the rejected alternatives. Glossary gains *enum label* and *label addition*; *Ferro-owned artifact* extends to derivation-based ownership for types. --- CONTEXT.md | 12 +++- ...label-addition-update-gated-append-only.md | 65 +++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 docs/adr/0011-label-addition-update-gated-append-only.md diff --git a/CONTEXT.md b/CONTEXT.md index c6ddf0c..e0b0874 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -129,13 +129,21 @@ The auto-migrate step that brings missing tables into existence — the table, i _Avoid_: Bootstrap, ensure-tables, table sync **Reconciliation pass**: -The `migrate_updates` step that alters existing tables to match the registered models — the only authority for DDL against a table that already exists. Within one table, column changes land before the indexes and constraints that reference them. +The `migrate_updates` step that alters existing schema objects — tables and ferro-owned enum types — to match the registered models; the only authority for DDL against an object that already exists. Within one table, column changes land before the indexes and constraints that reference them; label additions land before any table's changes. _Avoid_: Update pass, schema sync, drift repair **Ferro-owned artifact**: -An index or constraint whose name follows ferro's canonical naming (`idx_`, `uq_`, `fk_`, `ck_`), marking it as reconcilable: auto-migrate may create, rebuild, or drop it to match the declared model. Artifacts named any other way belong to the user and are never altered or dropped. +A schema object ferro may reconcile to match the declared model. Indexes and constraints are ferro-owned by naming (`idx_`, `uq_`, `fk_`, `ck_`); native enum types are ferro-owned by derivation — the type's name matches the name ferro derives from the model. Artifacts owned neither way belong to the user and are never altered or dropped. _Avoid_: Managed index, system constraint, internal index +**Enum label**: +One storable value of a native Postgres enum type, mirrored from a Python `StrEnum` member's value. Members are the Python-side declaration; labels are what the database accepts and stores. +_Avoid_: Enum value, variant, choice + +**Label addition**: +The reconciliation-pass operation appending model-declared labels missing from a live ferro-owned enum type. Append-only and metadata-only: rows are never touched, and labels the database has but the model lacks are warned about loudly and never removed — removal and rename are reviewed-migration territory. +_Avoid_: Enum sync, label reconciliation, enum evolution + **Constraint rebuild**: Drop-and-recreate of a ferro-owned constraint whose live definition no longer matches the declared model — a foreign key's `on_delete`, its target, its columns. Metadata-only: rows are never touched. On a backend that cannot alter constraints, ferro warns loudly and skips; it never diverges silently. _Avoid_: Constraint alter, FK patch, in-place constraint update diff --git a/docs/adr/0011-label-addition-update-gated-append-only.md b/docs/adr/0011-label-addition-update-gated-append-only.md new file mode 100644 index 0000000..7d45cab --- /dev/null +++ b/docs/adr/0011-label-addition-update-gated-append-only.md @@ -0,0 +1,65 @@ +# Label addition is update-gated and append-only + +A native Postgres enum type that already exists is an existing schema object, +so evolving its label set belongs to the **reconciliation pass** +(`migrate_updates`), not the create pass — the same line ADR-0010 drew for +indexes. The reconciliation pass performs **label addition** (`CONTEXT.md`): +it appends model-declared labels missing from a live ferro-owned enum type, +and does nothing else. Labels the database has but the model lacks are warned +about loudly and never removed — rows may still hold them, and old code may +still be running against the new schema during a rolling deploy. A type is +ferro-owned by **derivation**: its name matches the name ferro derives from +the model (enum types carry no `idx_`/`uq_`-style prefix, so prefix doctrine +cannot apply). The additive-only scope is what makes derivation-based +ownership safe: the worst misattribution appends a label; it never drops or +rewrites anything. + +Both migration doors consume one decision: the label diff (model labels + +live `pg_enum` labels → additions + warnings) lives in the Rust core and is +consumed mechanically by the auto-migrate planner and by an Alembic +autogenerate comparator in the bridge — the same one-decision-table seam as +`resolve_column_storage` (AGENTS.md I-1). Alembic autogenerate has no +`migrate_updates` gate: running autogenerate is itself the request for a +diff, so the comparator always reports; parity is in the decision, not the +gate. `ALTER TYPE ... ADD VALUE` executes outside transactions on both paths +(an autocommit pre-pass before the per-table transactions in auto-migrate; +`autocommit_block()` in generated revisions) — Prisma shipped the +in-transaction version and it is a graveyard (prisma#7251, #5290, #8424). + +Decision by owner (2026-08-05), grilling #328. + +Rejected alternatives: + +- **Ensure semantics under plain `auto_migrate`** ("the type guard already + runs in the create pass; label addition is just a stronger ensure"): + re-opens the two-owners divergence ADR-0010 closed — an existing object + whose shape plain `auto_migrate` sometimes changes. The create pass stays + introspection-free and silent about drift of every kind, enums included. +- **Automatic removal/rename** (full sync, as `alembic-postgresql-enum` + does): removal requires a type-replacement dance and human judgment about + rows holding the removed label; every implementation that automated it + grew a bug tracker around it. Reviewed-migration territory, permanently. +- **Recommending `alembic-postgresql-enum`** instead of our own comparator: + it autogenerates removals, contradicting append-only — the two doors would + disagree about what the same model means. +- **Ownership markers** (`COMMENT ON TYPE` stamped at create time): sound + provenance, but every existing deployment's types are unmarked, and the + backfill machinery buys nothing additive-only doesn't already guarantee. + +## Consequences + +- Under plain `auto_migrate=True`, an evolved `StrEnum` still fails at first + use with the database's `invalid input value for enum` error — silently at + boot, by design. The documented answer to drift, enum drift included, is + `migrate_updates=True`. Auto-migrate docs must carry this trap loudly: + fresh schemas always get the complete label set, so no app-side test + against a throwaway schema can catch it (#328). +- A new table created under plain `auto_migrate` that references an existing + stale type gets the stale label set — the create-pass type guard only + fires for missing types. Same flag fixes it. +- Appended labels land at the end of the Postgres enum ordering regardless of + their position in the Python declaration; `ORDER BY` on an enum column + follows database order, not declaration order. +- Live introspection must read `pg_enum` labels (today it records only + "is an enum"), and the cross-emitter parity tests extend to pin the label + diff across both consumers.