fix(webapp): correct mint-flag grace baseline and lock down derived fields

d-cs · d-cs · commit ce00de1665d6 · 2026-07-10T17:18:01.000+01:00
An org's first per-org mint-flag override computed its cutover grace window against the wrong baseline, which could skip the grace window and briefly let two concurrent triggers sharing an idempotency key resolve to different databases. Seed the baseline from the effective global flag, strip the derived stamp fields from the request body, and apply the read-and-stamp atomically.
diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts
@@ -6,8 +6,12 @@ import { env } from "~/env.server";
 import { prisma } from "~/db.server";
 import { requireAdminApiRequest } from "~/services/personalAccessToken.server";
 import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
-import { stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace";
+import {
+  selectMintBaselineSource,
+  stampMintKindFlip,
+} from "~/v3/runOpsMigration/mintFlipGrace";
 import { validatePartialFeatureFlags } from "~/v3/featureFlags";
+import { flags as getGlobalFlags } from "~/v3/featureFlags.server";
 
 const ParamsSchema = z.object({
   organizationId: z.string(),
@@ -51,20 +55,6 @@ export async function action({ request, params }: ActionFunctionArgs) {
 
   const { organizationId } = ParamsSchema.parse(params);
 
-  const organization = await prisma.organization.findFirst({
-    where: {
-      id: organizationId,
-    },
-    select: {
-      id: true,
-      featureFlags: true,
-    },
-  });
-
-  if (!organization) {
-    return json({ error: "Organization not found" }, { status: 404 });
-  }
-
   try {
     const body = await request.json();
 
@@ -80,40 +70,66 @@ export async function action({ request, params }: ActionFunctionArgs) {
       );
     }
 
-    // Merge new flags with existing flags
-    const existingFlags = organization.featureFlags
-      ? validatePartialFeatureFlags(organization.featureFlags as Record<string, unknown>)
-      : { success: false as const };
-
-    // Stamp the flip from the control-plane DB clock so the grace-window cutover is anchored to one
-    // authoritative time source, not whichever webapp process happened to handle this request.
-    const [{ now: controlPlaneNow }] = await prisma.$queryRaw<{ now: Date }[]>`SELECT now() AS now`;
-
-    const mergedFlags = stampMintKindFlip(
-      existingFlags.success ? existingFlags.data : {},
-      {
-        ...(existingFlags.success ? existingFlags.data : {}),
-        ...validationResult.data,
-      },
-      controlPlaneNow.getTime(),
-      env.RUN_OPS_MINT_FLIP_GRACE_MS
-    );
+    // Derived grace-stamp fields are computed server-side; never trust them from the body.
+    const {
+      runOpsMintKindPrev: _ignoredPrev,
+      runOpsMintKindFlippedAt: _ignoredFlippedAt,
+      ...requestedFlags
+    } = validationResult.data;
+
+    // Seed the flip baseline from the current GLOBAL mint flags so an org's FIRST per-org override
+    // is graced from the currently-effective global kind, not the hardcoded default "cuid".
+    const globalFlags = (await getGlobalFlags()) as Record<string, unknown>;
+
+    // Lock the org row for the whole read -> merge -> stamp -> write so a concurrent flag save
+    // can't clobber the grace metadata (read-then-write race). PK lookup, one row, held to commit.
+    const updatedOrganization = await prisma.$transaction(async (tx) => {
+      const rows = await tx.$queryRaw<{ featureFlags: unknown }[]>`
+        SELECT "featureFlags" FROM "Organization" WHERE "id" = ${organizationId} FOR UPDATE`;
+
+      if (rows.length === 0) {
+        return null;
+      }
+
+      const existingRaw = rows[0].featureFlags as Record<string, unknown> | null;
+      const existingResult = existingRaw
+        ? validatePartialFeatureFlags(existingRaw)
+        : ({ success: false } as const);
+      const existingData = existingResult.success ? existingResult.data : {};
+
+      // Stamp the flip from the control-plane DB clock so the grace-window cutover is anchored to
+      // one authoritative time source, not whichever webapp process handled this request.
+      const [{ now: controlPlaneNow }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`;
+
+      const mergedFlags = stampMintKindFlip(
+        selectMintBaselineSource(existingRaw, globalFlags),
+        {
+          ...existingData,
+          ...requestedFlags,
+        },
+        controlPlaneNow.getTime(),
+        env.RUN_OPS_MINT_FLIP_GRACE_MS
+      );
 
-    // Update the organization's feature flags
-    const updatedOrganization = await prisma.organization.update({
-      where: {
-        id: organizationId,
-      },
-      data: {
-        featureFlags: mergedFlags as Prisma.InputJsonValue,
-      },
-      select: {
-        id: true,
-        slug: true,
-        featureFlags: true,
-      },
+      return tx.organization.update({
+        where: {
+          id: organizationId,
+        },
+        data: {
+          featureFlags: mergedFlags as Prisma.InputJsonValue,
+        },
+        select: {
+          id: true,
+          slug: true,
+          featureFlags: true,
+        },
+      });
     });
 
+    if (!updatedOrganization) {
+      return json({ error: "Organization not found" }, { status: 404 });
+    }
+
     // Org feature flags are embedded in every env of the org; drop all its cached env rows.
     controlPlaneResolver.invalidateOrganization(organizationId);
 
diff --git a/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts b/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts
@@ -6,7 +6,10 @@ import { env } from "~/env.server";
 import { prisma } from "~/db.server";
 import { requireUser } from "~/services/session.server";
 import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
-import { stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace";
+import {
+  selectMintBaselineSource,
+  stampMintKindFlip,
+} from "~/v3/runOpsMigration/mintFlipGrace";
 import { flags as getGlobalFlags } from "~/v3/featureFlags.server";
 import {
   FEATURE_FLAG,
@@ -105,47 +108,78 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return json({ error: "Invalid JSON body" }, { status: 400 });
   }
 
-  let featureFlags: typeof Prisma.JsonNull | Record<string, unknown>;
-
   if (
     body === null ||
     (typeof body === "object" && !Array.isArray(body) && Object.keys(body).length === 0)
   ) {
-    featureFlags = Prisma.JsonNull;
-  } else {
-    const validationResult = validatePartialFeatureFlags(body as Record<string, unknown>);
-    if (!validationResult.success) {
-      return json(
-        { error: "Invalid feature flags", details: validationResult.error.issues },
-        { status: 400 }
-      );
+    // Clear all flags. No grace stamp (nothing to flip) and no read-then-write race.
+    try {
+      await prisma.organization.update({
+        where: { id: organizationId },
+        data: { featureFlags: Prisma.JsonNull },
+      });
+    } catch (e) {
+      if (e instanceof Prisma.PrismaClientKnownRequestError && e.code === "P2025") {
+        throw new Response("Organization not found", { status: 404 });
+      }
+      throw e;
     }
-    featureFlags = validationResult.data;
 
-    const existingOrg = await prisma.organization.findFirst({
-      where: { id: organizationId },
-      select: { featureFlags: true },
-    });
-    // Anchor the flip stamp to the control-plane DB clock (see the v1 route) rather than this process's.
-    const [{ now: controlPlaneNow }] = await prisma.$queryRaw<{ now: Date }[]>`SELECT now() AS now`;
-    featureFlags = stampMintKindFlip(
-      existingOrg?.featureFlags as Record<string, unknown> | null | undefined,
-      featureFlags,
+    controlPlaneResolver.invalidateOrganization(organizationId);
+    return json({ success: true });
+  }
+
+  const validationResult = validatePartialFeatureFlags(body as Record<string, unknown>);
+  if (!validationResult.success) {
+    return json(
+      { error: "Invalid feature flags", details: validationResult.error.issues },
+      { status: 400 }
+    );
+  }
+
+  // Derived grace-stamp fields are computed server-side; never trust them from the body.
+  const {
+    runOpsMintKindPrev: _ignoredPrev,
+    runOpsMintKindFlippedAt: _ignoredFlippedAt,
+    ...requestedFlags
+  } = validationResult.data;
+
+  // Seed the flip baseline from the current GLOBAL mint flags so an org's FIRST per-org override
+  // is graced from the currently-effective global kind, not the hardcoded default "cuid".
+  const globalFlags = (await getGlobalFlags()) as Record<string, unknown>;
+
+  // Lock the org row for the whole read -> stamp -> write so a concurrent flag save can't clobber
+  // the grace metadata (read-then-write race). PK lookup, one row, held to commit.
+  const updated = await prisma.$transaction(async (tx) => {
+    const rows = await tx.$queryRaw<{ featureFlags: unknown }[]>`
+      SELECT "featureFlags" FROM "Organization" WHERE "id" = ${organizationId} FOR UPDATE`;
+
+    if (rows.length === 0) {
+      return false;
+    }
+
+    const existingRaw = rows[0].featureFlags as Record<string, unknown> | null;
+
+    // Anchor the flip stamp to the control-plane DB clock (see the v1 route), not this process's.
+    const [{ now: controlPlaneNow }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`;
+
+    const stamped = stampMintKindFlip(
+      selectMintBaselineSource(existingRaw, globalFlags),
+      requestedFlags,
       controlPlaneNow.getTime(),
       env.RUN_OPS_MINT_FLIP_GRACE_MS
     );
-  }
 
-  try {
-    await prisma.organization.update({
+    await tx.organization.update({
       where: { id: organizationId },
-      data: { featureFlags: featureFlags as Prisma.InputJsonValue },
+      data: { featureFlags: stamped as Prisma.InputJsonValue },
     });
-  } catch (e) {
-    if (e instanceof Prisma.PrismaClientKnownRequestError && e.code === "P2025") {
-      throw new Response("Organization not found", { status: 404 });
-    }
-    throw e;
+
+    return true;
+  });
+
+  if (!updated) {
+    throw new Response("Organization not found", { status: 404 });
   }
 
   // Org feature flags are embedded in every env of the org; drop all its cached env rows.
diff --git a/apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts b/apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts
@@ -1,7 +1,9 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import {
   effectiveMintKind,
+  readMintResolution,
   resolveMintFlag,
+  selectMintBaselineSource,
   stampMintKindFlip,
   type MintFlagResolution,
 } from "./mintFlipGrace";
@@ -229,3 +231,92 @@ describe("resolveMintFlag", () => {
     });
   });
 });
+
+// #3b: an org's FIRST per-org runOpsMintKind override must be stamped against the currently
+// EFFECTIVE kind — the global FeatureFlag resolution when the org has no override yet — not the
+// hardcoded default "cuid". selectMintBaselineSource picks that same source (per-org blob if it
+// sets runOpsMintKind, else the global rows) so stampMintKindFlip's baseline (storedKind +
+// prev + carry-forward) is correct.
+describe("selectMintBaselineSource", () => {
+  it("returns the per-org blob when it sets runOpsMintKind (override owns the baseline)", () => {
+    const perOrg = { runOpsMintKind: "cuid" };
+    const global = { runOpsMintKind: "runOpsId" };
+    expect(selectMintBaselineSource(perOrg, global)).toBe(perOrg);
+  });
+
+  it("falls back to the global rows when the org has no runOpsMintKind override", () => {
+    const perOrg = { someOtherFlag: true };
+    const global = { runOpsMintKind: "runOpsId" };
+    expect(selectMintBaselineSource(perOrg, global)).toBe(global);
+  });
+
+  it("returns an empty record when neither source sets a kind", () => {
+    expect(selectMintBaselineSource(null, null)).toEqual({});
+    expect(selectMintBaselineSource({ someOtherFlag: true }, null)).toEqual({});
+  });
+});
+
+describe("first per-org override stamps prev against the effective GLOBAL kind (#3b)", () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  it("global=runOpsId, org's FIRST override -> cuid: genuine flip, prev=runOpsId, graced", () => {
+    const globalFlags = { runOpsMintKind: "runOpsId" };
+    const orgExisting = {}; // no per-org override yet
+    const outgoing = { runOpsMintKind: "cuid" };
+    const result = stampMintKindFlip(
+      selectMintBaselineSource(orgExisting, globalFlags),
+      outgoing,
+      T,
+      GRACE_MS
+    );
+
+    expect(result.runOpsMintKind).toBe("cuid");
+    // Previously stamped prev="cuid" (the hardcoded default) OR skipped the flip entirely; must
+    // now be "runOpsId" (the effective global kind) so the org serves it through the grace window.
+    expect(result.runOpsMintKindPrev).toBe("runOpsId");
+    expect(result.runOpsMintKindFlippedAt).toBe(new Date(T).toISOString());
+
+    const resolution = readMintResolution(result);
+    expect(effectiveMintKind(resolution, T + GRACE_MS - 1, GRACE_MS)).toBe("runOpsId");
+    expect(effectiveMintKind(resolution, T + GRACE_MS, GRACE_MS)).toBe("cuid");
+  });
+
+  it("global=runOpsId, org's FIRST override -> runOpsId (redundant): NOT a spurious flip, no phantom regression to cuid", () => {
+    const globalFlags = { runOpsMintKind: "runOpsId" };
+    const orgExisting = {};
+    const outgoing = { runOpsMintKind: "runOpsId" };
+    const result = stampMintKindFlip(
+      selectMintBaselineSource(orgExisting, globalFlags),
+      outgoing,
+      T,
+      GRACE_MS
+    );
+
+    expect(result.runOpsMintKind).toBe("runOpsId");
+    // Previously: storedKind defaulted to "cuid", so this looked like a genuine flip and stamped
+    // prev="cuid" — making the org serve cuid during a phantom grace window (a regression).
+    expect(result.runOpsMintKindPrev).toBeUndefined();
+    expect(result.runOpsMintKindFlippedAt).toBeUndefined();
+  });
+
+  it("mid-grace global flip carried into a redundant org override keeps the global stamp", () => {
+    const globalFlags = {
+      runOpsMintKind: "runOpsId",
+      runOpsMintKindPrev: "cuid",
+      runOpsMintKindFlippedAt: new Date(T).toISOString(),
+    };
+    const orgExisting = {};
+    const outgoing = { runOpsMintKind: "runOpsId" };
+    const result = stampMintKindFlip(
+      selectMintBaselineSource(orgExisting, globalFlags),
+      outgoing,
+      T + 10_000,
+      GRACE_MS
+    );
+
+    expect(result.runOpsMintKind).toBe("runOpsId");
+    expect(result.runOpsMintKindPrev).toBe("cuid");
+    expect(result.runOpsMintKindFlippedAt).toBe(new Date(T).toISOString());
+  });
+});
diff --git a/apps/webapp/app/v3/runOpsMigration/mintFlipGrace.ts b/apps/webapp/app/v3/runOpsMigration/mintFlipGrace.ts
@@ -58,6 +58,21 @@ export function resolveMintFlag(
   return readMintResolution(globalFlags);
 }
 
+// Picks the flag record that currently determines an org's effective mint kind: the per-org
+// override blob when it sets runOpsMintKind, otherwise the global FeatureFlag rows. Same source
+// resolveMintFlag() reads, but returned as a record so it can seed stampMintKindFlip's baseline
+// (storedKind for flip-detection, prev on a genuine flip, and stamp carry-forward). This makes an
+// org's first per-org override stamp against the effective GLOBAL kind, not the default "cuid".
+export function selectMintBaselineSource(
+  perOrgOverrides: Record<string, unknown> | null | undefined,
+  globalFlags: Record<string, unknown> | null | undefined
+): Record<string, unknown> {
+  if (readMintKind(perOrgOverrides ?? {}, "runOpsMintKind") !== undefined) {
+    return perOrgOverrides ?? {};
+  }
+  return globalFlags ?? {};
+}
+
 function resolveEffectiveFromFlags(
   flags: Record<string, unknown> | null | undefined,
   nowMs: number,
