fix(webapp): anchor batch item run-ops residency to the batch friendlyId

On the v4 (RunEngine V2) batch paths, RunEngineBatchTriggerService (api.v2) and
the BatchQueue item callback (api.v3), a parentless batch's item runs minted
their run-ops residency from a fresh per-org flag read at processing time. A
mint-flag flip between batch creation and item processing could then mint an
item into a different physical store than its BatchTaskRun row: an FK violation
against the run-ops DB's live TaskRun_batchId_fkey (batch on legacy, item on
new), or a silent orphan (batch on new, item on legacy).

Anchor each item's mint on the batch's own friendlyId (pure id-shape, zero new
queries), mirroring the already-safe BatchTriggerV3Service, and anchor the
pre-failed-run fallback the same way since it also sets batchId. Consolidate the
shared id-generation branch into mintFriendlyIdForKind so every mint path stays
in lockstep. Single-DB mode is unchanged: a cuid-shaped batch friendlyId yields
a cuid item.

Author: d-cs

.server-changes/fix-batch-item-mint-anchor.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Anchor batch item run-ops residency on the batch's own friendlyId (not a fresh per-org flag read) in the run-engine batch trigger service and the BatchQueue item callback, on both the success path and the pre-failed-run failure path, so a mid-batch mint-flag flip can no longer mint an item (or a pre-failed item) into a different physical store than its BatchTaskRun row.

apps/webapp/app/runEngine/services/batchTrigger.server.ts

@@ -18,6 +18,7 @@ import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { batchTriggerWorker } from "~/v3/batchTriggerWorker.server";
 import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
+import { mintAnchoredRunFriendlyId } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
 import { mintBatchFriendlyId } from "~/v3/runOpsMigration/mintBatchFriendlyId.server";
 import {
   downloadPacketFromObjectStore,
@@ -588,6 +589,9 @@ export class RunEngineBatchTriggerService extends WithRunEngine {
           parentRunId,
           resumeParentOnCompletion,
           batch: { id: batch.id, index: workingIndex },
+          // Anchor the pre-failed run on the BATCH's residency (same as the happy path) so it
+          // co-resides with its BatchTaskRun row regardless of a mid-batch mint-flag flip.
+          runFriendlyId: mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region),
           options: item.options as Record<string, unknown>,
           traceContext: options?.traceContext as Record<string, unknown> | undefined,
           spanParentAsLink: options?.spanParentAsLink,
@@ -677,6 +681,12 @@ export class RunEngineBatchTriggerService extends WithRunEngine {
 
     const triggerTaskService = new TriggerTaskService();
 
+    // Anchor the item's mint on the BATCH's own friendlyId (not a fresh per-org flag read) so
+    // an org's mint flag flipping between batch creation and this (possibly much later,
+    // worker-processed) item never splits the item from its BatchTaskRun row. See
+    // mintAnchoredRunFriendlyId.server.ts.
+    const runFriendlyId = mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region);
+
     const result = await triggerTaskService.call(
       item.task,
       environment,
@@ -695,6 +705,7 @@ export class RunEngineBatchTriggerService extends WithRunEngine {
         spanParentAsLink: options?.spanParentAsLink,
         batchId: batch.id,
         batchIndex: currentIndex,
+        runFriendlyId,
         realtimeStreamsVersion: options?.realtimeStreamsVersion,
         triggerSource: options?.triggerSource ?? "api",
         triggerAction: options?.triggerAction ?? "trigger",

apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it, vi } from "vitest";
+
+// Empty singletons satisfy the module-level wiring imports; the mint method under test is driven
+// directly via (service as any) and never touches the DB (same boundary as triggerTask.server.test.ts).
+vi.mock("~/db.server", () => ({
+  prisma: {},
+  $replica: {},
+  runOpsNewPrisma: {},
+  runOpsLegacyPrisma: {},
+  runOpsNewReplica: {},
+  runOpsLegacyReplica: {},
+}));
+vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false }));
+
+import { classifyKind, generateRunOpsId, RunId } from "@trigger.dev/core/v3/isomorphic";
+import { TriggerFailedTaskService } from "./triggerFailedTask.server";
+
+function buildService() {
+  return new TriggerFailedTaskService({ prisma: {} as any, engine: {} as any });
+}
+
+describe("TriggerFailedTaskService.mintFailedRunFriendlyId", () => {
+  it("returns the caller-supplied runFriendlyId verbatim (override wins over any mint)", async () => {
+    const override = RunId.toFriendlyId(generateRunOpsId());
+    const minted = await (buildService() as any).mintFailedRunFriendlyId({
+      organizationId: "org_1",
+      environmentId: "env_1",
+      runFriendlyId: override,
+    });
+    expect(minted).toBe(override);
+  });
+
+  it("without an override, still inherits a run-ops (NEW) parent by id-shape", async () => {
+    const parentRunFriendlyId = RunId.toFriendlyId(generateRunOpsId());
+    const minted = await (buildService() as any).mintFailedRunFriendlyId({
+      organizationId: "org_1",
+      environmentId: "env_1",
+      parentRunFriendlyId,
+    });
+    expect(classifyKind(minted)).toBe("runOpsId");
+  });
+});

apps/webapp/app/runEngine/services/triggerFailedTask.server.ts

@@ -43,6 +43,9 @@ export type TriggerFailedTaskRequest = {
   spanParentAsLink?: boolean;
 
   errorCode?: TaskRunErrorCodes;
+
+  /** Pre-minted friendlyId; when set it wins over the mint. Batch callers pass a batch-anchored id. */
+  runFriendlyId?: string;
 };
 
 /**
@@ -86,14 +89,20 @@ export class TriggerFailedTaskService {
 
   // Mint a failed run's friendlyId. The id-kind decides which store the run is
   // born in (cuid → legacy store, run-ops id → new store); the whole subgraph of a
-  // run must agree. Root failed runs mint by the environment's setting; child
-  // failed runs inherit the parent's current store so they never split.
+  // run must agree. A caller-supplied runFriendlyId (batch-anchored id) wins verbatim;
+  // otherwise root failed runs mint by the environment's setting and child failed runs
+  // inherit the parent's current store so they never split.
   private async mintFailedRunFriendlyId(args: {
     organizationId: string;
     environmentId: string;
     orgFeatureFlags?: unknown;
     parentRunFriendlyId?: string;
+    runFriendlyId?: string;
   }): Promise<string> {
+    if (args.runFriendlyId) {
+      return args.runFriendlyId;
+    }
+
     const mintKind = args.parentRunFriendlyId
       ? resolveInheritedMintKind(args.parentRunFriendlyId)
       : await resolveRunIdMintKind({
@@ -125,6 +134,7 @@ export class TriggerFailedTaskService {
         environmentId: request.environment.id,
         orgFeatureFlags: request.environment.organization.featureFlags,
         parentRunFriendlyId: request.parentRunId,
+        runFriendlyId: request.runFriendlyId,
       });
       mintedFriendlyId = failedRunFriendlyId;
 

apps/webapp/app/runEngine/services/triggerTask.server.ts

@@ -14,7 +14,6 @@ import {
   TriggerTraceContext,
 } from "@trigger.dev/core/v3";
 import {
-  generateRunOpsId,
   parseTraceparent,
   RunId,
   serializeTraceparent,
@@ -28,6 +27,7 @@ import { handleMetadataPacket } from "~/utils/packets";
 import { startSpan } from "~/v3/tracing.server";
 import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
 import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server";
+import { mintFriendlyIdForKind } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
 import type {
   TriggerTaskServiceOptions,
   TriggerTaskServiceResult,
@@ -153,9 +153,7 @@ export class RunEngineTriggerTaskService {
           orgFeatureFlags: environment.organization.featureFlags,
         });
 
-    return mintKind === "runOpsId"
-      ? RunId.toFriendlyId(generateRunOpsId(region))
-      : RunId.generate().friendlyId;
+    return mintFriendlyIdForKind(mintKind, region);
   }
 
   public async call({

apps/webapp/app/v3/runEngineHandlers.server.ts

@@ -28,6 +28,7 @@ import { getEventRepositoryForStore, recordRunDebugLog } from "./eventRepository
 import { roomFromFriendlyRunId, socketIo } from "./handleSocketIo.server";
 import { engine } from "./runEngine.server";
 import { runStore } from "./runStore.server";
+import { mintAnchoredRunFriendlyId } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
 import { isSplitEnabled } from "~/v3/runOpsMigration/splitMode.server";
 import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server";
 import {
@@ -837,6 +838,12 @@ export function setupBatchQueueCallbacks() {
                 parentRunId: meta.parentRunId,
                 resumeParentOnCompletion: meta.resumeParentOnCompletion,
                 batch: { id: batchId, index: itemIndex },
+                // Anchor the pre-failed run on the BATCH's residency so it co-resides with its
+                // BatchTaskRun row regardless of a mid-batch mint-flag flip.
+                runFriendlyId: mintAnchoredRunFriendlyId(
+                  friendlyId,
+                  (item.options as { region?: string } | undefined)?.region
+                ),
                 traceContext: meta.traceContext as Record<string, unknown> | undefined,
                 spanParentAsLink: meta.spanParentAsLink,
               });
@@ -874,6 +881,14 @@ export function setupBatchQueueCallbacks() {
             // Normalize payload - for application/store (R2 paths), this passes through as-is
             const payload = normalizePayload(item.payload, item.payloadType);
 
+            // Anchor the item's mint on the BATCH's own friendlyId (not a fresh per-org flag
+            // read) so an org's mint flag flipping between batch creation and this queue-driven
+            // (possibly much later) callback never splits the item from its BatchTaskRun row.
+            const runFriendlyId = mintAnchoredRunFriendlyId(
+              friendlyId,
+              (item.options as { region?: string } | undefined)?.region
+            );
+
             const result = await triggerTaskService.call(
               item.task,
               environment,
@@ -893,6 +908,7 @@ export function setupBatchQueueCallbacks() {
                 spanParentAsLink: meta.spanParentAsLink,
                 batchId,
                 batchIndex: itemIndex,
+                runFriendlyId,
                 realtimeStreamsVersion: meta.realtimeStreamsVersion,
                 planType: meta.planType,
                 triggerSource: meta.parentRunId ? "sdk" : (meta.triggerSource ?? "api"),
@@ -929,6 +945,10 @@ export function setupBatchQueueCallbacks() {
                   parentRunId: meta.parentRunId,
                   resumeParentOnCompletion: meta.resumeParentOnCompletion,
                   batch: { id: batchId, index: itemIndex },
+                  runFriendlyId: mintAnchoredRunFriendlyId(
+                    friendlyId,
+                    (item.options as { region?: string } | undefined)?.region
+                  ),
                   options: item.options as Record<string, unknown>,
                   traceContext: meta.traceContext as Record<string, unknown> | undefined,
                   spanParentAsLink: meta.spanParentAsLink,
@@ -1009,6 +1029,10 @@ export function setupBatchQueueCallbacks() {
                 parentRunId: meta.parentRunId,
                 resumeParentOnCompletion: meta.resumeParentOnCompletion,
                 batch: { id: batchId, index: itemIndex },
+                runFriendlyId: mintAnchoredRunFriendlyId(
+                  friendlyId,
+                  (item.options as { region?: string } | undefined)?.region
+                ),
                 options: item.options as Record<string, unknown>,
                 traceContext: meta.traceContext as Record<string, unknown> | undefined,
                 spanParentAsLink: meta.spanParentAsLink,

apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts

@@ -0,0 +1,31 @@
+import {
+  BatchId,
+  classifyKind,
+  generateRunOpsId,
+  parseRunId,
+  REGION_CODES,
+} from "@trigger.dev/core/v3/isomorphic";
+import { describe, expect, it } from "vitest";
+import { mintAnchoredRunFriendlyId } from "./mintAnchoredRunFriendlyId.server";
+
+describe("mintAnchoredRunFriendlyId", () => {
+  it("a run-ops (NEW) batch anchor yields a run-ops (NEW) item friendlyId", () => {
+    const batchFriendlyId = BatchId.toFriendlyId(generateRunOpsId());
+    const itemFriendlyId = mintAnchoredRunFriendlyId(batchFriendlyId);
+    expect(classifyKind(itemFriendlyId)).toBe("runOpsId");
+  });
+
+  it("a cuid (LEGACY) batch anchor yields a cuid (LEGACY) item friendlyId", () => {
+    const batchFriendlyId = BatchId.generate().friendlyId;
+    const itemFriendlyId = mintAnchoredRunFriendlyId(batchFriendlyId);
+    expect(classifyKind(itemFriendlyId)).toBe("cuid");
+  });
+
+  it("stamps the requested region char into a run-ops id", () => {
+    const batchFriendlyId = BatchId.toFriendlyId(generateRunOpsId());
+    const itemFriendlyId = mintAnchoredRunFriendlyId(batchFriendlyId, "us-east-1");
+    const parsed = parseRunId(itemFriendlyId);
+    expect(parsed.format).toBe("b32hex");
+    expect(parsed.format === "b32hex" && parsed.region).toBe(REGION_CODES["us-east-1"]);
+  });
+});

apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.ts

@@ -0,0 +1,15 @@
+import { generateRunOpsId, RunId, type ResidencyKind } from "@trigger.dev/core/v3/isomorphic";
+import { resolveInheritedMintKind } from "./resolveInheritedMintKind.server";
+
+// Shared id-generation branch for every run-mint path: "runOpsId" -> NEW store, "cuid" -> LEGACY.
+export function mintFriendlyIdForKind(mintKind: ResidencyKind, region?: string): string {
+  return mintKind === "runOpsId"
+    ? RunId.toFriendlyId(generateRunOpsId(region))
+    : RunId.generate().friendlyId;
+}
+
+// Anchor a batch item's mint on the BATCH's friendlyId (id-shape, zero I/O), never the per-org
+// flag, so the item and its BatchTaskRun stay co-resident across a mid-batch flag flip.
+export function mintAnchoredRunFriendlyId(batchFriendlyId: string, region?: string): string {
+  return mintFriendlyIdForKind(resolveInheritedMintKind(batchFriendlyId), region);
+}

apps/webapp/app/v3/services/batchTriggerV3.server.ts

@@ -12,7 +12,6 @@ import {
   Prisma,
 } from "@trigger.dev/database";
 import type { RunStore } from "@internal/run-store";
-import { generateRunOpsId, RunId } from "@trigger.dev/core/v3/isomorphic";
 import { z } from "zod";
 import type { PrismaClientOrTransaction } from "~/db.server";
 import { prisma } from "~/db.server";
@@ -26,6 +25,7 @@ import { getEntitlement } from "~/services/platform.v3.server";
 import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
 import { resolveRunIdMintKind, type RunIdMintKind } from "~/v3/engineVersion.server";
 import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server";
+import { mintFriendlyIdForKind } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
 import { mintBatchFriendlyId } from "~/v3/runOpsMigration/mintBatchFriendlyId.server";
 import { batchTriggerWorker } from "../batchTriggerWorker.server";
 import { legacyRunEngineWorker } from "../legacyRunEngineWorker.server";
@@ -361,9 +361,7 @@ export class BatchTriggerV3Service extends BaseService {
           orgFeatureFlags: environment.organization.featureFlags,
         });
 
-    return mintKind === "runOpsId"
-      ? RunId.toFriendlyId(generateRunOpsId(region))
-      : RunId.generate().friendlyId;
+    return mintFriendlyIdForKind(mintKind, region);
   }
 
   async #prepareRunData(