From abde82c033b12b0c1193817f7151750501c85db3 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 16:41:39 +0100 Subject: [PATCH 01/16] feat(core): export isValidShardChar for shard-descriptor validation Co-Authored-By: Claude Opus 4.8 --- .../core/src/v3/isomorphic/friendlyId.test.ts | 16 ++++++++++++++++ packages/core/src/v3/isomorphic/friendlyId.ts | 5 +++++ 2 files changed, 21 insertions(+) diff --git a/packages/core/src/v3/isomorphic/friendlyId.test.ts b/packages/core/src/v3/isomorphic/friendlyId.test.ts index 2e3ba4d83a5..f788b5940da 100644 --- a/packages/core/src/v3/isomorphic/friendlyId.test.ts +++ b/packages/core/src/v3/isomorphic/friendlyId.test.ts @@ -15,6 +15,7 @@ import { base32hexEncode, generateRunOpsId, generateRunOpsIdV2, + isValidShardChar, parseRunId, parseRunOpsIdBody, parseRunOpsIdV2Body, @@ -410,3 +411,18 @@ describe("parseRunId — v2 arm", () => { expect(parseRunId(`waitpoint_${generateRunOpsIdV2("a")}`).format).toBe("legacy"); }); }); + +describe("isValidShardChar", () => { + it("accepts a single [a-z0-9] char", () => { + expect(isValidShardChar("a")).toBe(true); + expect(isValidShardChar("0")).toBe(true); + expect(isValidShardChar("w")).toBe(true); + }); + it("rejects multi-char, empty, uppercase, and punctuation", () => { + expect(isValidShardChar("")).toBe(false); + expect(isValidShardChar("ab")).toBe(false); + expect(isValidShardChar("A")).toBe(false); + expect(isValidShardChar("-")).toBe(false); + expect(isValidShardChar("legacy")).toBe(false); + }); +}); diff --git a/packages/core/src/v3/isomorphic/friendlyId.ts b/packages/core/src/v3/isomorphic/friendlyId.ts index c468de65319..958d189d598 100644 --- a/packages/core/src/v3/isomorphic/friendlyId.ts +++ b/packages/core/src/v3/isomorphic/friendlyId.ts @@ -40,6 +40,11 @@ export const DEFAULT_REGION_CHAR = "0"; const REGION_CHAR_PATTERN = /^[a-z0-9]$/; // Same slot, same range: the gen-2 shard key is a region char's positional twin. const SHARD_CHAR_PATTERN = REGION_CHAR_PATTERN; +/** True iff `value` is a single valid gen-2 shard char. The descriptor validator and + * `resolveShard` share this so a configured key and a decoded key cannot drift. */ +export function isValidShardChar(value: string): boolean { + return SHARD_CHAR_PATTERN.test(value); +} /** One lowercase [a-z0-9] char per supported region, at RUN_OPS_ID_REGION_INDEX. */ export const REGION_CODES: Readonly> = { "us-east-1": "e", From f9fb0c01950165466ddbf2992dc129e9d523da76 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 16:59:03 +0100 Subject: [PATCH 02/16] feat(run-store): add UnknownShardKey and RoutingRunStore.fromShards with an injected shard resolver Co-Authored-By: Claude Opus 4.8 --- .../src/runOpsStore.fromShards.test.ts | 65 ++++++++++++++++ .../run-store/src/runOpsStore.ts | 74 +++++++++++++++++-- 2 files changed, 131 insertions(+), 8 deletions(-) create mode 100644 internal-packages/run-store/src/runOpsStore.fromShards.test.ts diff --git a/internal-packages/run-store/src/runOpsStore.fromShards.test.ts b/internal-packages/run-store/src/runOpsStore.fromShards.test.ts new file mode 100644 index 00000000000..e9f55332ad7 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.fromShards.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; +import { + generateRunOpsId, + generateRunOpsIdV2, + resolveShard, + type ShardKey, +} from "@trigger.dev/core/v3/isomorphic"; +import { RoutingRunStore, UnknownShardKey } from "./runOpsStore.js"; +import type { ReadClient, RunStore } from "./types.js"; + +// Pure routing unit test for the N-way fromShards factory. Each shard is a fake RunStore whose +// findRun records which slot answered, so the assertions are purely about WHICH store the router +// selects. No database. +type FakeStore = RunStore & { slot: ShardKey }; + +function fakeStore(slot: ShardKey): FakeStore { + const store: Partial = { + slot, + primaryReadClient: { __primary: slot } as unknown as ReadClient, + findRun: ((_where: unknown, _argsOrClient?: unknown, _client?: unknown) => + Promise.resolve({ slot } as never)) as FakeStore["findRun"], + }; + return store as FakeStore; +} + +function build(shardKeys: ShardKey[]) { + const shards = new Map(); + shards.set("legacy", fakeStore("legacy")); + shards.set("new", fakeStore("new")); + for (const k of shardKeys) shards.set(k, fakeStore(k)); + return RoutingRunStore.fromShards({ + shards, + probeOrder: ["new", ...shardKeys, "legacy"], + precedence: ["legacy", "new", ...shardKeys], + idlessRouteShard: "new", + idlessWaitpointShard: "legacy", + resolveShardKey: resolveShard, + }); +} + +describe("RoutingRunStore.fromShards", () => { + it("routes a gen-2 id to its own shard, not to new", async () => { + const store = build(["a"]); + const found = await store.findRun({ friendlyId: generateRunOpsIdV2("a") }); + expect(found).toMatchObject({ slot: "a" }); + }); + + it("routes a gen-1 v1 id to new", async () => { + const store = build(["a"]); + const found = await store.findRun({ friendlyId: generateRunOpsId() }); + expect(found).toMatchObject({ slot: "new" }); + }); + + it("routes a cuid id to legacy", async () => { + const store = build(["a"]); + const found = await store.findRun({ friendlyId: "clabc123def456ghi789jkl01" }); + expect(found).toMatchObject({ slot: "legacy" }); + }); + + it("raises UnknownShardKey for an unconfigured shard and does not fall back", () => { + const store = build(["a"]); // "b" is not configured + // The route resolves synchronously, so the throw is synchronous (before the promise is built). + expect(() => store.findRun({ friendlyId: generateRunOpsIdV2("b") })).toThrow(UnknownShardKey); + }); +}); diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index 27bd78866d4..8d1c7d5aba4 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -40,6 +40,33 @@ import { boundedIn } from "@trigger.dev/database"; const NEW_SHARD: ShardKey = "new"; const LEGACY_SHARD: ShardKey = "legacy"; +/** + * Raised when an id resolves to a shard key that is not configured. NEVER falls back to another + * store — a misconfiguration must fail loud, not misroute silently. Alarmed by ops. + */ +export class UnknownShardKey extends Error { + readonly key: string; + readonly configuredKeys: readonly string[]; + constructor(key: string, configuredKeys: readonly string[]) { + super( + `Unknown run-ops shard key ${JSON.stringify(key)}; configured: [${configuredKeys.join(", ")}]` + ); + this.name = "UnknownShardKey"; + this.key = key; + this.configuredKeys = configuredKeys; + } +} + +type ShardTopology = { + shards: ReadonlyMap; + probeOrder: readonly ShardKey[]; + precedence: readonly ShardKey[]; + idlessRouteShard: ShardKey; + idlessWaitpointShard: ShardKey; + resolveShardKey: (id: string) => ShardKey; + classify?: (id: string) => Residency; +}; + /** * Run-ops routing substrate for the TaskRun-core method group. Implements {@link RunStore} over a * map from shard key to store, selecting one by the residency classifier (`ownerEngine`: run-ops @@ -58,18 +85,24 @@ const LEGACY_SHARD: ShardKey = "legacy"; * each other, so swapping them changes behaviour. */ export class RoutingRunStore implements RunStore { - readonly #shards: ReadonlyMap; + // Not readonly: the compat constructor sets gen-1 defaults, and fromShards() overwrites these + // once (before the instance escapes) via #applyShardTopology. + #shards: ReadonlyMap; // Sequential probe for a lookup with no routable id. The first non-null result wins, and the LAST // entry owns the canonical not-found throw. - readonly #probeOrder: readonly ShardKey[]; + #probeOrder: readonly ShardKey[]; // Ascending authority for a merge. The last write wins, so the highest-authority shard wins a // duplicate id. Every merge in this class MUST use this order. - readonly #precedence: readonly ShardKey[]; + #precedence: readonly ShardKey[]; // The two id-less defaults. They differ by role on purpose: a route with no id lands on the // steady-state home, a waitpoint read with no id lands on the legacy store. - readonly #idlessRouteShard: ShardKey; - readonly #idlessWaitpointShard: ShardKey; + #idlessRouteShard: ShardKey; + #idlessWaitpointShard: ShardKey; readonly #classify: (id: string) => Residency; + // The shard that owns an id. Compat: binary over #classify. fromShards: resolveShard, which names + // a gen-2 id's own shard. NEVER throws (resolveShard is total) — an unconfigured key is caught at + // #shardStore, so #routeKeyOrDefault's catch cannot swallow it into a silent legacy read. + #resolveShardKey: (id: string) => ShardKey; // Compat constructor: the two gen-1 stores, keyed by their reserved shard keys. The options type // MUST stay closed — a union arm loosens the excess-property check and retires the @@ -84,6 +117,28 @@ export class RoutingRunStore implements RunStore { this.#idlessRouteShard = NEW_SHARD; this.#idlessWaitpointShard = LEGACY_SHARD; this.#classify = options.classify ?? ownerEngine; + this.#resolveShardKey = (id) => (this.#classify(id) === "NEW" ? NEW_SHARD : LEGACY_SHARD); + } + + // The N-way factory. Builds via the compat constructor (so the closed options type and its + // test-corpus lock are untouched), then installs the shard topology over the gen-1 defaults. + static fromShards(topology: ShardTopology): RoutingRunStore { + const store = new RoutingRunStore({ + new: topology.shards.get(NEW_SHARD)!, + legacy: topology.shards.get(LEGACY_SHARD)!, + classify: topology.classify, + }); + store.#applyShardTopology(topology); + return store; + } + + #applyShardTopology(topology: ShardTopology): void { + this.#shards = topology.shards; + this.#probeOrder = topology.probeOrder; + this.#precedence = topology.precedence; + this.#idlessRouteShard = topology.idlessRouteShard; + this.#idlessWaitpointShard = topology.idlessWaitpointShard; + this.#resolveShardKey = topology.resolveShardKey; } // A routing store spans two databases and has no single primary — routed reads resolve the @@ -108,14 +163,17 @@ export class RoutingRunStore implements RunStore { #shardStore(key: ShardKey): RunStore { const store = this.#shards.get(key); if (store === undefined) { - throw new Error(`RoutingRunStore: no store is configured for shard key "${key}"`); + // The ONLY place an unconfigured key fails. Keep it here, never in #resolveShardKey: + // #routeKeyOrDefault catches resolver throws and downgrades to legacy, which would turn a + // misconfiguration into a silent legacy read. This throw is outside that catch. + throw new UnknownShardKey(key, [...this.#shards.keys()]); } return store; } - // The shard that owns an existing id. Throws only when an injected classifier throws. + // The shard that owns an existing id. Delegates to the installed resolver (see #resolveShardKey). #shardKeyOf(id: string): ShardKey { - return this.#classify(id) === "NEW" ? NEW_SHARD : LEGACY_SHARD; + return this.#resolveShardKey(id); } // An unclassifiable id is treated as LEGACY (probe the control-plane DB rather than drop a From 36a2bf907f306bfefe012851fc7f5416d9289bd2 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:04:18 +0100 Subject: [PATCH 03/16] feat(webapp): add the RUN_OPS_SHARDS zod descriptor validated at boot Includes the cross-field boot refinement requiring RUN_OPS_DATABASE_URL when the shard list is non-empty, since gen-1 v1 ids resolve to the new store permanently. Co-Authored-By: Claude Opus 4.8 --- apps/webapp/app/env.server.ts | 11 ++ apps/webapp/app/v3/runOpsShards.server.ts | 124 ++++++++++++++++++++++ apps/webapp/test/runOpsShards.test.ts | 71 +++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 apps/webapp/app/v3/runOpsShards.server.ts create mode 100644 apps/webapp/test/runOpsShards.test.ts diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index c9179306124..dcbb751ec15 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { MachinePresetName } from "@trigger.dev/core/v3"; import { BoolEnv } from "./utils/boolEnv"; import { isValidDatabaseUrl } from "./utils/db"; +import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server"; import { isValidRegex } from "./utils/regex"; import { isValidDuration } from "./services/realtime/duration.server"; @@ -310,6 +311,8 @@ const EnvironmentSchema = z RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER: z.string().default("0"), RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER: z.string().default("0"), RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER: z.string().default("0"), + // Gen-2 shard descriptors as a JSON array. Unset/"" -> [] (today). See runOpsShards.server.ts. + RUN_OPS_SHARDS: z.string().optional().transform(parseRunOpsShards), // Control-plane cache relax knobs. Unset -> defaults (DEFAULT_CP_CACHE_TTL_MS / _MAX_ENTRIES). CONTROL_PLANE_CACHE_TTL_MS: z.coerce.number().int().optional(), CONTROL_PLANE_CACHE_MAX_ENTRIES: z.coerce.number().int().optional(), @@ -2467,6 +2470,14 @@ const EnvironmentSchema = z }); } } + if (!validateShardListAgainstNewUrl(env.RUN_OPS_SHARDS, env.RUN_OPS_DATABASE_URL)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["RUN_OPS_SHARDS"], + message: + "RUN_OPS_SHARDS is non-empty but RUN_OPS_DATABASE_URL is unset; a shard requires the gen-1 new store", + }); + } }); export type Environment = z.infer; diff --git a/apps/webapp/app/v3/runOpsShards.server.ts b/apps/webapp/app/v3/runOpsShards.server.ts new file mode 100644 index 00000000000..23c45efa404 --- /dev/null +++ b/apps/webapp/app/v3/runOpsShards.server.ts @@ -0,0 +1,124 @@ +import { z } from "zod"; +import { isValidShardChar } from "@trigger.dev/core/v3/isomorphic"; +import { isValidDatabaseUrl } from "~/utils/db"; + +const KnobsSchema = z + .object({ + writerPoolTimeout: z.number().int().optional(), + writerConnectionTimeout: z.number().int().optional(), + writerDriverAdapter: z.boolean().optional(), + connectionLimit: z.number().int().optional(), + replicaConnectionLimit: z.number().int().optional(), + replicaPoolTimeout: z.number().int().optional(), + replicaConnectionTimeout: z.number().int().optional(), + replicaDriverAdapter: z.boolean().optional(), + transactionMaxWaitMs: z.number().int().optional(), + transactionStartRetryEnabled: z.boolean().optional(), + transactionStartRetryMaxAttempts: z.number().int().optional(), + transactionStartRetryBackoffMinMs: z.number().int().optional(), + transactionStartRetryBackoffMaxMs: z.number().int().optional(), + transactionStartRetryBudgetPerSec: z.number().int().optional(), + transactionStartRetryBudgetBurst: z.number().int().optional(), + }) + .strict(); +export type RunOpsShardKnobs = z.infer; + +const ReplicationSchema = z.object({ + slotName: z.string().min(1), + publicationName: z.string().min(1), + originGeneration: z.number().int().min(2).max(255), +}); + +const DescriptorSchema = z + .object({ + key: z.string().refine(isValidShardChar, "shard key must be a single [a-z0-9] char"), + region: z.string().min(1), + url: z.string().refine(isValidDatabaseUrl, "url is invalid").optional(), + replicaUrl: z.string().refine(isValidDatabaseUrl, "replicaUrl is invalid").optional(), + directUrl: z.string().refine(isValidDatabaseUrl, "directUrl is invalid").optional(), + replication: ReplicationSchema.optional(), + knobs: KnobsSchema.optional(), + aliasOf: z.literal("new").optional(), + }) + .strict() + .superRefine((d, ctx) => { + const hasUrl = d.url !== undefined; + const hasAlias = d.aliasOf !== undefined; + if (hasUrl === hasAlias) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "exactly one of url or aliasOf is required", + }); + } + if (!hasAlias && d.replication === undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "replication is required unless aliasOf is set", + }); + } + }); + +export type RunOpsShardDescriptor = z.infer; + +// Boot-validated transform, in the style of parseMachinePresetCsv. Undefined and "" both mean the +// off state and resolve to []. The undefined guard is load-bearing: an unguarded JSON.parse would +// kill every single-DB boot, which never sets this variable. +export function parseRunOpsShards( + raw: string | undefined, + ctx: z.RefinementCtx +): RunOpsShardDescriptor[] { + if (raw === undefined || raw.trim() === "") return []; + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "RUN_OPS_SHARDS is not valid JSON" }); + return z.NEVER; + } + + const result = z.array(DescriptorSchema).safeParse(parsed); + if (!result.success) { + for (const issue of result.error.issues) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `RUN_OPS_SHARDS[${issue.path.join(".")}]: ${issue.message}`, + }); + } + return z.NEVER; + } + + const keys = new Set(); + const gens = new Set(); + for (const d of result.data) { + if (keys.has(d.key)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `RUN_OPS_SHARDS: duplicate key ${d.key}`, + }); + return z.NEVER; + } + keys.add(d.key); + if (d.replication) { + if (gens.has(d.replication.originGeneration)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `RUN_OPS_SHARDS: duplicate originGeneration ${d.replication.originGeneration}`, + }); + return z.NEVER; + } + gens.add(d.replication.originGeneration); + } + } + + return result.data; +} + +// A non-empty shard list requires the gen-1 new store, because gen-1 v1 ids resolve to "new" +// forever (append-only). Pure so the boot refinement and its test share one rule. +export function validateShardListAgainstNewUrl( + shards: RunOpsShardDescriptor[], + newUrl: string | undefined +): boolean { + return shards.length === 0 || !!newUrl; +} diff --git a/apps/webapp/test/runOpsShards.test.ts b/apps/webapp/test/runOpsShards.test.ts new file mode 100644 index 00000000000..868ee8fa010 --- /dev/null +++ b/apps/webapp/test/runOpsShards.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; +import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server"; + +function run(raw: string | undefined) { + const schema = z.string().optional().transform(parseRunOpsShards); + return schema.safeParse(raw); +} + +const valid = { + key: "a", + region: "us-east-1", + url: "postgres://h/db", + replication: { slotName: "s", publicationName: "p", originGeneration: 2 }, +}; + +describe("parseRunOpsShards", () => { + it("returns [] for undefined", () => { + const r = run(undefined); + expect(r.success && r.data).toEqual([]); + }); + it("returns [] for an empty array literal", () => { + const r = run("[]"); + expect(r.success && r.data).toEqual([]); + }); + it("parses a valid single descriptor", () => { + const r = run(JSON.stringify([valid])); + expect(r.success).toBe(true); + if (r.success) expect(r.data[0].key).toBe("a"); + }); + it("fails on malformed JSON", () => { + expect(run("{not json").success).toBe(false); + }); + it("fails on a multi-char key", () => { + expect(run(JSON.stringify([{ ...valid, key: "ab" }])).success).toBe(false); + }); + it("fails on duplicate keys", () => { + const b = { ...valid, replication: { slotName: "s2", publicationName: "p2", originGeneration: 3 } }; + expect(run(JSON.stringify([valid, b])).success).toBe(false); + }); + it("fails on duplicate origin generations", () => { + const b = { ...valid, key: "b", url: "postgres://h/b" }; + expect(run(JSON.stringify([valid, b])).success).toBe(false); + }); + it("fails when both url and aliasOf are set", () => { + expect(run(JSON.stringify([{ key: "a", region: "x", url: "postgres://h/db", aliasOf: "new" }])).success).toBe(false); + }); + it("accepts aliasOf without url or replication", () => { + expect(run(JSON.stringify([{ key: "a", region: "x", aliasOf: "new" }])).success).toBe(true); + }); + it("fails on an origin generation below 2 or above 255", () => { + const mk = (g: number) => run(JSON.stringify([{ ...valid, replication: { slotName: "s", publicationName: "p", originGeneration: g } }])); + expect(mk(1).success).toBe(false); + expect(mk(256).success).toBe(false); + }); + it("fails when a non-aliased descriptor omits replication", () => { + expect(run(JSON.stringify([{ key: "a", region: "x", url: "postgres://h/db" }])).success).toBe(false); + }); +}); + +describe("validateShardListAgainstNewUrl", () => { + it("passes when the list is empty and no new url", () => { + expect(validateShardListAgainstNewUrl([], undefined)).toBe(true); + }); + it("passes when the list is non-empty and new url is set", () => { + expect(validateShardListAgainstNewUrl([valid as never], "postgres://h/new")).toBe(true); + }); + it("fails when the list is non-empty and new url is unset", () => { + expect(validateShardListAgainstNewUrl([valid as never], undefined)).toBe(false); + }); +}); From ae745440ed06da506fb71eca942755c6147e15c2 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:06:12 +0100 Subject: [PATCH 04/16] feat(webapp): resolve per-role run-ops pool knobs in one module Co-Authored-By: Claude Opus 4.8 --- apps/webapp/app/v3/runOpsPoolKnobs.server.ts | 91 ++++++++++++++++++++ apps/webapp/test/runOpsPoolKnobs.test.ts | 49 +++++++++++ 2 files changed, 140 insertions(+) create mode 100644 apps/webapp/app/v3/runOpsPoolKnobs.server.ts create mode 100644 apps/webapp/test/runOpsPoolKnobs.test.ts diff --git a/apps/webapp/app/v3/runOpsPoolKnobs.server.ts b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts new file mode 100644 index 00000000000..f5da3b7f2b2 --- /dev/null +++ b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts @@ -0,0 +1,91 @@ +import { env } from "~/env.server"; +import type { RunOpsShardKnobs } from "~/v3/runOpsShards.server"; + +// Pool configuration for one run-ops client, resolved at the app boundary (IoC). Every value is a +// number/boolean/string the generic buildWriterClient/buildReplicaClient consumes directly. Kept +// separate from db.server (which ~156 tests mock wholesale) so a new export breaks no mock. +export type ResolvedPoolKnobs = { + writerPoolTimeout: number; + writerConnectionTimeout: number; + writerDriverAdapter: boolean; + connectionLimit: number; + replicaConnectionLimit: number; + replicaPoolTimeout: number; + replicaConnectionTimeout: number; + replicaDriverAdapter: boolean; + // stdoutLogs and label are role constants, never overridable by a descriptor. The run-ops + // builders had no stdout arms and their own log labels; carrying these keeps the merge inert. + stdoutLogs: boolean; + label: string; +}; + +type Role = "new" | "legacy"; + +// Resolve the pool knobs for a run-ops role, reproducing today's builder expressions exactly. +// descriptorKnobs (gen-2 shards only) override the pool fields; stdoutLogs and label stay fixed. +// Transaction resilience is a SEPARATE mechanism (resolveTransactionResilience) and is not here. +export function resolveRunOpsPoolKnobs( + role: Role, + descriptorKnobs?: RunOpsShardKnobs +): ResolvedPoolKnobs { + const k = descriptorKnobs; + + if (role === "legacy") { + return { + writerPoolTimeout: + k?.writerPoolTimeout ?? + env.RUN_OPS_LEGACY_DATABASE_WRITER_POOL_TIMEOUT ?? + env.DATABASE_POOL_TIMEOUT, + writerConnectionTimeout: + k?.writerConnectionTimeout ?? + env.RUN_OPS_LEGACY_DATABASE_WRITER_CONNECTION_TIMEOUT ?? + env.DATABASE_CONNECTION_TIMEOUT, + writerDriverAdapter: + k?.writerDriverAdapter ?? env.RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER === "1", + connectionLimit: k?.connectionLimit ?? env.DATABASE_CONNECTION_LIMIT, + replicaConnectionLimit: k?.replicaConnectionLimit ?? env.DATABASE_CONNECTION_LIMIT, + replicaPoolTimeout: + k?.replicaPoolTimeout ?? + env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? + env.DATABASE_POOL_TIMEOUT, + replicaConnectionTimeout: + k?.replicaConnectionTimeout ?? + env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? + env.DATABASE_CONNECTION_TIMEOUT, + replicaDriverAdapter: + k?.replicaDriverAdapter ?? env.RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + stdoutLogs: true, + label: "legacy run-ops", + }; + } + + return { + writerPoolTimeout: + k?.writerPoolTimeout ?? + env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? + env.DATABASE_POOL_TIMEOUT, + writerConnectionTimeout: + k?.writerConnectionTimeout ?? + env.RUN_OPS_DATABASE_WRITER_CONNECTION_TIMEOUT ?? + env.DATABASE_CONNECTION_TIMEOUT, + writerDriverAdapter: + k?.writerDriverAdapter ?? env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1", + connectionLimit: k?.connectionLimit ?? env.DATABASE_CONNECTION_LIMIT, + replicaConnectionLimit: + k?.replicaConnectionLimit ?? + env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? + env.DATABASE_CONNECTION_LIMIT, + replicaPoolTimeout: + k?.replicaPoolTimeout ?? + env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? + env.DATABASE_POOL_TIMEOUT, + replicaConnectionTimeout: + k?.replicaConnectionTimeout ?? + env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? + env.DATABASE_CONNECTION_TIMEOUT, + replicaDriverAdapter: + k?.replicaDriverAdapter ?? env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + stdoutLogs: false, + label: "run-ops", + }; +} diff --git a/apps/webapp/test/runOpsPoolKnobs.test.ts b/apps/webapp/test/runOpsPoolKnobs.test.ts new file mode 100644 index 00000000000..a567390ab5d --- /dev/null +++ b/apps/webapp/test/runOpsPoolKnobs.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { resolveRunOpsPoolKnobs } from "~/v3/runOpsPoolKnobs.server"; +import { env } from "~/env.server"; + +describe("resolveRunOpsPoolKnobs", () => { + it("new role: reproduces the run-ops builder expressions and stdoutLogs is false", () => { + const k = resolveRunOpsPoolKnobs("new"); + expect(k.connectionLimit).toBe(env.DATABASE_CONNECTION_LIMIT); + expect(k.replicaConnectionLimit).toBe( + env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT + ); + expect(k.writerPoolTimeout).toBe( + env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT + ); + expect(k.replicaPoolTimeout).toBe( + env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT + ); + expect(k.writerDriverAdapter).toBe(env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1"); + expect(k.replicaDriverAdapter).toBe(env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1"); + expect(k.stdoutLogs).toBe(false); + }); + + it("legacy role: uses RUN_OPS_LEGACY_* timeouts, generic connection limit, and stdoutLogs true", () => { + const k = resolveRunOpsPoolKnobs("legacy"); + expect(k.stdoutLogs).toBe(true); + expect(k.connectionLimit).toBe(env.DATABASE_CONNECTION_LIMIT); + expect(k.replicaConnectionLimit).toBe(env.DATABASE_CONNECTION_LIMIT); + expect(k.writerPoolTimeout).toBe( + env.RUN_OPS_LEGACY_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT + ); + expect(k.replicaPoolTimeout).toBe( + env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT + ); + expect(k.writerDriverAdapter).toBe(env.RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER === "1"); + expect(k.replicaDriverAdapter).toBe(env.RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER === "1"); + }); + + it("a descriptor knob overrides its field", () => { + const k = resolveRunOpsPoolKnobs("new", { connectionLimit: 7, writerDriverAdapter: true }); + expect(k.connectionLimit).toBe(7); + expect(k.writerDriverAdapter).toBe(true); + }); + + it("descriptor knobs never override the role's stdoutLogs or label", () => { + const k = resolveRunOpsPoolKnobs("new", { connectionLimit: 7 }); + expect(k.stdoutLogs).toBe(false); + expect(k.label).toContain("run-ops"); + }); +}); From 42d10b0eb971240ec24353eaf8c4559092045619 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:18:38 +0100 Subject: [PATCH 05/16] refactor(webapp): collapse the two run-ops client builders into one factory Dedupes buildRunOpsWriterClient/buildRunOpsReplicaClient into a single buildRunOpsClient parameterized by role and the resolved pool knobs. The control-plane builders (buildWriterClient/buildReplicaClient) are a separate path and stay untouched. Every resolved value matches the former builders, so split-on deployments are byte-identical. Co-Authored-By: Claude Opus 4.8 --- apps/webapp/app/db.server.ts | 198 ++++++------------- apps/webapp/app/v3/runOpsPoolKnobs.server.ts | 16 +- apps/webapp/test/runOpsPoolKnobs.test.ts | 12 +- 3 files changed, 62 insertions(+), 164 deletions(-) diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index a69c83cd375..b2831de4c1e 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -31,6 +31,7 @@ import { assertSplitRealtimeInterlock, } from "./v3/runOpsMigration/splitMode.server"; import { computeRunOpsSplitReadEnabled } from "./v3/runOpsMigration/runOpsSplitReadGate"; +import { resolveRunOpsPoolKnobs } from "./v3/runOpsPoolKnobs.server"; import { assertControlPlaneCoresidencyAdvisory } from "./v3/runOpsMigration/controlPlaneCoresidencySentinel.server"; import { DATASOURCE_CONTEXT_KEY, startActiveSpan } from "./v3/tracer.server"; import { @@ -376,6 +377,8 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { ); } + const newPoolKnobs = resolveRunOpsPoolKnobs("new"); + return selectRunOpsTopology( { splitEnabled, @@ -392,10 +395,14 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { captureInfraErrorsRunOps( tagDatasourceRunOps( "run-ops-writer", - buildRunOpsWriterClient({ + buildRunOpsClient({ url, clientType, - useDriverAdapter: env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1", + role: "writer", + connectionLimit: newPoolKnobs.connectionLimit, + poolTimeout: newPoolKnobs.writerPoolTimeout, + connectTimeout: newPoolKnobs.writerConnectionTimeout, + useDriverAdapter: newPoolKnobs.writerDriverAdapter, }) ) ), @@ -409,10 +416,14 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { captureInfraErrorsRunOps( tagDatasourceRunOps( "run-ops-replica", - buildRunOpsReplicaClient({ + buildRunOpsClient({ url, clientType, - useDriverAdapter: env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + role: "replica", + connectionLimit: newPoolKnobs.replicaConnectionLimit, + poolTimeout: newPoolKnobs.replicaPoolTimeout, + connectTimeout: newPoolKnobs.replicaConnectionTimeout, + useDriverAdapter: newPoolKnobs.replicaDriverAdapter, }) ) ) @@ -924,161 +935,62 @@ export function buildReplicaClient({ return replicaClient; } -function buildRunOpsWriterClient({ +// One factory for the run-ops writer and replica clients, backed by the dedicated RunOpsPrismaClient +// (a separately generated Prisma package). Parameterized by role and the resolved pool knobs, so a +// gen-1 new store and every gen-2 shard share this single builder. The control-plane builders +// (buildWriterClient/buildReplicaClient) are a DIFFERENT path and are untouched — this reuses only +// the shared low-level helpers (buildPrismaConnectionUrl, buildDriverAdapterPool). +function buildRunOpsClient({ url, clientType, + role, + connectionLimit, + poolTimeout, + connectTimeout, useDriverAdapter = false, }: { url: string; clientType: string; + role: "writer" | "replica"; + connectionLimit: number; + poolTimeout: number; + connectTimeout: number; useDriverAdapter?: boolean; }): RunOpsPrismaClient { - const databaseUrl = buildPrismaConnectionUrl(url, { - connectionLimit: env.DATABASE_CONNECTION_LIMIT.toString(), - poolTimeout: (env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT).toString(), - connectTimeout: ( - env.RUN_OPS_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT - ).toString(), - applicationName: env.SERVICE_NAME, - }); - - console.log( - `🔌 setting up run-ops prisma client to ${redactUrlSecrets(databaseUrl)}${ - useDriverAdapter ? " (pg driver adapter)" : "" - }` - ); - - const driverPool = useDriverAdapter - ? buildDriverAdapterPool( - url, - clientType, - env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, - env.DATABASE_CONNECTION_LIMIT - ) - : undefined; - - const client = driverPool - ? new RunOpsPrismaClient({ - adapter: driverPool.adapter, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }) - : new RunOpsPrismaClient({ - datasources: { db: { url: databaseUrl.href } }, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }); - - registerDatabaseMetricsSource( - driverPool - ? { - clientType, - usesDriverAdapter: true, - client, - pool: driverPool.pool, - poolCounters: driverPool.poolCounters, - } - : { clientType, usesDriverAdapter: false, client } - ); - - if (process.env.PRISMA_LOG_TO_STDOUT !== "1") { - client.$on("info", (log) => logger.info("RunOpsPrismaClient info", { clientType, event: log })); - client.$on("warn", (log) => logger.warn("RunOpsPrismaClient warn", { clientType, event: log })); - client.$on("error", (log) => - logger.error("RunOpsPrismaClient error", { clientType, event: log, ignoreError: true }) - ); - } - - client.$on("query", (log) => queryPerformanceMonitor.onQuery("writer", log)); + const isWriter = role === "writer"; + const setupLabel = isWriter ? "run-ops prisma client" : "run-ops read replica connection"; + const connectedLabel = isWriter ? "run-ops prisma client connected" : "run-ops read replica connected"; - const connectPromise = client.$connect(); - if (env.NODE_ENV === "test") { - connectPromise.catch((error) => { - logger.warn("Failed to eagerly connect run-ops prisma client (writer)", { error }); - }); - } - - console.log(`🔌 run-ops prisma client connected`); - - return client; -} - -function buildRunOpsReplicaClient({ - url, - clientType, - useDriverAdapter = false, -}: { - url: string; - clientType: string; - useDriverAdapter?: boolean; -}): RunOpsPrismaClient { - const replicaUrl = buildPrismaConnectionUrl(url, { - connectionLimit: ( - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT - ).toString(), - poolTimeout: ( - env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT - ).toString(), - connectTimeout: ( - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT - ).toString(), + const connectionUrl = buildPrismaConnectionUrl(url, { + connectionLimit: connectionLimit.toString(), + poolTimeout: poolTimeout.toString(), + connectTimeout: connectTimeout.toString(), applicationName: env.SERVICE_NAME, }); console.log( - `🔌 setting up run-ops read replica connection to ${redactUrlSecrets(replicaUrl)}${ + `🔌 setting up ${setupLabel} to ${redactUrlSecrets(connectionUrl)}${ useDriverAdapter ? " (pg driver adapter)" : "" }` ); + const log = [ + { emit: "event", level: "error" }, + { emit: "event", level: "info" }, + { emit: "event", level: "warn" }, + ...((process.env.VERBOSE_PRISMA_LOGS === "1" || + process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined + ? [{ emit: "event", level: "query" }] + : []) as { emit: "event"; level: "query" }[]), + ] as const; + const driverPool = useDriverAdapter - ? buildDriverAdapterPool( - url, - clientType, - env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT - ) + ? buildDriverAdapterPool(url, clientType, poolTimeout, connectionLimit) : undefined; const client = driverPool - ? new RunOpsPrismaClient({ - adapter: driverPool.adapter, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }) - : new RunOpsPrismaClient({ - datasources: { db: { url: replicaUrl.href } }, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }); + ? new RunOpsPrismaClient({ adapter: driverPool.adapter, log: [...log] }) + : new RunOpsPrismaClient({ datasources: { db: { url: connectionUrl.href } }, log: [...log] }); registerDatabaseMetricsSource( driverPool @@ -1096,20 +1008,22 @@ function buildRunOpsReplicaClient({ client.$on("info", (log) => logger.info("RunOpsPrismaClient info", { clientType, event: log })); client.$on("warn", (log) => logger.warn("RunOpsPrismaClient warn", { clientType, event: log })); client.$on("error", (log) => - logger.error("RunOpsPrismaClient error", { clientType, event: log }) + // The writer bridges P2002 -> 422 at the store boundary, so its infra errors are logged once + // there (ignoreError). Replica errors are not on that write path, so they log normally. + logger.error("RunOpsPrismaClient error", { clientType, event: log, ...(isWriter ? { ignoreError: true } : {}) }) ); } - client.$on("query", (log) => queryPerformanceMonitor.onQuery("replica", log)); + client.$on("query", (log) => queryPerformanceMonitor.onQuery(role, log)); const connectPromise = client.$connect(); if (env.NODE_ENV === "test") { connectPromise.catch((error) => { - logger.warn("Failed to eagerly connect run-ops prisma client (replica)", { error }); + logger.warn(`Failed to eagerly connect run-ops prisma client (${role})`, { error }); }); } - console.log(`🔌 run-ops read replica connected`); + console.log(`🔌 ${connectedLabel}`); return client; } diff --git a/apps/webapp/app/v3/runOpsPoolKnobs.server.ts b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts index f5da3b7f2b2..f7bb7b72cf2 100644 --- a/apps/webapp/app/v3/runOpsPoolKnobs.server.ts +++ b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts @@ -1,9 +1,9 @@ import { env } from "~/env.server"; import type { RunOpsShardKnobs } from "~/v3/runOpsShards.server"; -// Pool configuration for one run-ops client, resolved at the app boundary (IoC). Every value is a -// number/boolean/string the generic buildWriterClient/buildReplicaClient consumes directly. Kept -// separate from db.server (which ~156 tests mock wholesale) so a new export breaks no mock. +// Pool configuration for one run-ops store (writer + replica), resolved at the app boundary (IoC). +// Every value reproduces today's run-ops builder expressions. Kept separate from db.server (which +// ~156 tests mock wholesale) so a new export breaks no mock. export type ResolvedPoolKnobs = { writerPoolTimeout: number; writerConnectionTimeout: number; @@ -13,16 +13,12 @@ export type ResolvedPoolKnobs = { replicaPoolTimeout: number; replicaConnectionTimeout: number; replicaDriverAdapter: boolean; - // stdoutLogs and label are role constants, never overridable by a descriptor. The run-ops - // builders had no stdout arms and their own log labels; carrying these keeps the merge inert. - stdoutLogs: boolean; - label: string; }; type Role = "new" | "legacy"; // Resolve the pool knobs for a run-ops role, reproducing today's builder expressions exactly. -// descriptorKnobs (gen-2 shards only) override the pool fields; stdoutLogs and label stay fixed. +// descriptorKnobs (gen-2 shards only) override the pool fields. // Transaction resilience is a SEPARATE mechanism (resolveTransactionResilience) and is not here. export function resolveRunOpsPoolKnobs( role: Role, @@ -54,8 +50,6 @@ export function resolveRunOpsPoolKnobs( env.DATABASE_CONNECTION_TIMEOUT, replicaDriverAdapter: k?.replicaDriverAdapter ?? env.RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER === "1", - stdoutLogs: true, - label: "legacy run-ops", }; } @@ -85,7 +79,5 @@ export function resolveRunOpsPoolKnobs( env.DATABASE_CONNECTION_TIMEOUT, replicaDriverAdapter: k?.replicaDriverAdapter ?? env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1", - stdoutLogs: false, - label: "run-ops", }; } diff --git a/apps/webapp/test/runOpsPoolKnobs.test.ts b/apps/webapp/test/runOpsPoolKnobs.test.ts index a567390ab5d..e281b8ce19e 100644 --- a/apps/webapp/test/runOpsPoolKnobs.test.ts +++ b/apps/webapp/test/runOpsPoolKnobs.test.ts @@ -3,7 +3,7 @@ import { resolveRunOpsPoolKnobs } from "~/v3/runOpsPoolKnobs.server"; import { env } from "~/env.server"; describe("resolveRunOpsPoolKnobs", () => { - it("new role: reproduces the run-ops builder expressions and stdoutLogs is false", () => { + it("new role: reproduces the run-ops builder expressions", () => { const k = resolveRunOpsPoolKnobs("new"); expect(k.connectionLimit).toBe(env.DATABASE_CONNECTION_LIMIT); expect(k.replicaConnectionLimit).toBe( @@ -17,12 +17,10 @@ describe("resolveRunOpsPoolKnobs", () => { ); expect(k.writerDriverAdapter).toBe(env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1"); expect(k.replicaDriverAdapter).toBe(env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1"); - expect(k.stdoutLogs).toBe(false); }); - it("legacy role: uses RUN_OPS_LEGACY_* timeouts, generic connection limit, and stdoutLogs true", () => { + it("legacy role: uses RUN_OPS_LEGACY_* timeouts and the generic connection limit", () => { const k = resolveRunOpsPoolKnobs("legacy"); - expect(k.stdoutLogs).toBe(true); expect(k.connectionLimit).toBe(env.DATABASE_CONNECTION_LIMIT); expect(k.replicaConnectionLimit).toBe(env.DATABASE_CONNECTION_LIMIT); expect(k.writerPoolTimeout).toBe( @@ -40,10 +38,4 @@ describe("resolveRunOpsPoolKnobs", () => { expect(k.connectionLimit).toBe(7); expect(k.writerDriverAdapter).toBe(true); }); - - it("descriptor knobs never override the role's stdoutLogs or label", () => { - const k = resolveRunOpsPoolKnobs("new", { connectionLimit: 7 }); - expect(k.stdoutLogs).toBe(false); - expect(k.label).toContain("run-ops"); - }); }); From 2222f4c3dde0079d90c02124cab78ae5293431c9 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:19:27 +0100 Subject: [PATCH 06/16] feat(webapp): export per-shard transaction resilience with an own budget per pool Co-Authored-By: Claude Opus 4.8 --- .../webapp/app/v3/transactionResilience.server.ts | 7 +++++-- apps/webapp/test/transactionResilience.test.ts | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 apps/webapp/test/transactionResilience.test.ts diff --git a/apps/webapp/app/v3/transactionResilience.server.ts b/apps/webapp/app/v3/transactionResilience.server.ts index ae5678c987c..c3c7eb7d2f3 100644 --- a/apps/webapp/app/v3/transactionResilience.server.ts +++ b/apps/webapp/app/v3/transactionResilience.server.ts @@ -17,8 +17,11 @@ export type TransactionResilienceConfig = { startRetry: TransactionStartRetryConfig; }; -function resolveTransactionResilience( - pool: "control-plane" | "run-ops" | "run-ops-legacy", +// Exported so the topology singleton can build a per-shard config (each call creates its OWN +// TokenBucketRetryBudget, so one shard's retry storm cannot drain another's). `pool` is a free +// string — it only labels a log line, never keys any behaviour. +export function resolveTransactionResilience( + pool: string, overrides: { maxWaitMs?: number; enabled?: boolean; diff --git a/apps/webapp/test/transactionResilience.test.ts b/apps/webapp/test/transactionResilience.test.ts new file mode 100644 index 00000000000..575d059b11c --- /dev/null +++ b/apps/webapp/test/transactionResilience.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vitest"; +import { resolveTransactionResilience } from "~/v3/transactionResilience.server"; + +describe("resolveTransactionResilience per-shard", () => { + it("builds a distinct budget per call, so one shard's storm cannot drain another's", () => { + const a = resolveTransactionResilience("run-ops-shard-a", {}); + const b = resolveTransactionResilience("run-ops-shard-b", {}); + expect(a.startRetry.budget).not.toBe(b.startRetry.budget); + }); + + it("accepts an arbitrary pool label and honours a maxWait override", () => { + expect(() => resolveTransactionResilience("run-ops-shard-z", { maxWaitMs: 1234 })).not.toThrow(); + expect(resolveTransactionResilience("run-ops-shard-z", { maxWaitMs: 1234 }).maxWait).toBe(1234); + }); +}); From 432d7f5ff1e6ff12c6e039700e051acd82e3f980 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:24:12 +0100 Subject: [PATCH 07/16] feat(webapp): build one run-ops client pair per shard descriptor selectRunOpsTopology gains a shard loop and returns a keyed shard map. An aliasOf:"new" descriptor reuses the new store's clients by reference and opens no pool. Each real shard gets its own resilience budget and the new-role pool knobs merged with its per-shard overrides. Co-Authored-By: Claude Opus 4.8 --- apps/webapp/app/db.server.ts | 96 +++++++++++++++++-- .../app/v3/transactionResilience.server.ts | 38 ++++++++ apps/webapp/test/runOpsDbTopology.test.ts | 51 ++++++++++ 3 files changed, 178 insertions(+), 7 deletions(-) diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index b2831de4c1e..52d55f2773a 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -32,6 +32,7 @@ import { } from "./v3/runOpsMigration/splitMode.server"; import { computeRunOpsSplitReadEnabled } from "./v3/runOpsMigration/runOpsSplitReadGate"; import { resolveRunOpsPoolKnobs } from "./v3/runOpsPoolKnobs.server"; +import { resolveShardResilience } from "./v3/transactionResilience.server"; import { assertControlPlaneCoresidencyAdvisory } from "./v3/runOpsMigration/controlPlaneCoresidencySentinel.server"; import { DATASOURCE_CONTEXT_KEY, startActiveSpan } from "./v3/tracer.server"; import { @@ -276,10 +277,19 @@ export const webhookReplica: WebhookReplicaDatabase = singleton("webhookReplica" type RunOpsClients = { writer: PrismaClient; replica: PrismaReplicaClient }; type NewRunOpsClients = { writer: RunOpsPrismaClient; replica: RunOpsPrismaClient }; +export type ShardTopologyDescriptor = { + key: string; + url?: string; + replicaUrl?: string; + aliasOf?: "new"; +}; export type RunOpsTopology = { newRunOps: NewRunOpsClients; legacyRunOps: RunOpsClients; controlPlane: RunOpsClients; + // One client pair per gen-2 shard descriptor. Empty unless RUN_OPS_SHARDS is configured. An + // aliasOf:"new" descriptor maps to the newRunOps pair BY REFERENCE (no new pool). + shards: Map; }; export type SelectRunOpsTopologyConfig = { splitEnabled: boolean; @@ -289,6 +299,7 @@ export type SelectRunOpsTopologyConfig = { newReplicaUrl?: string; // When true, legacy reuses the control-plane client instead of opening its own pool. Defaults to false. legacySharesControlPlane?: boolean; + shards?: ShardTopologyDescriptor[]; }; export type RunOpsClientBuilders = { controlPlane: RunOpsClients; @@ -298,6 +309,10 @@ export type RunOpsClientBuilders = { // RunOpsPrismaClient double-cast needed): the legacy DB carries the full control-plane schema. buildLegacyWriter: (url: string, clientType: string) => PrismaClient; buildLegacyReplica: (url: string, clientType: string) => PrismaReplicaClient; + // Receive the whole descriptor so the singleton can resolve per-shard knobs and resilience by key. + // Optional so the existing test literals (which build no shards) need no change. + buildShardWriter?: (shard: ShardTopologyDescriptor) => RunOpsPrismaClient; + buildShardReplica?: (shard: ShardTopologyDescriptor) => RunOpsPrismaClient; }; // Pure run-ops client selector. No env, no isSplitEnabled() — those @@ -316,11 +331,11 @@ export function selectRunOpsTopology( }; if (!config.splitEnabled) { - return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane }; + return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane, shards: new Map() }; } if (!config.legacyUrl || !config.newUrl) { - return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane }; + return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane, shards: new Map() }; } // Same-DB legacy reuses the control-plane pool; only build a separate pool once the DSNs diverge. @@ -339,12 +354,28 @@ export function selectRunOpsTopology( const newReplica: RunOpsPrismaClient = config.newReplicaUrl ? builders.buildNewReplica(config.newReplicaUrl, "run-ops-replica") : newWriter; + const newRunOps: NewRunOpsClients = { writer: newWriter, replica: newReplica }; + + const shards = new Map(); + for (const shard of config.shards ?? []) { + if (shard.aliasOf === "new") { + // Aliased: share the new store's clients by reference. No builder, no new pool — the soak path. + shards.set(shard.key, newRunOps); + continue; + } + if (!shard.url || !builders.buildShardWriter || !builders.buildShardReplica) { + throw new Error( + `selectRunOpsTopology: shard "${shard.key}" needs a url and shard builders when not aliased` + ); + } + const shardWriter = builders.buildShardWriter(shard); + const shardReplica: RunOpsPrismaClient = shard.replicaUrl + ? builders.buildShardReplica(shard) + : shardWriter; + shards.set(shard.key, { writer: shardWriter, replica: shardReplica }); + } - return { - newRunOps: { writer: newWriter, replica: newReplica }, - legacyRunOps, - controlPlane, - }; + return { newRunOps, legacyRunOps, controlPlane, shards }; } // The env-bound run-ops topology singleton. The split decision uses @@ -378,6 +409,7 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { } const newPoolKnobs = resolveRunOpsPoolKnobs("new"); + const shardDescriptorsByKey = new Map(env.RUN_OPS_SHARDS.map((d) => [d.key, d])); return selectRunOpsTopology( { @@ -387,6 +419,12 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { newUrl, newReplicaUrl: env.RUN_OPS_DATABASE_READ_REPLICA_URL, legacySharesControlPlane, + shards: env.RUN_OPS_SHARDS.map((d) => ({ + key: d.key, + url: d.url, + replicaUrl: d.replicaUrl, + aliasOf: d.aliasOf, + })), }, { controlPlane: { writer: prisma, replica: $replica }, @@ -461,6 +499,50 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { ) ) ), + // A gen-2 shard is a dedicated run-ops DB, so it mirrors buildNewWriter/buildNewReplica: same + // client class, same wrapper stack, its OWN resilience budget, and the "new"-role pool knobs + // merged with the descriptor's per-shard overrides. Shards share the run-ops datasource tag. + buildShardWriter: (shard) => { + const descriptor = shardDescriptorsByKey.get(shard.key); + const knobs = resolveRunOpsPoolKnobs("new", descriptor?.knobs); + return registerTransactionResilience( + captureInfraErrorsRunOps( + tagDatasourceRunOps( + "run-ops-writer", + buildRunOpsClient({ + url: shard.url!, + clientType: `run-ops-shard-${shard.key}-writer`, + role: "writer", + connectionLimit: knobs.connectionLimit, + poolTimeout: knobs.writerPoolTimeout, + connectTimeout: knobs.writerConnectionTimeout, + useDriverAdapter: knobs.writerDriverAdapter, + }) + ) + ), + resolveShardResilience(shard.key, descriptor?.knobs) + ); + }, + buildShardReplica: (shard) => { + const descriptor = shardDescriptorsByKey.get(shard.key); + const knobs = resolveRunOpsPoolKnobs("new", descriptor?.knobs); + return markReadReplicaClient( + captureInfraErrorsRunOps( + tagDatasourceRunOps( + "run-ops-replica", + buildRunOpsClient({ + url: shard.replicaUrl!, + clientType: `run-ops-shard-${shard.key}-replica`, + role: "replica", + connectionLimit: knobs.replicaConnectionLimit, + poolTimeout: knobs.replicaPoolTimeout, + connectTimeout: knobs.replicaConnectionTimeout, + useDriverAdapter: knobs.replicaDriverAdapter, + }) + ) + ) + ); + }, } ); }); diff --git a/apps/webapp/app/v3/transactionResilience.server.ts b/apps/webapp/app/v3/transactionResilience.server.ts index c3c7eb7d2f3..eabde6691d8 100644 --- a/apps/webapp/app/v3/transactionResilience.server.ts +++ b/apps/webapp/app/v3/transactionResilience.server.ts @@ -67,6 +67,44 @@ export const runOpsTransactionResilience = resolveTransactionResilience("run-ops budgetBurst: env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BUDGET_BURST, }); +// A gen-2 shard's resilience. Defaults to the RUN_OPS_DATABASE_TRANSACTION_* values (so a shard with +// no overrides matches the gen-1 new store), then applies the descriptor's per-shard overrides. Each +// call builds its OWN budget, so a storm on one shard cannot drain another's. +export function resolveShardResilience( + key: string, + overrides?: { + transactionMaxWaitMs?: number; + transactionStartRetryEnabled?: boolean; + transactionStartRetryMaxAttempts?: number; + transactionStartRetryBackoffMinMs?: number; + transactionStartRetryBackoffMaxMs?: number; + transactionStartRetryBudgetPerSec?: number; + transactionStartRetryBudgetBurst?: number; + } +): TransactionResilienceConfig { + return resolveTransactionResilience(`run-ops-shard-${key}`, { + maxWaitMs: overrides?.transactionMaxWaitMs ?? env.RUN_OPS_DATABASE_TRANSACTION_MAX_WAIT_MS, + enabled: + overrides?.transactionStartRetryEnabled ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_ENABLED, + maxAttempts: + overrides?.transactionStartRetryMaxAttempts ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_MAX_ATTEMPTS, + backoffMinMs: + overrides?.transactionStartRetryBackoffMinMs ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BACKOFF_MIN_MS, + backoffMaxMs: + overrides?.transactionStartRetryBackoffMaxMs ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BACKOFF_MAX_MS, + budgetPerSec: + overrides?.transactionStartRetryBudgetPerSec ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BUDGET_PER_SEC, + budgetBurst: + overrides?.transactionStartRetryBudgetBurst ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BUDGET_BURST, + }); +} + export const runOpsLegacyTransactionResilience = resolveTransactionResilience("run-ops-legacy", { maxWaitMs: env.RUN_OPS_LEGACY_DATABASE_TRANSACTION_MAX_WAIT_MS, enabled: env.RUN_OPS_LEGACY_DATABASE_TRANSACTION_START_RETRY_ENABLED, diff --git a/apps/webapp/test/runOpsDbTopology.test.ts b/apps/webapp/test/runOpsDbTopology.test.ts index 8890fdbb662..f6895a03d29 100644 --- a/apps/webapp/test/runOpsDbTopology.test.ts +++ b/apps/webapp/test/runOpsDbTopology.test.ts @@ -142,6 +142,57 @@ describe("selectRunOpsTopology (pure)", () => { expect(topo.legacyRunOps.replica).toBe(legacyWriter); expect(buildLegacyReplica).not.toHaveBeenCalled(); }); + + const baseSplit = { + splitEnabled: true, + legacyUrl: "postgres://legacy", + newUrl: "postgres://new", + }; + const baseBuilders = () => ({ + controlPlane: cp, + buildNewWriter: vi.fn().mockReturnValue({ tag: "nw" } as any), + buildNewReplica: vi.fn().mockReturnValue({ tag: "nr" } as any), + buildLegacyWriter: vi.fn().mockReturnValue({ tag: "lw" } as any), + buildLegacyReplica: vi.fn().mockReturnValue({ tag: "lr" } as any), + }); + + it("no descriptors: the shards map is empty", () => { + const topo = selectRunOpsTopology(baseSplit, baseBuilders()); + expect(topo.shards.size).toBe(0); + }); + + it("two descriptors: two shard client pairs, each built once", () => { + const buildShardWriter = vi.fn((s: any) => ({ tag: `w:${s.key}` }) as any); + const buildShardReplica = vi.fn((s: any) => ({ tag: `r:${s.key}` }) as any); + const topo = selectRunOpsTopology( + { + ...baseSplit, + shards: [ + { key: "a", url: "postgres://a", replicaUrl: "postgres://a-r" }, + { key: "b", url: "postgres://b" }, + ], + }, + { ...baseBuilders(), buildShardWriter, buildShardReplica } + ); + expect(topo.shards.size).toBe(2); + expect(topo.shards.get("a")!.writer).toEqual({ tag: "w:a" }); + // b has no replicaUrl, so its replica falls back to its writer (buildShardReplica not called for b). + expect(topo.shards.get("b")!.replica).toEqual({ tag: "w:b" }); + expect(buildShardWriter).toHaveBeenCalledTimes(2); + expect(buildShardReplica).toHaveBeenCalledTimes(1); + }); + + it("an alias descriptor reuses newRunOps by reference and calls no shard builder", () => { + const buildShardWriter = vi.fn(); + const buildShardReplica = vi.fn(); + const topo = selectRunOpsTopology( + { ...baseSplit, shards: [{ key: "a", aliasOf: "new" }] }, + { ...baseBuilders(), buildShardWriter, buildShardReplica } + ); + expect(topo.shards.get("a")).toBe(topo.newRunOps); + expect(buildShardWriter).not.toHaveBeenCalled(); + expect(buildShardReplica).not.toHaveBeenCalled(); + }); }); describe("sameDatabaseTarget", () => { From f908f1460c2734751cbca717b333a542be7fef1c Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:28:25 +0100 Subject: [PATCH 08/16] feat(webapp): build N dedicated stores and the keyed router, and log the shard table at boot buildRunStore now produces one dedicated store per shard descriptor and the N-way router via RoutingRunStore.fromShards, keeping the two-store compat router when no shards are configured. The topology singleton logs the resolved shard table (key, address fingerprint, role) only when RUN_OPS_SHARDS is non-empty, so the unset case adds no output. The fingerprint is an address, never an identity claim. Co-Authored-By: Claude Opus 4.8 --- apps/webapp/app/db.server.ts | 45 +++++++++++++ apps/webapp/app/v3/runStore.server.ts | 67 +++++++++++++++++-- apps/webapp/test/runOpsShardBootTable.test.ts | 34 ++++++++++ apps/webapp/test/runStoreShardWiring.test.ts | 44 ++++++++++++ 4 files changed, 184 insertions(+), 6 deletions(-) create mode 100644 apps/webapp/test/runOpsShardBootTable.test.ts create mode 100644 apps/webapp/test/runStoreShardWiring.test.ts diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index 52d55f2773a..b67c3a9b4aa 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -411,6 +411,14 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { const newPoolKnobs = resolveRunOpsPoolKnobs("new"); const shardDescriptorsByKey = new Map(env.RUN_OPS_SHARDS.map((d) => [d.key, d])); + // Boot table: emit ONLY when shards are configured, so the inert (RUN_OPS_SHARDS unset) merge adds + // no new log output. The fingerprint is an address, not an identity claim (see runOpsAddressFingerprint). + if (env.RUN_OPS_SHARDS.length > 0) { + logger.info("run-ops shard topology (fingerprint is an address, NOT an identity claim)", { + shards: buildRunOpsShardTable(env.RUN_OPS_SHARDS), + }); + } + return selectRunOpsTopology( { splitEnabled, @@ -568,6 +576,17 @@ export const runOpsLegacyPrismaClient: RunOpsPrismaClient = runOpsTopology.legac export const runOpsLegacyReplicaClient: RunOpsPrismaClient = runOpsTopology.legacyRunOps .replica as unknown as RunOpsPrismaClient; +// Gen-2 shard handles for the run-store boundary. Empty unless RUN_OPS_SHARDS is configured. +export const runOpsShardHandles: Array<{ + key: string; + writer: RunOpsPrismaClient; + replica: RunOpsPrismaClient; +}> = [...runOpsTopology.shards.entries()].map(([key, clients]) => ({ + key, + writer: clients.writer, + replica: clients.replica, +})); + export const runOpsSplitReadEnabled: boolean = computeRunOpsSplitReadEnabled({ newReplica: runOpsNewReplicaClient, controlPlaneWriter: prisma, @@ -1135,6 +1154,32 @@ function redactUrlSecrets(hrefOrUrl: string | URL) { return url.href; } +// A host:port/db address, with NO username and NO query params — never a secret, and deliberately +// NOT an identity claim (two DSNs can share an address yet be different databases; that proof is the +// distinctness sentinel's, not this line's). Same tuple sameDatabaseTarget compares, kept in step. +export function runOpsAddressFingerprint(url: string): string { + try { + const u = new URL(url); + return `${u.hostname}:${u.port || "5432"}${u.pathname}`; + } catch { + return "unparseable"; + } +} + +export type RunOpsShardTableRow = { key: string; fingerprint: string; role: string }; + +// The resolved shard table for the boot log: one row per descriptor. An alias reports its role and +// carries no address (it shares the new store's pool). +export function buildRunOpsShardTable( + descriptors: Array<{ key: string; url?: string; aliasOf?: "new" }> +): RunOpsShardTableRow[] { + return descriptors.map((d) => + d.aliasOf + ? { key: d.key, fingerprint: "alias(new)", role: "alias(new)" } + : { key: d.key, fingerprint: runOpsAddressFingerprint(d.url ?? ""), role: "shard" } + ); +} + export type { PrismaClient } from "@trigger.dev/database"; function getDatabaseSchema() { diff --git a/apps/webapp/app/v3/runStore.server.ts b/apps/webapp/app/v3/runStore.server.ts index 9ccf84b5117..3bc21c14097 100644 --- a/apps/webapp/app/v3/runStore.server.ts +++ b/apps/webapp/app/v3/runStore.server.ts @@ -1,5 +1,5 @@ import { PostgresRunStore, RoutingRunStore, type RunStore } from "@internal/run-store"; -import { ownerEngine, type Residency } from "@trigger.dev/core/v3/isomorphic"; +import { ownerEngine, resolveShard, type Residency, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; import type { PrismaClient, PrismaReplicaClient } from "@trigger.dev/database"; import type { RunOpsPrismaClient } from "@internal/run-ops-database"; import { @@ -9,6 +9,7 @@ import { runOpsLegacyReplica, runOpsNewPrismaClient, runOpsNewReplicaClient, + runOpsShardHandles, } from "~/db.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; @@ -31,6 +32,16 @@ type BuildRunStoreDeps = { singleReplica: PrismaReplicaClient; /** Residency classifier; defaults to ownerEngine inside RoutingRunStore. */ classify?: (id: string) => Residency; + /** Gen-2 shard handles. When non-empty, buildRunStore produces N dedicated stores + the keyed + * router (fromShards). Empty/absent keeps today's two-store compat router. */ + shards?: Array<{ + key: ShardKey; + writer: RunOpsPrismaClient; + replica: RunOpsPrismaClient; + resilience?: TransactionResilienceConfig; + }>; + /** Shard-key resolver for the fromShards path; defaults to resolveShard. */ + resolveShardKey?: (id: string) => ShardKey; /** Per-pool transaction-resilience configs threaded into the store(s) this builds (IoC). */ singleResilience?: TransactionResilienceConfig; newResilience?: TransactionResilienceConfig; @@ -79,10 +90,45 @@ export function buildRunStore(deps: BuildRunStoreDeps): RunStore { transactionStartRetry: deps.legacyResilience?.startRetry, }); - return new RoutingRunStore({ - new: newStore, - legacy: legacyStore, - classify: deps.classify ?? ownerEngine, + // No gen-2 shards: today's two-store compat router, byte-identical. + if (!deps.shards || deps.shards.length === 0) { + return new RoutingRunStore({ + new: newStore, + legacy: legacyStore, + classify: deps.classify ?? ownerEngine, + }); + } + + // Gen-2 shards: one dedicated store per descriptor, then the keyed N-way router. Every shard is a + // schemaVariant:"dedicated" instance, exactly like the gen-1 new store. + const shardStores = deps.shards.map((shard) => ({ + key: shard.key, + store: new PostgresRunStore({ + prisma: shard.writer, + readOnlyPrisma: shard.replica, + schemaVariant: "dedicated", + maxWait: shard.resilience?.maxWait, + transactionStartRetry: shard.resilience?.startRetry, + }), + })); + + const shardKeys = shardStores.map((s) => s.key); + const shardMap = new Map([ + ["legacy", legacyStore], + ["new", newStore], + ...shardStores.map(({ key, store }) => [key, store] as const), + ]); + + return RoutingRunStore.fromShards({ + shards: shardMap, + // Ascending authority for a merge: legacy -> new -> shards in configured order. + precedence: ["legacy", "new", ...shardKeys], + // Probe order for an id-less lookup: the reverse of precedence. + probeOrder: ["new", ...shardKeys, "legacy"], + idlessRouteShard: "new", + idlessWaitpointShard: "legacy", + resolveShardKey: deps.resolveShardKey ?? resolveShard, + classify: deps.classify, }); } @@ -110,6 +156,8 @@ function tryResolveRunOpsHandles() { newReplica: runOpsNewReplicaClient, legacyWriter: runOpsLegacyPrisma, legacyReplica: runOpsLegacyReplica, + // Absent under a minimal db.server mock; default to no shards so the compat router is built. + shardHandles: runOpsShardHandles ?? [], }; } catch { return null; @@ -127,9 +175,16 @@ export const runStore: RunStore = singleton("RunStore", () => { singleResilience: resilienceForClient(prisma), }); } + const { shardHandles, ...storeHandles } = handles; return buildRunStore({ splitEnabled: true, - ...handles, + ...storeHandles, + shards: shardHandles.map((shard) => ({ + key: shard.key, + writer: shard.writer, + replica: shard.replica, + resilience: resilienceForClient(shard.writer), + })), singleWriter: prisma, singleReplica: $replica, singleResilience: resilienceForClient(prisma), diff --git a/apps/webapp/test/runOpsShardBootTable.test.ts b/apps/webapp/test/runOpsShardBootTable.test.ts new file mode 100644 index 00000000000..9c7706e3cbd --- /dev/null +++ b/apps/webapp/test/runOpsShardBootTable.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { runOpsAddressFingerprint, buildRunOpsShardTable } from "~/db.server"; + +describe("runOpsAddressFingerprint", () => { + it("returns host:port/db with no username or query params", () => { + const fp = runOpsAddressFingerprint( + "postgres://user:pw@host.example:5433/mydb?schema=public&pool_timeout=20" + ); + expect(fp).toBe("host.example:5433/mydb"); + expect(fp).not.toContain("user"); + expect(fp).not.toContain("pool_timeout"); + }); + it("defaults the port to 5432", () => { + expect(runOpsAddressFingerprint("postgres://h/db")).toBe("h:5432/db"); + }); + it("returns a marker on unparseable input rather than throwing", () => { + expect(runOpsAddressFingerprint("not a url")).toBe("unparseable"); + }); +}); + +describe("buildRunOpsShardTable", () => { + it("one row per descriptor, with key, fingerprint, and role", () => { + const rows = buildRunOpsShardTable([ + { key: "a", url: "postgres://user:pw@h/adb?schema=public" }, + { key: "b", aliasOf: "new" }, + ]); + expect(rows).toHaveLength(2); + expect(rows[0]).toEqual({ key: "a", fingerprint: "h:5432/adb", role: "shard" }); + expect(rows[1]).toEqual({ key: "b", fingerprint: "alias(new)", role: "alias(new)" }); + }); + it("is empty for an empty descriptor list", () => { + expect(buildRunOpsShardTable([])).toEqual([]); + }); +}); diff --git a/apps/webapp/test/runStoreShardWiring.test.ts b/apps/webapp/test/runStoreShardWiring.test.ts new file mode 100644 index 00000000000..728f8ee5623 --- /dev/null +++ b/apps/webapp/test/runStoreShardWiring.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { RoutingRunStore } from "@internal/run-store"; +import { buildRunStore } from "~/v3/runStore.server"; + +// Construction-only: buildRunStore wraps clients but never connects, so stub handles suffice. This +// asserts the wiring shape (compat router vs N-way router), not query behaviour. +const stub = () => ({}) as any; + +const baseSplit = { + splitEnabled: true as const, + newWriter: stub(), + newReplica: stub(), + legacyWriter: stub(), + legacyReplica: stub(), + singleWriter: stub(), + singleReplica: stub(), +}; + +describe("buildRunStore shard wiring", () => { + it("split ON with no shards builds the two-store compat router", () => { + const store = buildRunStore(baseSplit); + expect(store).toBeInstanceOf(RoutingRunStore); + }); + + it("split ON with two shard descriptors builds the N-way router", () => { + const store = buildRunStore({ + ...baseSplit, + shards: [ + { key: "a", writer: stub(), replica: stub() }, + { key: "b", writer: stub(), replica: stub() }, + ], + }); + expect(store).toBeInstanceOf(RoutingRunStore); + }); + + it("split OFF builds the single-store passthrough (not a router)", () => { + const store = buildRunStore({ + splitEnabled: false, + singleWriter: stub(), + singleReplica: stub(), + }); + expect(store).not.toBeInstanceOf(RoutingRunStore); + }); +}); From dbc22fd3fd9cb0dccf7298be7ccb366acde4109a Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:31:03 +0100 Subject: [PATCH 09/16] feat(webapp): bound the active mint list against the configured shard descriptor keys computeMintShard now intersects the active shard set with routableKeys (the RUN_OPS_SHARDS descriptor keys), so a stored key with no descriptor is never minted into and falls back to gen-1. The empty-set check runs first, so an unconfigured deployment is unchanged. Inert until the gen-2 write path wires in resolveMintShard. Co-Authored-By: Claude Opus 4.8 --- .../mintShardAssignment.test.ts | 34 +++++++++++++++++++ .../v3/runOpsMigration/mintShardAssignment.ts | 18 +++++++++- .../runOpsMigration/runOpsMintShard.server.ts | 2 ++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts index d88e64e1d75..e73f526e665 100644 --- a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts @@ -473,3 +473,37 @@ describe("computeMintShard — the global override wins the complete cutover", ( ); }); }); + +describe("routableKeys bound (the shard descriptor keys this deployment can route)", () => { + it("drops an active key that is not routable, so the hash never returns it", () => { + // "z" is in the active list but not configured as a descriptor -> only "a" is selectable. + const ids = envIds(200); + for (const id of ids) { + const shard = computeMintShard({ id }, deps({ set: ["a", "z"] }, { routableKeys: ["a"] })); + expect(shard).toBe("a"); + } + }); + + it("returns new when the active list holds only non-routable keys (fail-safe to gen-1)", () => { + expect( + computeMintShard({ id: "env_1" }, deps({ set: ["z"] }, { routableKeys: ["a"] })) + ).toBe("new"); + }); + + it("rejects a per-org pin to a non-routable key and falls through to the hash", () => { + const shard = computeMintShard( + { id: "env_1" }, + deps({ set: ["a", "z"] }, { ...orgFlags({ runOpsMintShard: "z" }), routableKeys: ["a"] }) + ); + expect(shard).toBe("a"); + }); + + it("with no routableKeys given, behaviour is unchanged", () => { + const ids = envIds(200); + for (const id of ids) { + const withBound = computeMintShard({ id }, deps({ set: ["a", "b"] }, { routableKeys: ["a", "b"] })); + const without = computeMintShard({ id }, deps({ set: ["a", "b"] })); + expect(withBound).toBe(without); + } + }); +}); diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts index a49a1a6a60d..2855f936249 100644 --- a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts +++ b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts @@ -19,6 +19,10 @@ export type MintShardDeps = { nowMs: number; graceMs: number; orgFeatureFlags: unknown; + // The shard keys this deployment can actually route (the RUN_OPS_SHARDS descriptor keys). The + // active set is bounded to these, so a stored key with no descriptor is never minted into. + // Undefined means "no bound" (today's behaviour). + routableKeys?: readonly string[]; onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void; onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void; }; @@ -94,7 +98,17 @@ function hrwSelect(environmentId: string, activeSet: string[]): string { // would leak the drain the active list performs, and throwing would fail customer triggers // whenever a pinned shard drains. export function computeMintShard(environment: { id: string }, deps: MintShardDeps): ShardKey { - const activeSet = effectiveMintShardSet(deps.resolution, deps.nowMs, deps.graceMs); + const rawActiveSet = effectiveMintShardSet(deps.resolution, deps.nowMs, deps.graceMs); + // Empty check BEFORE the bound, so an unconfigured deployment returns "new" exactly as today. + if (rawActiveSet.length === 0) { + return "new"; + } + + // Bound the active set to the keys this deployment can route. A stored key with no descriptor is + // dropped, never minted into. If nothing survives, fall back to gen-1 (fail-safe, never a throw). + const activeSet = deps.routableKeys + ? rawActiveSet.filter((key) => deps.routableKeys!.includes(key)) + : rawActiveSet; if (activeSet.length === 0) { return "new"; } @@ -148,6 +162,7 @@ export type ResolveMintShardDeps = { ttlMs: number; graceMs: number; orgFeatureFlags: unknown; + routableKeys?: readonly string[]; onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void; onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void; onReadFailed?: (error: unknown) => void; @@ -200,6 +215,7 @@ export async function resolveMintShardWith( nowMs: deps.nowMs, graceMs: deps.graceMs, orgFeatureFlags: deps.orgFeatureFlags, + routableKeys: deps.routableKeys, onPinRejected: deps.onPinRejected, onOverrideRejected: deps.onOverrideRejected, }); diff --git a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts index 542384e16f8..c1c2b9ddd48 100644 --- a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts @@ -84,6 +84,8 @@ export async function resolveMintShard(environment: { ttlMs: env.RUN_OPS_MINT_FLAG_CACHE_TTL_MS, graceMs: env.RUN_OPS_MINT_FLIP_GRACE_MS, orgFeatureFlags: environment.orgFeatureFlags, + // Bound the active list to the shards this deployment can actually route. + routableKeys: env.RUN_OPS_SHARDS.map((shard) => shard.key), onPinRejected: reportPinRejected, onOverrideRejected: reportOverrideRejected, onReadFailed: (error) => From 46e97cfe6537a68a54fb088225ffbf6efb5d7bf6 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:32:56 +0100 Subject: [PATCH 10/16] chore(webapp): unexport internal shard descriptor type and apply format/lint Co-Authored-By: Claude Opus 4.8 --- apps/webapp/app/db.server.ts | 18 ++++++++++------ .../mintShardAssignment.test.ts | 11 ++++++---- apps/webapp/app/v3/runOpsPoolKnobs.server.ts | 4 +--- apps/webapp/app/v3/runStore.server.ts | 7 ++++++- apps/webapp/test/runOpsShards.test.ts | 21 +++++++++++++++---- .../webapp/test/transactionResilience.test.ts | 4 +++- 6 files changed, 46 insertions(+), 19 deletions(-) diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index b67c3a9b4aa..1658219a0c3 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -32,16 +32,16 @@ import { } from "./v3/runOpsMigration/splitMode.server"; import { computeRunOpsSplitReadEnabled } from "./v3/runOpsMigration/runOpsSplitReadGate"; import { resolveRunOpsPoolKnobs } from "./v3/runOpsPoolKnobs.server"; -import { resolveShardResilience } from "./v3/transactionResilience.server"; -import { assertControlPlaneCoresidencyAdvisory } from "./v3/runOpsMigration/controlPlaneCoresidencySentinel.server"; -import { DATASOURCE_CONTEXT_KEY, startActiveSpan } from "./v3/tracer.server"; import { + resolveShardResilience, controlPlaneTransactionResilience, registerTransactionResilience, resilienceForClient, runOpsLegacyTransactionResilience, runOpsTransactionResilience, } from "./v3/transactionResilience.server"; +import { assertControlPlaneCoresidencyAdvisory } from "./v3/runOpsMigration/controlPlaneCoresidencySentinel.server"; +import { DATASOURCE_CONTEXT_KEY, startActiveSpan } from "./v3/tracer.server"; import type { Span } from "@opentelemetry/api"; import { context, trace } from "@opentelemetry/api"; import { queryPerformanceMonitor } from "./utils/queryPerformanceMonitor.server"; @@ -277,7 +277,7 @@ export const webhookReplica: WebhookReplicaDatabase = singleton("webhookReplica" type RunOpsClients = { writer: PrismaClient; replica: PrismaReplicaClient }; type NewRunOpsClients = { writer: RunOpsPrismaClient; replica: RunOpsPrismaClient }; -export type ShardTopologyDescriptor = { +type ShardTopologyDescriptor = { key: string; url?: string; replicaUrl?: string; @@ -1060,7 +1060,9 @@ function buildRunOpsClient({ }): RunOpsPrismaClient { const isWriter = role === "writer"; const setupLabel = isWriter ? "run-ops prisma client" : "run-ops read replica connection"; - const connectedLabel = isWriter ? "run-ops prisma client connected" : "run-ops read replica connected"; + const connectedLabel = isWriter + ? "run-ops prisma client connected" + : "run-ops read replica connected"; const connectionUrl = buildPrismaConnectionUrl(url, { connectionLimit: connectionLimit.toString(), @@ -1111,7 +1113,11 @@ function buildRunOpsClient({ client.$on("error", (log) => // The writer bridges P2002 -> 422 at the store boundary, so its infra errors are logged once // there (ignoreError). Replica errors are not on that write path, so they log normally. - logger.error("RunOpsPrismaClient error", { clientType, event: log, ...(isWriter ? { ignoreError: true } : {}) }) + logger.error("RunOpsPrismaClient error", { + clientType, + event: log, + ...(isWriter ? { ignoreError: true } : {}), + }) ); } diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts index e73f526e665..a4e4a64d36d 100644 --- a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts @@ -485,9 +485,9 @@ describe("routableKeys bound (the shard descriptor keys this deployment can rout }); it("returns new when the active list holds only non-routable keys (fail-safe to gen-1)", () => { - expect( - computeMintShard({ id: "env_1" }, deps({ set: ["z"] }, { routableKeys: ["a"] })) - ).toBe("new"); + expect(computeMintShard({ id: "env_1" }, deps({ set: ["z"] }, { routableKeys: ["a"] }))).toBe( + "new" + ); }); it("rejects a per-org pin to a non-routable key and falls through to the hash", () => { @@ -501,7 +501,10 @@ describe("routableKeys bound (the shard descriptor keys this deployment can rout it("with no routableKeys given, behaviour is unchanged", () => { const ids = envIds(200); for (const id of ids) { - const withBound = computeMintShard({ id }, deps({ set: ["a", "b"] }, { routableKeys: ["a", "b"] })); + const withBound = computeMintShard( + { id }, + deps({ set: ["a", "b"] }, { routableKeys: ["a", "b"] }) + ); const without = computeMintShard({ id }, deps({ set: ["a", "b"] })); expect(withBound).toBe(without); } diff --git a/apps/webapp/app/v3/runOpsPoolKnobs.server.ts b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts index f7bb7b72cf2..195f1305f36 100644 --- a/apps/webapp/app/v3/runOpsPoolKnobs.server.ts +++ b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts @@ -55,9 +55,7 @@ export function resolveRunOpsPoolKnobs( return { writerPoolTimeout: - k?.writerPoolTimeout ?? - env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? - env.DATABASE_POOL_TIMEOUT, + k?.writerPoolTimeout ?? env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, writerConnectionTimeout: k?.writerConnectionTimeout ?? env.RUN_OPS_DATABASE_WRITER_CONNECTION_TIMEOUT ?? diff --git a/apps/webapp/app/v3/runStore.server.ts b/apps/webapp/app/v3/runStore.server.ts index 3bc21c14097..c1cd6eafb19 100644 --- a/apps/webapp/app/v3/runStore.server.ts +++ b/apps/webapp/app/v3/runStore.server.ts @@ -1,5 +1,10 @@ import { PostgresRunStore, RoutingRunStore, type RunStore } from "@internal/run-store"; -import { ownerEngine, resolveShard, type Residency, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import { + ownerEngine, + resolveShard, + type Residency, + type ShardKey, +} from "@trigger.dev/core/v3/isomorphic"; import type { PrismaClient, PrismaReplicaClient } from "@trigger.dev/database"; import type { RunOpsPrismaClient } from "@internal/run-ops-database"; import { diff --git a/apps/webapp/test/runOpsShards.test.ts b/apps/webapp/test/runOpsShards.test.ts index 868ee8fa010..fef7e925e75 100644 --- a/apps/webapp/test/runOpsShards.test.ts +++ b/apps/webapp/test/runOpsShards.test.ts @@ -35,7 +35,10 @@ describe("parseRunOpsShards", () => { expect(run(JSON.stringify([{ ...valid, key: "ab" }])).success).toBe(false); }); it("fails on duplicate keys", () => { - const b = { ...valid, replication: { slotName: "s2", publicationName: "p2", originGeneration: 3 } }; + const b = { + ...valid, + replication: { slotName: "s2", publicationName: "p2", originGeneration: 3 }, + }; expect(run(JSON.stringify([valid, b])).success).toBe(false); }); it("fails on duplicate origin generations", () => { @@ -43,18 +46,28 @@ describe("parseRunOpsShards", () => { expect(run(JSON.stringify([valid, b])).success).toBe(false); }); it("fails when both url and aliasOf are set", () => { - expect(run(JSON.stringify([{ key: "a", region: "x", url: "postgres://h/db", aliasOf: "new" }])).success).toBe(false); + expect( + run(JSON.stringify([{ key: "a", region: "x", url: "postgres://h/db", aliasOf: "new" }])) + .success + ).toBe(false); }); it("accepts aliasOf without url or replication", () => { expect(run(JSON.stringify([{ key: "a", region: "x", aliasOf: "new" }])).success).toBe(true); }); it("fails on an origin generation below 2 or above 255", () => { - const mk = (g: number) => run(JSON.stringify([{ ...valid, replication: { slotName: "s", publicationName: "p", originGeneration: g } }])); + const mk = (g: number) => + run( + JSON.stringify([ + { ...valid, replication: { slotName: "s", publicationName: "p", originGeneration: g } }, + ]) + ); expect(mk(1).success).toBe(false); expect(mk(256).success).toBe(false); }); it("fails when a non-aliased descriptor omits replication", () => { - expect(run(JSON.stringify([{ key: "a", region: "x", url: "postgres://h/db" }])).success).toBe(false); + expect(run(JSON.stringify([{ key: "a", region: "x", url: "postgres://h/db" }])).success).toBe( + false + ); }); }); diff --git a/apps/webapp/test/transactionResilience.test.ts b/apps/webapp/test/transactionResilience.test.ts index 575d059b11c..a033f3be985 100644 --- a/apps/webapp/test/transactionResilience.test.ts +++ b/apps/webapp/test/transactionResilience.test.ts @@ -9,7 +9,9 @@ describe("resolveTransactionResilience per-shard", () => { }); it("accepts an arbitrary pool label and honours a maxWait override", () => { - expect(() => resolveTransactionResilience("run-ops-shard-z", { maxWaitMs: 1234 })).not.toThrow(); + expect(() => + resolveTransactionResilience("run-ops-shard-z", { maxWaitMs: 1234 }) + ).not.toThrow(); expect(resolveTransactionResilience("run-ops-shard-z", { maxWaitMs: 1234 }).maxWait).toBe(1234); }); }); From 20f0ee3635bc1ed989c50c3fea99452bb0db4d61 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:57:30 +0100 Subject: [PATCH 11/16] refactor(webapp): address review feedback on shard wiring - Make probeOrder a true reverse of precedence so the merge and probe paths agree on a duplicate id, matching the RoutingRunStore invariant. - Split resolveRunOpsPoolKnobs into a pure applyPoolKnobOverrides (tested with literal defaults, no env import) plus an env-reading defaults function. - Move the pure boot-table helpers to runOpsShardTable.ts so their test does not construct the db.server Prisma topology. Co-Authored-By: Claude Opus 4.8 --- apps/webapp/app/db.server.ts | 27 +----- apps/webapp/app/v3/runOpsPoolKnobs.server.ts | 89 +++++++++---------- apps/webapp/app/v3/runOpsShardTable.ts | 28 ++++++ apps/webapp/app/v3/runStore.server.ts | 12 ++- apps/webapp/test/runOpsPoolKnobs.test.ts | 67 +++++++------- apps/webapp/test/runOpsShardBootTable.test.ts | 2 +- 6 files changed, 116 insertions(+), 109 deletions(-) create mode 100644 apps/webapp/app/v3/runOpsShardTable.ts diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index 1658219a0c3..f6f1e519d4a 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -32,6 +32,7 @@ import { } from "./v3/runOpsMigration/splitMode.server"; import { computeRunOpsSplitReadEnabled } from "./v3/runOpsMigration/runOpsSplitReadGate"; import { resolveRunOpsPoolKnobs } from "./v3/runOpsPoolKnobs.server"; +import { buildRunOpsShardTable } from "./v3/runOpsShardTable"; import { resolveShardResilience, controlPlaneTransactionResilience, @@ -1160,32 +1161,6 @@ function redactUrlSecrets(hrefOrUrl: string | URL) { return url.href; } -// A host:port/db address, with NO username and NO query params — never a secret, and deliberately -// NOT an identity claim (two DSNs can share an address yet be different databases; that proof is the -// distinctness sentinel's, not this line's). Same tuple sameDatabaseTarget compares, kept in step. -export function runOpsAddressFingerprint(url: string): string { - try { - const u = new URL(url); - return `${u.hostname}:${u.port || "5432"}${u.pathname}`; - } catch { - return "unparseable"; - } -} - -export type RunOpsShardTableRow = { key: string; fingerprint: string; role: string }; - -// The resolved shard table for the boot log: one row per descriptor. An alias reports its role and -// carries no address (it shares the new store's pool). -export function buildRunOpsShardTable( - descriptors: Array<{ key: string; url?: string; aliasOf?: "new" }> -): RunOpsShardTableRow[] { - return descriptors.map((d) => - d.aliasOf - ? { key: d.key, fingerprint: "alias(new)", role: "alias(new)" } - : { key: d.key, fingerprint: runOpsAddressFingerprint(d.url ?? ""), role: "shard" } - ); -} - export type { PrismaClient } from "@trigger.dev/database"; function getDatabaseSchema() { diff --git a/apps/webapp/app/v3/runOpsPoolKnobs.server.ts b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts index 195f1305f36..0396af0f6d2 100644 --- a/apps/webapp/app/v3/runOpsPoolKnobs.server.ts +++ b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts @@ -1,8 +1,7 @@ import { env } from "~/env.server"; import type { RunOpsShardKnobs } from "~/v3/runOpsShards.server"; -// Pool configuration for one run-ops store (writer + replica), resolved at the app boundary (IoC). -// Every value reproduces today's run-ops builder expressions. Kept separate from db.server (which +// Pool configuration for one run-ops store (writer + replica). Kept separate from db.server (which // ~156 tests mock wholesale) so a new export breaks no mock. export type ResolvedPoolKnobs = { writerPoolTimeout: number; @@ -17,65 +16,65 @@ export type ResolvedPoolKnobs = { type Role = "new" | "legacy"; -// Resolve the pool knobs for a run-ops role, reproducing today's builder expressions exactly. -// descriptorKnobs (gen-2 shards only) override the pool fields. -// Transaction resilience is a SEPARATE mechanism (resolveTransactionResilience) and is not here. -export function resolveRunOpsPoolKnobs( - role: Role, - descriptorKnobs?: RunOpsShardKnobs +// PURE: overlay a gen-2 shard's descriptor knobs on a role's resolved defaults. This holds the only +// logic (per-field override), so a test drives it with literal defaults and literal overrides — +// no env import, no circular assertion against the same env expression the impl reads. +export function applyPoolKnobOverrides( + defaults: ResolvedPoolKnobs, + k?: RunOpsShardKnobs ): ResolvedPoolKnobs { - const k = descriptorKnobs; + return { + writerPoolTimeout: k?.writerPoolTimeout ?? defaults.writerPoolTimeout, + writerConnectionTimeout: k?.writerConnectionTimeout ?? defaults.writerConnectionTimeout, + writerDriverAdapter: k?.writerDriverAdapter ?? defaults.writerDriverAdapter, + connectionLimit: k?.connectionLimit ?? defaults.connectionLimit, + replicaConnectionLimit: k?.replicaConnectionLimit ?? defaults.replicaConnectionLimit, + replicaPoolTimeout: k?.replicaPoolTimeout ?? defaults.replicaPoolTimeout, + replicaConnectionTimeout: k?.replicaConnectionTimeout ?? defaults.replicaConnectionTimeout, + replicaDriverAdapter: k?.replicaDriverAdapter ?? defaults.replicaDriverAdapter, + }; +} +// The env-derived defaults for a role, reproducing today's run-ops builder expressions exactly. A +// flat mapping (no logic), verified by inspection against the former builders. Transaction +// resilience is a SEPARATE mechanism (resolveTransactionResilience) and is not here. +function poolKnobDefaults(role: Role): ResolvedPoolKnobs { if (role === "legacy") { return { writerPoolTimeout: - k?.writerPoolTimeout ?? - env.RUN_OPS_LEGACY_DATABASE_WRITER_POOL_TIMEOUT ?? - env.DATABASE_POOL_TIMEOUT, + env.RUN_OPS_LEGACY_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, writerConnectionTimeout: - k?.writerConnectionTimeout ?? - env.RUN_OPS_LEGACY_DATABASE_WRITER_CONNECTION_TIMEOUT ?? - env.DATABASE_CONNECTION_TIMEOUT, - writerDriverAdapter: - k?.writerDriverAdapter ?? env.RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER === "1", - connectionLimit: k?.connectionLimit ?? env.DATABASE_CONNECTION_LIMIT, - replicaConnectionLimit: k?.replicaConnectionLimit ?? env.DATABASE_CONNECTION_LIMIT, + env.RUN_OPS_LEGACY_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, + writerDriverAdapter: env.RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER === "1", + connectionLimit: env.DATABASE_CONNECTION_LIMIT, + replicaConnectionLimit: env.DATABASE_CONNECTION_LIMIT, replicaPoolTimeout: - k?.replicaPoolTimeout ?? - env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? - env.DATABASE_POOL_TIMEOUT, + env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, replicaConnectionTimeout: - k?.replicaConnectionTimeout ?? env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, - replicaDriverAdapter: - k?.replicaDriverAdapter ?? env.RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + replicaDriverAdapter: env.RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER === "1", }; } return { - writerPoolTimeout: - k?.writerPoolTimeout ?? env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + writerPoolTimeout: env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, writerConnectionTimeout: - k?.writerConnectionTimeout ?? - env.RUN_OPS_DATABASE_WRITER_CONNECTION_TIMEOUT ?? - env.DATABASE_CONNECTION_TIMEOUT, - writerDriverAdapter: - k?.writerDriverAdapter ?? env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1", - connectionLimit: k?.connectionLimit ?? env.DATABASE_CONNECTION_LIMIT, + env.RUN_OPS_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, + writerDriverAdapter: env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1", + connectionLimit: env.DATABASE_CONNECTION_LIMIT, replicaConnectionLimit: - k?.replicaConnectionLimit ?? - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? - env.DATABASE_CONNECTION_LIMIT, - replicaPoolTimeout: - k?.replicaPoolTimeout ?? - env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? - env.DATABASE_POOL_TIMEOUT, + env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT, + replicaPoolTimeout: env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, replicaConnectionTimeout: - k?.replicaConnectionTimeout ?? - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? - env.DATABASE_CONNECTION_TIMEOUT, - replicaDriverAdapter: - k?.replicaDriverAdapter ?? env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, + replicaDriverAdapter: env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1", }; } + +export function resolveRunOpsPoolKnobs( + role: Role, + descriptorKnobs?: RunOpsShardKnobs +): ResolvedPoolKnobs { + return applyPoolKnobOverrides(poolKnobDefaults(role), descriptorKnobs); +} diff --git a/apps/webapp/app/v3/runOpsShardTable.ts b/apps/webapp/app/v3/runOpsShardTable.ts new file mode 100644 index 00000000000..6741b3cb5b0 --- /dev/null +++ b/apps/webapp/app/v3/runOpsShardTable.ts @@ -0,0 +1,28 @@ +// Pure boot-table helpers. Dependency-free (no db.server, no env) so a test of these two string +// functions never constructs a Prisma client. db.server imports them for the boot log. + +// A host:port/db address, with NO username and NO query params — never a secret, and deliberately +// NOT an identity claim (two DSNs can share an address yet be different databases; that proof is the +// distinctness sentinel's, not this line's). Same tuple sameDatabaseTarget compares, kept in step. +export function runOpsAddressFingerprint(url: string): string { + try { + const u = new URL(url); + return `${u.hostname}:${u.port || "5432"}${u.pathname}`; + } catch { + return "unparseable"; + } +} + +export type RunOpsShardTableRow = { key: string; fingerprint: string; role: string }; + +// The resolved shard table for the boot log: one row per descriptor. An alias reports its role and +// carries no address (it shares the new store's pool). +export function buildRunOpsShardTable( + descriptors: Array<{ key: string; url?: string; aliasOf?: "new" }> +): RunOpsShardTableRow[] { + return descriptors.map((d) => + d.aliasOf + ? { key: d.key, fingerprint: "alias(new)", role: "alias(new)" } + : { key: d.key, fingerprint: runOpsAddressFingerprint(d.url ?? ""), role: "shard" } + ); +} diff --git a/apps/webapp/app/v3/runStore.server.ts b/apps/webapp/app/v3/runStore.server.ts index c1cd6eafb19..3fd0fcfa3d0 100644 --- a/apps/webapp/app/v3/runStore.server.ts +++ b/apps/webapp/app/v3/runStore.server.ts @@ -124,12 +124,16 @@ export function buildRunStore(deps: BuildRunStoreDeps): RunStore { ...shardStores.map(({ key, store }) => [key, store] as const), ]); + // Ascending authority for a merge: legacy -> new -> shards in configured order. The router + // requires probeOrder to be the exact reverse (see the class invariant in runOpsStore.ts), so a + // duplicate id resolves the same way on the merge path and the probe path. + const precedence: ShardKey[] = ["legacy", "new", ...shardKeys]; + const probeOrder = [...precedence].reverse(); + return RoutingRunStore.fromShards({ shards: shardMap, - // Ascending authority for a merge: legacy -> new -> shards in configured order. - precedence: ["legacy", "new", ...shardKeys], - // Probe order for an id-less lookup: the reverse of precedence. - probeOrder: ["new", ...shardKeys, "legacy"], + precedence, + probeOrder, idlessRouteShard: "new", idlessWaitpointShard: "legacy", resolveShardKey: deps.resolveShardKey ?? resolveShard, diff --git a/apps/webapp/test/runOpsPoolKnobs.test.ts b/apps/webapp/test/runOpsPoolKnobs.test.ts index e281b8ce19e..38353682cb2 100644 --- a/apps/webapp/test/runOpsPoolKnobs.test.ts +++ b/apps/webapp/test/runOpsPoolKnobs.test.ts @@ -1,41 +1,42 @@ import { describe, expect, it } from "vitest"; -import { resolveRunOpsPoolKnobs } from "~/v3/runOpsPoolKnobs.server"; -import { env } from "~/env.server"; +import { applyPoolKnobOverrides, type ResolvedPoolKnobs } from "~/v3/runOpsPoolKnobs.server"; -describe("resolveRunOpsPoolKnobs", () => { - it("new role: reproduces the run-ops builder expressions", () => { - const k = resolveRunOpsPoolKnobs("new"); - expect(k.connectionLimit).toBe(env.DATABASE_CONNECTION_LIMIT); - expect(k.replicaConnectionLimit).toBe( - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT - ); - expect(k.writerPoolTimeout).toBe( - env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT - ); - expect(k.replicaPoolTimeout).toBe( - env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT - ); - expect(k.writerDriverAdapter).toBe(env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1"); - expect(k.replicaDriverAdapter).toBe(env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1"); +// Literal defaults, so the assertions lock the override logic against fixed values rather than +// against the same env expression the implementation reads. No env import (webapp test rule). +const DEFAULTS: ResolvedPoolKnobs = { + writerPoolTimeout: 10, + writerConnectionTimeout: 20, + writerDriverAdapter: false, + connectionLimit: 30, + replicaConnectionLimit: 40, + replicaPoolTimeout: 50, + replicaConnectionTimeout: 60, + replicaDriverAdapter: false, +}; + +describe("applyPoolKnobOverrides", () => { + it("returns the defaults verbatim when no descriptor knobs are given", () => { + expect(applyPoolKnobOverrides(DEFAULTS)).toEqual(DEFAULTS); + expect(applyPoolKnobOverrides(DEFAULTS, {})).toEqual(DEFAULTS); }); - it("legacy role: uses RUN_OPS_LEGACY_* timeouts and the generic connection limit", () => { - const k = resolveRunOpsPoolKnobs("legacy"); - expect(k.connectionLimit).toBe(env.DATABASE_CONNECTION_LIMIT); - expect(k.replicaConnectionLimit).toBe(env.DATABASE_CONNECTION_LIMIT); - expect(k.writerPoolTimeout).toBe( - env.RUN_OPS_LEGACY_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT - ); - expect(k.replicaPoolTimeout).toBe( - env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT - ); - expect(k.writerDriverAdapter).toBe(env.RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER === "1"); - expect(k.replicaDriverAdapter).toBe(env.RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER === "1"); + it("overrides only the fields the descriptor sets", () => { + const result = applyPoolKnobOverrides(DEFAULTS, { + connectionLimit: 999, + writerDriverAdapter: true, + replicaPoolTimeout: 555, + }); + expect(result.connectionLimit).toBe(999); + expect(result.writerDriverAdapter).toBe(true); + expect(result.replicaPoolTimeout).toBe(555); + // Untouched fields keep the defaults. + expect(result.writerPoolTimeout).toBe(10); + expect(result.replicaConnectionLimit).toBe(40); + expect(result.replicaDriverAdapter).toBe(false); }); - it("a descriptor knob overrides its field", () => { - const k = resolveRunOpsPoolKnobs("new", { connectionLimit: 7, writerDriverAdapter: true }); - expect(k.connectionLimit).toBe(7); - expect(k.writerDriverAdapter).toBe(true); + it("does not read the transaction knobs off the descriptor", () => { + const result = applyPoolKnobOverrides(DEFAULTS, { transactionMaxWaitMs: 1234 }); + expect(result).toEqual(DEFAULTS); }); }); diff --git a/apps/webapp/test/runOpsShardBootTable.test.ts b/apps/webapp/test/runOpsShardBootTable.test.ts index 9c7706e3cbd..3031608d701 100644 --- a/apps/webapp/test/runOpsShardBootTable.test.ts +++ b/apps/webapp/test/runOpsShardBootTable.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { runOpsAddressFingerprint, buildRunOpsShardTable } from "~/db.server"; +import { runOpsAddressFingerprint, buildRunOpsShardTable } from "~/v3/runOpsShardTable"; describe("runOpsAddressFingerprint", () => { it("returns host:port/db with no username or query params", () => { From 908fcb587e7e622faea68339bfc74e0cf0f101a5 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:59:43 +0100 Subject: [PATCH 12/16] test(webapp): cover the dedicated-shard misconfiguration throw in selectRunOpsTopology Co-Authored-By: Claude Opus 4.8 --- apps/webapp/test/runOpsDbTopology.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/apps/webapp/test/runOpsDbTopology.test.ts b/apps/webapp/test/runOpsDbTopology.test.ts index f6895a03d29..f2bcc0bf5ea 100644 --- a/apps/webapp/test/runOpsDbTopology.test.ts +++ b/apps/webapp/test/runOpsDbTopology.test.ts @@ -193,6 +193,24 @@ describe("selectRunOpsTopology (pure)", () => { expect(buildShardWriter).not.toHaveBeenCalled(); expect(buildShardReplica).not.toHaveBeenCalled(); }); + + it("throws when a non-aliased shard has no url (guards the shard.url non-null assertion)", () => { + expect(() => + selectRunOpsTopology( + { ...baseSplit, shards: [{ key: "a" }] }, + { ...baseBuilders(), buildShardWriter: vi.fn(), buildShardReplica: vi.fn() } + ) + ).toThrow(/shard "a" needs a url/); + }); + + it("throws when a non-aliased shard is configured but the shard builders are absent", () => { + expect(() => + selectRunOpsTopology( + { ...baseSplit, shards: [{ key: "a", url: "postgres://a" }] }, + baseBuilders() + ) + ).toThrow(/shard "a" needs a url and shard builders/); + }); }); describe("sameDatabaseTarget", () => { From 67163dd127c096b3fc41619e8ee2d166515fc56b Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 12:51:55 +0100 Subject: [PATCH 13/16] fix(webapp): don't let an unreachable run-ops shard crash webapp startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The run-ops client factory eagerly $connects for warm-up, but only caught the rejection under NODE_ENV=test — outside test an unreachable shard/run-ops DB at boot surfaced as an unhandled promise rejection. Always catch and log instead; Prisma reconnects lazily on first query, so one unreachable shard must not take down startup. Scoped to the run-ops factory only; the control-plane/legacy builders are unchanged, so the RUN_OPS_SHARDS-unset path stays byte-identical. Co-Authored-By: Claude Opus 4.8 --- apps/webapp/app/db.server.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index f6f1e519d4a..11a8aa307e9 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -1124,12 +1124,13 @@ function buildRunOpsClient({ client.$on("query", (log) => queryPerformanceMonitor.onQuery(role, log)); - const connectPromise = client.$connect(); - if (env.NODE_ENV === "test") { - connectPromise.catch((error) => { - logger.warn(`Failed to eagerly connect run-ops prisma client (${role})`, { error }); - }); - } + // Eager connect is a warm-up only — Prisma reconnects lazily on first query. ALWAYS catch the + // rejection (not just under NODE_ENV=test), so a shard/run-ops DB that is unreachable at boot + // logs a warning instead of surfacing as an unhandled promise rejection. One unreachable shard + // must not take down webapp startup. + client.$connect().catch((error) => { + logger.warn(`Failed to eagerly connect run-ops prisma client (${role})`, { error }); + }); console.log(`🔌 ${connectedLabel}`); From ff1f6ca19657e6c026c937a336c71595084ba6da Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 16:55:08 +0100 Subject: [PATCH 14/16] fix(run-store): fan out waitpoint resolution across all shards, not just the first When a gen-2 shard is configured, RoutingRunStore builds three or more stores (legacy + new + shard) and a waitpoint that is not on its home/run store must be found by probing the others. Three call sites took only the first "other" store (`#shardsExcept(key)[0]`), which was correct with the two-store compat router but silently skips the remaining stores once a shard exists. The effect, observed with a single shard configured: waitpoint lookups return "Waitpoint not found", pending-token counts undercount (which prematurely unblocks a still-waiting run), and many-waitpoint reads miss rows. Fix `#resolveWaitpointStore`, `countPendingWaitpoints` and `#collectManyWaitpoints` to fan out over every other store and merge. Adds a routing unit test that reproduces all three at the production probe order, with the target placed on the store the first-other truncation skipped. Co-Authored-By: Claude Opus 4.8 --- .../run-store/src/runOpsStore.ts | 52 +++++----- .../runOpsStore.waitpointShardFanout.test.ts | 95 +++++++++++++++++++ 2 files changed, 123 insertions(+), 24 deletions(-) create mode 100644 internal-packages/run-store/src/runOpsStore.waitpointShardFanout.test.ts diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index 8d1c7d5aba4..b697182e7e5 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -247,8 +247,8 @@ export class RoutingRunStore implements RunStore { } // Every shard other than `key`, in probe order. With the compat constructor this yields exactly - // one entry, which is why each caller may take the first. At more than two shards a caller MUST - // fan out over all of them instead. + // one entry; with three+ stores it yields several, so callers fan out over all of them (a waitpoint + // absent from its home/run store can live on any one of the others). #shardsExcept(key: ShardKey): Array<{ key: ShardKey; store: RunStore }> { return this.#probeOrder .filter((k) => k !== key) @@ -333,16 +333,14 @@ export class RoutingRunStore implements RunStore { ) { return home; } - const [other] = this.#shardsExcept(homeKey); - if (other === undefined) { - return home; + for (const { store } of this.#shardsExcept(homeKey)) { + if ( + await store.findWaitpoint({ where: { id } }, onPrimary ? store.primaryReadClient : undefined) + ) { + return store; + } } - return (await other.store.findWaitpoint( - { where: { id } }, - onPrimary ? other.store.primaryReadClient : undefined - )) - ? other.store - : home; + return home; } static #waitpointId(clause: unknown): string | undefined { @@ -1231,15 +1229,16 @@ export class RoutingRunStore implements RunStore { if (missing.length === 0) { return pendingIds.length; } - const [other] = this.#shardsExcept(runKey); - if (other === undefined) { + const others = this.#shardsExcept(runKey); + if (others.length === 0) { return pendingIds.length; } - const otherPending = await other.store.countPendingWaitpoints( - missing, - RoutingRunStore.#ownPrimary(other.store, client) + const otherPending = await Promise.all( + others.map(({ store }) => + store.countPendingWaitpoints(missing, RoutingRunStore.#ownPrimary(store, client)) + ) ); - return pendingIds.length + otherPending; + return pendingIds.length + otherPending.reduce((sum, n) => sum + n, 0); } // Fan out and union: an id lives on exactly one store in steady state (a drain-mirror can put it on @@ -1433,15 +1432,20 @@ export class RoutingRunStore implements RunStore { if (missing.length === 0) { return fromRun; } - const [other] = this.#shardsExcept(runKey); - if (other === undefined) { + const others = this.#shardsExcept(runKey); + if (others.length === 0) { return fromRun; } - const fromOther = (await other.store.findManyWaitpoints( - narrowArgsToIds(scalarArgs, missing) as Prisma.WaitpointFindManyArgs, - RoutingRunStore.#ownPrimary(other.store, client) - )) as Record[]; - return [...fromRun, ...fromOther]; + const fromOthers = await Promise.all( + others.map( + ({ store }) => + store.findManyWaitpoints( + narrowArgsToIds(scalarArgs, missing) as Prisma.WaitpointFindManyArgs, + RoutingRunStore.#ownPrimary(store, client) + ) as Promise[]> + ) + ); + return [...fromRun, ...fromOthers.flat()]; } // No bounded id set to partition on → fall through to the fan-out path. } diff --git a/internal-packages/run-store/src/runOpsStore.waitpointShardFanout.test.ts b/internal-packages/run-store/src/runOpsStore.waitpointShardFanout.test.ts new file mode 100644 index 00000000000..13545fa043c --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.waitpointShardFanout.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; +import { generateRunOpsId, resolveShard, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { ReadClient, RunStore } from "./types.js"; + +// Regression guard for the N-way waitpoint fan-out. A waitpoint that is not on its home/run store is +// resolved by probing the OTHER shards. With two stores there is exactly one other, so taking the +// first was correct; with three+ stores (legacy + new + a gen-2 shard) taking only the first other +// silently skips the rest, missing a waitpoint that lives on a later shard. Each scenario places the +// target on the store the old first-other truncation skipped (empty shard "a" sorts first). + +type Held = { waitpoints?: string[]; pending?: string[] }; +type FakeStore = RunStore & { slot: ShardKey }; + +function idsFromArgs(args: unknown): string[] { + const where = (args as { where?: { id?: unknown } })?.where ?? {}; + const id = where.id; + if (typeof id === "string") return [id]; + if (id && typeof id === "object" && Array.isArray((id as { in?: unknown[] }).in)) { + return (id as { in: unknown[] }).in.filter((x): x is string => typeof x === "string"); + } + return []; +} + +function fakeStore(slot: ShardKey, held: Held = {}): FakeStore { + const has = new Set(held.waitpoints ?? []); + const pending = new Set(held.pending ?? []); + const findWaitpoint = (args: unknown) => { + const [id] = idsFromArgs(args); + return Promise.resolve(id && has.has(id) ? ({ id, slot } as never) : null); + }; + const store: Partial = { + slot, + primaryReadClient: { __primary: slot } as unknown as ReadClient, + findWaitpoint: findWaitpoint as FakeStore["findWaitpoint"], + findWaitpointOnPrimary: findWaitpoint as FakeStore["findWaitpointOnPrimary"], + countPendingWaitpoints: ((ids: string[]) => + Promise.resolve(ids.filter((id) => pending.has(id)).length)) as FakeStore["countPendingWaitpoints"], + countPendingWaitpointsWithPresence: ((ids: string[]) => + Promise.resolve({ + pendingIds: ids.filter((id) => pending.has(id)), + presentIds: ids.filter((id) => has.has(id)), + })) as FakeStore["countPendingWaitpointsWithPresence"], + findManyWaitpoints: ((args: unknown) => + Promise.resolve( + idsFromArgs(args) + .filter((id) => has.has(id)) + .map((id) => ({ id, slot })) + )) as unknown as FakeStore["findManyWaitpoints"], + }; + return store as FakeStore; +} + +// Production topology: precedence legacy -> new -> shards; probeOrder its exact reverse. +function build(stores: { legacy?: Held; new?: Held; a?: Held }) { + const shards = new Map(); + shards.set("legacy", fakeStore("legacy", stores.legacy)); + shards.set("new", fakeStore("new", stores.new)); + shards.set("a", fakeStore("a", stores.a)); + return RoutingRunStore.fromShards({ + shards, + probeOrder: ["a", "new", "legacy"], + precedence: ["legacy", "new", "a"], + idlessRouteShard: "new", + idlessWaitpointShard: "legacy", + resolveShardKey: resolveShard, + }); +} + +// A cuid waitpoint id resolves home to "legacy"; a gen-1 run id routes the run store to "new". +const CUID_WAITPOINT = "clabc123def456ghi789jkl01"; + +describe("RoutingRunStore N-way waitpoint fan-out", () => { + it("#resolveWaitpointStore finds a waitpoint that lives past the first other shard", async () => { + const store = build({ new: { waitpoints: [CUID_WAITPOINT] } }); + const row = await store.findWaitpoint({ where: { id: CUID_WAITPOINT } }); + expect(row).toMatchObject({ id: CUID_WAITPOINT, slot: "new" }); + }); + + it("countPendingWaitpoints counts a pending token on a later shard (never undercounts)", async () => { + const runId = generateRunOpsId(); // gen-1 -> routes to "new" + const token = "wp_pending_on_legacy"; + const store = build({ legacy: { waitpoints: [token], pending: [token] } }); + const count = await store.countPendingWaitpoints([token], undefined, runId); + expect(count).toBe(1); + }); + + it("#collectManyWaitpoints collects a waitpoint on a later shard", async () => { + const runId = generateRunOpsId(); // gen-1 -> routes to "new" + const token = "wp_on_legacy"; + const store = build({ legacy: { waitpoints: [token] } }); + const rows = await store.findManyWaitpoints({ where: { id: { in: [token] } } }, undefined, runId); + expect(rows).toEqual([{ id: token, slot: "legacy" }]); + }); +}); From 24eb536301c363cffd709d2160f199ddfc7c0486 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 17:33:42 +0100 Subject: [PATCH 15/16] test(run-store): drop mock-based waitpoint fan-out test; main's nShardMatrix covers it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Main's runOpsStore.nShardMatrix.test.ts is a four-store testcontainer matrix that already guards the N-way waitpoint fan-out on real databases — the gen-2-shard union with no double count, the mirrored-cuid case, alias dedup, and cross-tree completion. The removed test used fakeStore() stubs, which both duplicates that coverage and violates the repo's "never mock, use testcontainers" rule (CodeRabbit). Removing it also clears the code-quality oxfmt --check failure the unformatted file caused. --- .../runOpsStore.waitpointShardFanout.test.ts | 96 ------------------- 1 file changed, 96 deletions(-) delete mode 100644 internal-packages/run-store/src/runOpsStore.waitpointShardFanout.test.ts diff --git a/internal-packages/run-store/src/runOpsStore.waitpointShardFanout.test.ts b/internal-packages/run-store/src/runOpsStore.waitpointShardFanout.test.ts deleted file mode 100644 index f1eb9e3335f..00000000000 --- a/internal-packages/run-store/src/runOpsStore.waitpointShardFanout.test.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - generateRunOpsId, - generateRunOpsIdV2, - resolveShard, - type ShardKey, -} from "@trigger.dev/core/v3/isomorphic"; -import { RoutingRunStore } from "./runOpsStore.js"; -import type { ReadClient, RunStore } from "./types.js"; - -// Regression guard for the N-way waitpoint fan-out. A waitpoint that is not on its home/run store is -// resolved by probing the other stores: a cuid can only be drain-relocated between the two gen-1 -// stores, and a gen-2 id names exactly one shard. Each scenario places the target on a store other -// than the run's own and asserts the router still finds/counts it. - -type Held = { waitpoints?: string[]; pending?: string[] }; -type FakeStore = RunStore & { slot: ShardKey }; - -function idsFromArgs(args: unknown): string[] { - const where = (args as { where?: { id?: unknown } })?.where ?? {}; - const id = where.id; - if (typeof id === "string") return [id]; - if (id && typeof id === "object" && Array.isArray((id as { in?: unknown[] }).in)) { - return (id as { in: unknown[] }).in.filter((x): x is string => typeof x === "string"); - } - return []; -} - -function fakeStore(slot: ShardKey, held: Held = {}): FakeStore { - const has = new Set(held.waitpoints ?? []); - const pending = new Set(held.pending ?? []); - const findWaitpoint = (args: unknown) => { - const [id] = idsFromArgs(args); - return Promise.resolve(id && has.has(id) ? ({ id, slot } as never) : null); - }; - const store: Partial = { - slot, - primaryReadClient: { __primary: slot } as unknown as ReadClient, - findWaitpoint: findWaitpoint as FakeStore["findWaitpoint"], - findWaitpointOnPrimary: findWaitpoint as FakeStore["findWaitpointOnPrimary"], - countPendingWaitpoints: ((ids: string[]) => - Promise.resolve(ids.filter((id) => pending.has(id)).length)) as FakeStore["countPendingWaitpoints"], - countPendingWaitpointsWithPresence: ((ids: string[]) => - Promise.resolve({ - pendingIds: ids.filter((id) => pending.has(id)), - presentIds: ids.filter((id) => has.has(id)), - })) as FakeStore["countPendingWaitpointsWithPresence"], - findManyWaitpoints: ((args: unknown) => - Promise.resolve( - idsFromArgs(args) - .filter((id) => has.has(id)) - .map((id) => ({ id, slot })) - )) as unknown as FakeStore["findManyWaitpoints"], - }; - return store as FakeStore; -} - -// One gen-2 shard "a" alongside the gen-1 pair. The constructor derives probe/precedence order. -function build(stores: { legacy?: Held; new?: Held; a?: Held }) { - return new RoutingRunStore({ - new: fakeStore("new", stores.new), - legacy: fakeStore("legacy", stores.legacy), - shards: [{ key: "a", store: fakeStore("a", stores.a) }], - resolveShard, - }); -} - -// A cuid waitpoint id resolves home to "legacy"; a gen-1 run id routes the run store to "new". -const CUID_WAITPOINT = "clabc123def456ghi789jkl01"; - -describe("RoutingRunStore N-way waitpoint fan-out", () => { - it("resolves a cuid waitpoint that was drain-relocated onto the other gen-1 store", async () => { - const store = build({ new: { waitpoints: [CUID_WAITPOINT] } }); - const row = await store.findWaitpoint({ where: { id: CUID_WAITPOINT } }); - expect(row).toMatchObject({ id: CUID_WAITPOINT, slot: "new" }); - }); - - it("counts a pending cuid token on the other gen-1 store (never undercounts)", async () => { - const runId = generateRunOpsId(); // gen-1 -> run store is "new" - const store = build({ legacy: { waitpoints: [CUID_WAITPOINT], pending: [CUID_WAITPOINT] } }); - const count = await store.countPendingWaitpoints([CUID_WAITPOINT], undefined, runId); - expect(count).toBe(1); - }); - - it("collects a gen-2 waitpoint that lives on its own shard, not the run's store", async () => { - const runId = generateRunOpsId(); // gen-1 -> run store is "new" - const shardWaitpoint = generateRunOpsIdV2("a"); // resolves to shard "a" - const store = build({ a: { waitpoints: [shardWaitpoint] } }); - const rows = await store.findManyWaitpoints( - { where: { id: { in: [shardWaitpoint] } } }, - undefined, - runId - ); - expect(rows).toEqual([{ id: shardWaitpoint, slot: "a" }]); - }); -}); From aa2bb6835314c3b9769d29a8cdb9a95738fdfa5b Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 17:38:00 +0100 Subject: [PATCH 16/16] test(run-store): remove stale fromShards test after merging main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runOpsStore.fromShards.test.ts imported UnknownShardKey and called RoutingRunStore.fromShards — both removed in main's RoutingRunStore refactor (constructor-based shards). The file is unique to this branch and now references APIs that no longer exist, so it fails the run-store suite. Its routing coverage lives in main's shardMap/runKeyedRouting/nShardMatrix tests. --- .../src/runOpsStore.fromShards.test.ts | 65 ------------------- 1 file changed, 65 deletions(-) delete mode 100644 internal-packages/run-store/src/runOpsStore.fromShards.test.ts diff --git a/internal-packages/run-store/src/runOpsStore.fromShards.test.ts b/internal-packages/run-store/src/runOpsStore.fromShards.test.ts deleted file mode 100644 index e9f55332ad7..00000000000 --- a/internal-packages/run-store/src/runOpsStore.fromShards.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - generateRunOpsId, - generateRunOpsIdV2, - resolveShard, - type ShardKey, -} from "@trigger.dev/core/v3/isomorphic"; -import { RoutingRunStore, UnknownShardKey } from "./runOpsStore.js"; -import type { ReadClient, RunStore } from "./types.js"; - -// Pure routing unit test for the N-way fromShards factory. Each shard is a fake RunStore whose -// findRun records which slot answered, so the assertions are purely about WHICH store the router -// selects. No database. -type FakeStore = RunStore & { slot: ShardKey }; - -function fakeStore(slot: ShardKey): FakeStore { - const store: Partial = { - slot, - primaryReadClient: { __primary: slot } as unknown as ReadClient, - findRun: ((_where: unknown, _argsOrClient?: unknown, _client?: unknown) => - Promise.resolve({ slot } as never)) as FakeStore["findRun"], - }; - return store as FakeStore; -} - -function build(shardKeys: ShardKey[]) { - const shards = new Map(); - shards.set("legacy", fakeStore("legacy")); - shards.set("new", fakeStore("new")); - for (const k of shardKeys) shards.set(k, fakeStore(k)); - return RoutingRunStore.fromShards({ - shards, - probeOrder: ["new", ...shardKeys, "legacy"], - precedence: ["legacy", "new", ...shardKeys], - idlessRouteShard: "new", - idlessWaitpointShard: "legacy", - resolveShardKey: resolveShard, - }); -} - -describe("RoutingRunStore.fromShards", () => { - it("routes a gen-2 id to its own shard, not to new", async () => { - const store = build(["a"]); - const found = await store.findRun({ friendlyId: generateRunOpsIdV2("a") }); - expect(found).toMatchObject({ slot: "a" }); - }); - - it("routes a gen-1 v1 id to new", async () => { - const store = build(["a"]); - const found = await store.findRun({ friendlyId: generateRunOpsId() }); - expect(found).toMatchObject({ slot: "new" }); - }); - - it("routes a cuid id to legacy", async () => { - const store = build(["a"]); - const found = await store.findRun({ friendlyId: "clabc123def456ghi789jkl01" }); - expect(found).toMatchObject({ slot: "legacy" }); - }); - - it("raises UnknownShardKey for an unconfigured shard and does not fall back", () => { - const store = build(["a"]); // "b" is not configured - // The route resolves synchronously, so the throw is synchronous (before the promise is built). - expect(() => store.findRun({ friendlyId: generateRunOpsIdV2("b") })).toThrow(UnknownShardKey); - }); -});