Skip to content

Commit 3c00f14

Browse files
committed
test(run-store,run-engine): cover dedicated snapshot-write atomicity and drop an unused engine
Add atomicity coverage for the dedicated (#new) leg of createExecutionSnapshot and lockRunToWorker: both must roll the snapshot back when the CompletedWaitpoint join insert fails, so a lagging replica can never serve a waitpoint-less resume. The existing coverage only exercised the legacy _completedWaitpoints path. Also drop an unused RunEngine from the duplicate-order repair test; it drives getExecutionSnapshotsSince directly and needs no Redis or worker resources.
1 parent 339ed71 commit 3c00f14

2 files changed

Lines changed: 115 additions & 19 deletions

File tree

internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1473,24 +1473,10 @@ describe("RunEngine getSnapshotsSince", () => {
14731473
// fires on every poll for such a snapshot even against a caught-up replica - silently defeating offload.
14741474
containerTest(
14751475
"does not repair from the primary when completedWaitpointOrder has duplicates but the join is complete",
1476-
async ({ prisma, redisOptions }) => {
1476+
async ({ prisma }) => {
14771477
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
1478-
const engine = new RunEngine({
1479-
prisma,
1480-
worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 },
1481-
queue: { redis: redisOptions },
1482-
runLock: { redis: redisOptions },
1483-
machines: {
1484-
defaultMachine: "small-1x",
1485-
machines: {
1486-
"small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 },
1487-
},
1488-
baseCostInCents: 0.0001,
1489-
},
1490-
tracer: trace.getTracer("test", "0.0.0"),
1491-
});
1492-
1493-
try {
1478+
// Drives getExecutionSnapshotsSince directly, so it needs no RunEngine (Redis/worker) at all.
1479+
{
14941480
const scenario = await setupTestScenario(prisma, authenticatedEnvironment, {
14951481
totalWaitpoints: 1,
14961482
outputSizeKB: 1,
@@ -1538,8 +1524,6 @@ describe("RunEngine getSnapshotsSince", () => {
15381524
expect(latest.completedWaitpoints.map((w) => w.id)).toEqual([waitpointId, waitpointId]);
15391525
// No spurious primary read: distinct(order) === join count, so the repair must not fire.
15401526
expect(repairCalls.queryRaw).toBe(0);
1541-
} finally {
1542-
await engine.quit();
15431527
}
15441528
}
15451529
);

internal-packages/run-store/src/PostgresRunStore.writeAtomicity.test.ts

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -594,3 +594,115 @@ describe("createRun / createFailedRun write the run and its associated waitpoint
594594
}
595595
);
596596
});
597+
598+
// The dedicated (#new) leg connects completed waitpoints through the `CompletedWaitpoint` join table
599+
// (createMany), where the legacy leg uses the implicit `_completedWaitpoints` M2M. Both must commit the
600+
// snapshot and its links together: a snapshot that commits before its links can be read waitpoint-less
601+
// from a lagging replica, and the runner's EXECUTING branch no-ops on an empty set -> the resume hangs.
602+
describe("createExecutionSnapshot / lockRunToWorker write the snapshot and its links atomically (dedicated)", () => {
603+
heteroRunOpsPostgresTest(
604+
"createExecutionSnapshot rolls the snapshot back if the CompletedWaitpoint insert fails",
605+
async ({ prisma17 }) => {
606+
const newStore = makeDedicatedStore(prisma17);
607+
const env = await seedEnvironment(prisma17, "dedicated", "ces_ded");
608+
const runId = `run_${NEW_ID_26}`;
609+
await newStore.createRun(
610+
buildCreateRunInput({
611+
runId,
612+
friendlyId: "run_ces_ded",
613+
organizationId: env.organization.id,
614+
projectId: env.project.id,
615+
runtimeEnvironmentId: env.environment.id,
616+
})
617+
);
618+
619+
// Force the dedicated join insert (completedWaitpoint.createMany) to fail mid-write.
620+
await prisma17.$executeRawUnsafe('DROP TABLE "CompletedWaitpoint"');
621+
622+
await expect(
623+
// Base client as `tx` = how the engine threads its base prisma through
624+
// (continueRunIfUnblocked -> executionSnapshotSystem.createExecutionSnapshot(prisma, ...)).
625+
// It is NOT an interactive transaction, so the store must still open its own to stay atomic.
626+
newStore.createExecutionSnapshot(
627+
{
628+
run: { id: runId, status: "EXECUTING", attemptNumber: 1 },
629+
snapshot: {
630+
executionStatus: "EXECUTING_WITH_WAITPOINTS",
631+
description: "Run was blocked by a waitpoint.",
632+
},
633+
environmentId: env.environment.id,
634+
environmentType: "DEVELOPMENT",
635+
projectId: env.project.id,
636+
organizationId: env.project.id,
637+
completedWaitpoints: [{ id: `wp_${NEW_ID_26}`, index: 0 }],
638+
},
639+
prisma17 as never
640+
)
641+
).rejects.toThrow();
642+
643+
const snap = await prisma17.taskRunExecutionSnapshot.findFirst({
644+
where: { runId, executionStatus: "EXECUTING_WITH_WAITPOINTS" },
645+
});
646+
expect(snap).toBeNull();
647+
}
648+
);
649+
650+
heteroRunOpsPostgresTest(
651+
"lockRunToWorker rolls the snapshot and run lock back if the CompletedWaitpoint insert fails",
652+
async ({ prisma17 }) => {
653+
const newStore = makeDedicatedStore(prisma17);
654+
const env = await seedEnvironment(prisma17, "dedicated", "lock_ded");
655+
const runId = `run_${NEW_ID_26}`;
656+
await newStore.createRun(
657+
buildCreateRunInput({
658+
runId,
659+
friendlyId: "run_lock_ded",
660+
organizationId: env.organization.id,
661+
projectId: env.project.id,
662+
runtimeEnvironmentId: env.environment.id,
663+
})
664+
);
665+
const prior = await prisma17.taskRunExecutionSnapshot.findFirstOrThrow({ where: { runId } });
666+
667+
await prisma17.$executeRawUnsafe('DROP TABLE "CompletedWaitpoint"');
668+
669+
const snapshotId = `snap_${NEW_ID_26}`;
670+
await expect(
671+
// lockedById/lockedToVersionId/lockedQueueId are FK-free scalars on the dedicated subset, so
672+
// synthetic ids are fine; the base client as `tx` mirrors the dequeue path (no interactive tx).
673+
newStore.lockRunToWorker(
674+
runId,
675+
{
676+
lockedAt: new Date(),
677+
lockedById: `bwt_${NEW_ID_26}`,
678+
lockedToVersionId: `bw_${NEW_ID_26}`,
679+
lockedQueueId: `queue_${NEW_ID_26}`,
680+
startedAt: new Date(),
681+
baseCostInCents: 5,
682+
machinePreset: "small-1x",
683+
taskVersion: "20260601.1",
684+
sdkVersion: "3.0.0",
685+
cliVersion: "3.0.0",
686+
maxDurationInSeconds: null,
687+
snapshot: {
688+
id: snapshotId,
689+
previousSnapshotId: prior.id,
690+
environmentId: env.environment.id,
691+
environmentType: "DEVELOPMENT",
692+
projectId: env.project.id,
693+
organizationId: env.project.id,
694+
completedWaitpointIds: [`wp_${NEW_ID_26}`],
695+
completedWaitpointOrder: [`wp_${NEW_ID_26}`],
696+
},
697+
},
698+
prisma17 as never
699+
)
700+
).rejects.toThrow();
701+
702+
const snap = await prisma17.taskRunExecutionSnapshot.findUnique({ where: { id: snapshotId } });
703+
expect(snap).toBeNull();
704+
const run = await prisma17.taskRun.findUniqueOrThrow({ where: { id: runId } });
705+
expect(run.status).not.toBe("DEQUEUED");
706+
}
707+
);
708+
});

0 commit comments

Comments
 (0)