fix(webapp): move region and error_fingerprint filters to PREWHERE

Both are set once and never change across a run version history, so they are
safe in PREWHERE (evaluated before FINAL): error_fingerprint is empty until a
terminal error status and then fixed (those statuses are final, so it is never
cleared), and region is set once at dequeue and never reassigned. The region
filter also drops the worker_queue fallback and matches on region directly,
since region is populated for effectively all non-development runs and the
fallback only affected not-yet-dequeued runs (which have not run in any region
yet). status and machine_preset stay in WHERE because they genuinely change as
a run runs (lifecycle, and OOM machine escalation).

Author: ericallam

apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts

@@ -416,17 +416,16 @@ export class ClickHouseRunsRepository implements IRunsRepository {
  *
  * A filter may go in PREWHERE only if its truth value can never flip true->false across a run's
  * versions, because PREWHERE is evaluated before FINAL reconciles versions and would otherwise keep
- * a stale matching version and drop the winning one. That holds for trigger-time identity columns
- * that never change (task_identifier, task_version, schedule_id, is_test, root_run_id, batch_id,
- * friendly_id, queue, task_kind) and for append-only arrays under `hasAny`/`hasAll` (tags,
- * bulk_action_group_ids), so those go in PREWHERE to filter (and, for tags, use the skip index)
- * before FINAL and before materialising the wide columns, which is what bounds memory on these
- * scans. Columns that reflect execution/outcome and change as a run runs stay in WHERE (post-FINAL):
- * `status`, `machine_preset` (can escalate on OOM retry), `error_fingerprint` (set/cleared with
- * status), and the `regions` expression (`if(region != '', region, worker_queue)` yields the
- * worker_queue before dequeue and the region after). The `(organization_id, project_id,
- * environment_id)` primary-key prefix and the `created_at` range also stay in WHERE so they keep
- * driving primary-key and partition pruning.
+ * a stale matching version and drop the winning one. That holds for columns that are only ever set
+ * once and never change: trigger-time identity columns (task_identifier, task_version, schedule_id,
+ * is_test, root_run_id, batch_id, friendly_id, queue, task_kind), append-only arrays under
+ * `hasAny`/`hasAll` (tags, bulk_action_group_ids), `region` (set once at dequeue, `''` -> value,
+ * never changes), and `error_fingerprint` (empty until a terminal error status, then fixed). Those
+ * go in PREWHERE to filter (and, for tags, use the skip index) before FINAL and before materialising
+ * the wide columns, which is what bounds memory on these scans. Columns that change as a run runs
+ * stay in WHERE (post-FINAL): `status` (lifecycle) and `machine_preset` (escalates on OOM retry).
+ * The `(organization_id, project_id, environment_id)` primary-key prefix and the `created_at` range
+ * also stay in WHERE so they keep driving primary-key and partition pruning.
  */
 function applyRunFiltersToQueryBuilder<T>(
   queryBuilder: ClickhouseQueryBuilder<T>,
@@ -447,24 +446,12 @@ function applyRunFiltersToQueryBuilder<T>(
     queryBuilder.where("status IN {statuses: Array(String)}", { statuses: options.statuses });
   }
 
-  if (options.regions && options.regions.length > 0) {
-    queryBuilder.where("if(region != '', region, worker_queue) IN {regions: Array(String)}", {
-      regions: options.regions,
-    });
-  }
-
   if (options.machines && options.machines.length > 0) {
     queryBuilder.where("machine_preset IN {machines: Array(String)}", {
       machines: options.machines,
     });
   }
 
-  if (options.errorId) {
-    queryBuilder.where("error_fingerprint = {errorFingerprint: String}", {
-      errorFingerprint: ErrorId.toId(options.errorId),
-    });
-  }
-
   // Period is a number of milliseconds duration
   if (options.period) {
     queryBuilder.where("created_at >= fromUnixTimestamp64Milli({period: Int64})", {
@@ -533,6 +520,16 @@ function applyRunFiltersToQueryBuilder<T>(
     queryBuilder.prewhere("queue IN {queues: Array(String)}", { queues: options.queues });
   }
 
+  if (options.regions && options.regions.length > 0) {
+    queryBuilder.prewhere("region IN {regions: Array(String)}", { regions: options.regions });
+  }
+
+  if (options.errorId) {
+    queryBuilder.prewhere("error_fingerprint = {errorFingerprint: String}", {
+      errorFingerprint: ErrorId.toId(options.errorId),
+    });
+  }
+
   if (options.taskKinds && options.taskKinds.length > 0) {
     const includesStandard = options.taskKinds.includes("STANDARD");
     // Include empty string when filtering for STANDARD (default value for pre-existing runs)

apps/webapp/test/runsListQueryShape.test.ts

@@ -76,25 +76,33 @@ describe("runs list query shape (PREWHERE routing under FINAL)", () => {
   );
 
   containerTest(
-    "region filter uses the post-FINAL effective region, not a pre-dequeue worker_queue version",
+    "region filter matches a dequeued run by its region; a still-queued run is not matched",
     async ({ clickhouseContainer, prisma }) => {
       const clickhouse = new ClickHouse({
         url: clickhouseContainer.getConnectionUrl(),
         name: "query-shape-region-test",
       });
 
       const ctx = await seedParents(prisma, "region");
-      const run = await createRun(prisma, ctx, { friendlyId: "run_region" });
+      const dequeued = await createRun(prisma, ctx, { friendlyId: "run_dequeued" });
+      const queued = await createRun(prisma, ctx, { friendlyId: "run_queued" });
 
       const shared = {
         taskIdentifier: "webhook.deliver",
-        workerQueue: "wq-legacy",
         createdAt: new Date(Date.now() - 1 * DAY_MS),
       };
 
       await insertTaskRunV2Rows(clickhouse, [
-        { ...run, ...shared, region: "", updatedAt: new Date(Date.now() - 2 * DAY_MS) },
-        { ...run, ...shared, region: "us-east-1", updatedAt: new Date(Date.now() - 1 * DAY_MS) },
+        { ...dequeued, ...shared, region: "", updatedAt: new Date(Date.now() - 2 * DAY_MS) },
+        {
+          ...dequeued,
+          ...shared,
+          region: "us-east-1",
+          updatedAt: new Date(Date.now() - 1 * DAY_MS),
+        },
+      ]);
+      await insertTaskRunV2Rows(clickhouse, [
+        { ...queued, ...shared, region: "", updatedAt: new Date(Date.now() - 1 * DAY_MS) },
       ]);
 
       const repository = new RunsRepository({ prisma, clickhouse });
@@ -107,10 +115,10 @@ describe("runs list query shape (PREWHERE routing under FINAL)", () => {
       };
 
       const byRegion = await repository.listRunIds({ ...listArgs, regions: ["us-east-1"] });
-      expect(byRegion.runIds).toEqual([run.id]);
+      expect(byRegion.runIds).toEqual([dequeued.id]);
 
-      const byWorkerQueue = await repository.listRunIds({ ...listArgs, regions: ["wq-legacy"] });
-      expect(byWorkerQueue.runIds).toEqual([]);
+      const otherRegion = await repository.listRunIds({ ...listArgs, regions: ["us-west-2"] });
+      expect(otherRegion.runIds).toEqual([]);
     }
   );
 });