perf(run-store,webapp): parallelize idempotency fan-out and cap lookup concurrency

Run the routing store's new and legacy idempotency lookups concurrently to
match the existing findRuns fan-out, and cap the per-batch idempotency
point-lookup concurrency at 5 to leave headroom in the connection pool.

Author: ericallam

apps/webapp/app/v3/services/batchTriggerV3.server.ts

@@ -34,7 +34,7 @@ import { OutOfEntitlementError, TriggerTaskService } from "./triggerTask.server"
 
 const PROCESSING_BATCH_SIZE = 50;
 const IDEMPOTENCY_KEY_LOOKUP_CHUNK_SIZE = 50;
-const IDEMPOTENCY_KEY_LOOKUP_CONCURRENCY = 10;
+const IDEMPOTENCY_KEY_LOOKUP_CONCURRENCY = 5;
 
 function chunkArray<T>(items: T[], size: number): T[][] {
   const chunks: T[][] = [];

internal-packages/run-store/src/runOpsStore.ts

@@ -468,14 +468,13 @@ export class RoutingRunStore implements RunStore {
     if (args.idempotencyKeys.length === 0) {
       return [];
     }
-    const newRows = await this.#new.findRunsByIdempotencyKeys(
-      args,
-      RoutingRunStore.#ownPrimary(this.#new, client)
-    );
-    const legacyRows = await this.#legacy.findRunsByIdempotencyKeys(
-      args,
-      RoutingRunStore.#ownPrimary(this.#legacy, client)
-    );
+    const [newRows, legacyRows] = await Promise.all([
+      this.#new.findRunsByIdempotencyKeys(args, RoutingRunStore.#ownPrimary(this.#new, client)),
+      this.#legacy.findRunsByIdempotencyKeys(
+        args,
+        RoutingRunStore.#ownPrimary(this.#legacy, client)
+      ),
+    ]);
     const byKey = new Map<string, IdempotencyKeyRunMatch>();
     for (const row of legacyRows) {
       if (row.idempotencyKey != null) byKey.set(row.idempotencyKey, row);