Skip to content

Commit 871b1e4

Browse files
committed
fix(data-drains): preserve row-distinguishing index when BigQuery insertId overflows
Truncating from the left dropped the index suffix, so any overflow would collapse all rows in a chunk to the same insertId and BigQuery would silently dedupe them. Path is unreachable today (UUIDs keep raw ~85 chars), but the overflow branch is now correct: hash the prefix, keep the index intact.
1 parent c35b5b9 commit 871b1e4

1 file changed

Lines changed: 9 additions & 2 deletions

File tree

apps/sim/lib/data-drains/destinations/bigquery.ts

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { createHash } from 'node:crypto'
12
import { createLogger } from '@sim/logger'
23
import { toError } from '@sim/utils/errors'
34
import { JWT } from 'google-auth-library'
@@ -133,12 +134,18 @@ async function postInsertAll(
133134
/**
134135
* Builds a stable `insertId` for best-effort dedup (~60s window). Prefixed
135136
* with `drainId` so (runId, sequence) collisions across drains do not
136-
* accidentally dedupe each other's rows.
137+
* accidentally dedupe each other's rows. With UUID drain/run IDs the raw
138+
* form fits well under 128 chars; if anything pushes it over (e.g. a future
139+
* non-UUID id), hash the prefix and keep the row-distinguishing `index`
140+
* suffix intact so BigQuery does not silently dedupe distinct rows.
137141
*/
138142
function buildInsertId(metadata: DeliveryMetadata, index: number): string {
139143
const raw = `${metadata.drainId}-${metadata.runId}-${metadata.sequence}-${index}`
140144
if (raw.length <= MAX_INSERT_ID_LENGTH) return raw
141-
return raw.slice(0, MAX_INSERT_ID_LENGTH)
145+
const prefixHash = createHash('sha1')
146+
.update(`${metadata.drainId}-${metadata.runId}-${metadata.sequence}`)
147+
.digest('hex')
148+
return `${prefixHash}-${index}`
142149
}
143150

144151
const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504])

0 commit comments

Comments
 (0)