fix(tables): stop row writes 500ing on a column outside the table schema (#6323)

createTableWriteProvenanceTargets (added in #6247) required every submitted
column to translate to exactly one storage id and threw otherwise. The wire
translator has always dropped keys naming no column in the schema, so any
internal-JWT write carrying such a key threw an uncaught error and surfaced
as a 500 — where the same write previously succeeded, since the write path
drops the column identically.

Give a dropped column a null column id instead of throwing. It still gets a
target, so the bundle completeness check that pairs one selection per
submitted column is unchanged, but no provenance is recorded for a value
that is never stored.

Author: waleedlatif1

apps/sim/app/api/table/row-secret-provenance.test.ts

@@ -0,0 +1,223 @@
+/**
+ * @vitest-environment node
+ */
+import { createMockRequest } from '@sim/testing'
+import { describe, expect, it } from 'vitest'
+import { AuthType } from '@/lib/auth/hybrid'
+import {
+  PRIVATE_SECRET_PROVENANCE_BUNDLE_V1,
+  PRIVATE_SECRET_PROVENANCE_FIELD,
+  PRIVATE_SECRET_PROVENANCE_HEADER,
+} from '@/lib/execution/private-tool-metadata'
+import { rowDataNameToId } from '@/lib/table/column-keys'
+import { tableRowSecretProvenanceSelectionKey } from '@/lib/table/secret-provenance-selection'
+import type { RowData } from '@/lib/table/types'
+import {
+  createTableWriteProvenanceTargets,
+  resolveTableWriteSecretProvenance,
+} from '@/app/api/table/row-secret-provenance'
+
+const USER_ID = 'user-1'
+const WORKSPACE_ID = 'ws-1'
+
+/** Mirrors the internal-JWT wire translator: names → ids, unknown names dropped. */
+const ID_BY_NAME = new Map([
+  ['email', 'col_email'],
+  ['company', 'col_company'],
+])
+
+const translateNames = (data: RowData): RowData => rowDataNameToId(data, ID_BY_NAME)
+const translateIdentity = (data: RowData): RowData => data
+
+function traceProvenance() {
+  return {
+    version: 1,
+    complete: true,
+    entries: [],
+    scope: { userId: USER_ID, workspaceId: WORKSPACE_ID },
+  }
+}
+
+function bundleRequest(selectionKeys: string[]) {
+  const payload = {
+    [PRIVATE_SECRET_PROVENANCE_FIELD]: {
+      version: 1,
+      complete: true,
+      selections: selectionKeys.map((key) => ({ key, provenance: traceProvenance() })),
+    },
+  }
+  const request = createMockRequest('POST', payload, {
+    [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1,
+  })
+  return { request, payload }
+}
+
+describe('createTableWriteProvenanceTargets', () => {
+  it('maps column names to their storage ids', () => {
+    const targets = createTableWriteProvenanceTargets([{ email: 'a@b.c' }], translateNames)
+
+    expect(targets).toEqual([
+      {
+        selectionKey: tableRowSecretProvenanceSelectionKey(0, 'email'),
+        rowKey: '0',
+        columnId: 'col_email',
+      },
+    ])
+  })
+
+  it('returns a null column id for a column the wire translator drops', () => {
+    const targets = createTableWriteProvenanceTargets(
+      [{ email: 'a@b.c', notAColumn: 'x' }],
+      translateNames
+    )
+
+    expect(targets).toHaveLength(2)
+    expect(targets[0].columnId).toBe('col_email')
+    expect(targets[1]).toEqual({
+      selectionKey: tableRowSecretProvenanceSelectionKey(0, 'notAColumn'),
+      rowKey: '0',
+      columnId: null,
+    })
+  })
+
+  it('keeps one target per submitted column so bundle selections stay paired', () => {
+    const targets = createTableWriteProvenanceTargets(
+      [{ notAColumn: 'x', alsoNotAColumn: 'y' }],
+      translateNames
+    )
+
+    expect(targets.map((target) => target.columnId)).toEqual([null, null])
+  })
+
+  it('passes column ids through for identity (session) translation', () => {
+    const targets = createTableWriteProvenanceTargets([{ col_email: 'a@b.c' }], translateIdentity)
+
+    expect(targets[0].columnId).toBe('col_email')
+  })
+
+  it('keys targets by row index across multiple rows', () => {
+    const targets = createTableWriteProvenanceTargets(
+      [{ email: 'a@b.c' }, { company: 'Acme' }],
+      translateNames
+    )
+
+    expect(targets.map((target) => target.rowKey)).toEqual(['0', '1'])
+    expect(targets[1].selectionKey).toBe(tableRowSecretProvenanceSelectionKey(1, 'company'))
+  })
+})
+
+describe('resolveTableWriteSecretProvenance', () => {
+  it('records no provenance for a dropped column on an unsupported session write', () => {
+    const rows = [{ email: 'a@b.c', notAColumn: 'x' }]
+    const result = resolveTableWriteSecretProvenance({
+      request: createMockRequest('POST', { rows }),
+      payload: { rows },
+      authType: AuthType.SESSION,
+      userId: USER_ID,
+      workspaceId: WORKSPACE_ID,
+      targets: createTableWriteProvenanceTargets(rows, translateNames),
+      rowKeys: ['0'],
+    })
+
+    expect(result.success).toBe(true)
+    if (!result.success) return
+    expect(Object.keys(result.provenanceByRowKey?.['0'].columns ?? {})).toEqual(['col_email'])
+  })
+
+  it('accepts a complete bundle that covers a dropped column', () => {
+    const rows = [{ email: 'a@b.c', notAColumn: 'x' }]
+    const { request, payload } = bundleRequest([
+      tableRowSecretProvenanceSelectionKey(0, 'email'),
+      tableRowSecretProvenanceSelectionKey(0, 'notAColumn'),
+    ])
+
+    const result = resolveTableWriteSecretProvenance({
+      request,
+      payload,
+      authType: AuthType.INTERNAL_JWT,
+      userId: USER_ID,
+      workspaceId: WORKSPACE_ID,
+      targets: createTableWriteProvenanceTargets(rows, translateNames),
+      rowKeys: ['0'],
+    })
+
+    expect(result.success).toBe(true)
+    if (!result.success) return
+    expect(Object.keys(result.provenanceByRowKey?.['0'].columns ?? {})).toEqual(['col_email'])
+  })
+
+  it('stores provenance for a fully translatable bundle', () => {
+    const rows = [{ email: 'a@b.c', company: 'Acme' }]
+    const { request, payload } = bundleRequest([
+      tableRowSecretProvenanceSelectionKey(0, 'email'),
+      tableRowSecretProvenanceSelectionKey(0, 'company'),
+    ])
+
+    const result = resolveTableWriteSecretProvenance({
+      request,
+      payload,
+      authType: AuthType.INTERNAL_JWT,
+      userId: USER_ID,
+      workspaceId: WORKSPACE_ID,
+      targets: createTableWriteProvenanceTargets(rows, translateNames),
+      rowKeys: ['0'],
+    })
+
+    expect(result.success).toBe(true)
+    if (!result.success) return
+    expect(Object.keys(result.provenanceByRowKey?.['0'].columns ?? {}).sort()).toEqual([
+      'col_company',
+      'col_email',
+    ])
+  })
+
+  it('rejects a bundle whose selection matches no submitted column', () => {
+    const rows = [{ email: 'a@b.c' }]
+    const { request, payload } = bundleRequest([tableRowSecretProvenanceSelectionKey(0, 'company')])
+
+    const result = resolveTableWriteSecretProvenance({
+      request,
+      payload,
+      authType: AuthType.INTERNAL_JWT,
+      userId: USER_ID,
+      workspaceId: WORKSPACE_ID,
+      targets: createTableWriteProvenanceTargets(rows, translateNames),
+      rowKeys: ['0'],
+    })
+
+    expect(result.success).toBe(false)
+  })
+
+  it('rejects a bundle whose selection scope does not match the caller', () => {
+    const rows = [{ email: 'a@b.c' }]
+    const payload = {
+      [PRIVATE_SECRET_PROVENANCE_FIELD]: {
+        version: 1,
+        complete: true,
+        selections: [
+          {
+            key: tableRowSecretProvenanceSelectionKey(0, 'email'),
+            provenance: {
+              ...traceProvenance(),
+              scope: { userId: 'someone-else', workspaceId: WORKSPACE_ID },
+            },
+          },
+        ],
+      },
+    }
+
+    const result = resolveTableWriteSecretProvenance({
+      request: createMockRequest('POST', payload, {
+        [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1,
+      }),
+      payload,
+      authType: AuthType.INTERNAL_JWT,
+      userId: USER_ID,
+      workspaceId: WORKSPACE_ID,
+      targets: createTableWriteProvenanceTargets(rows, translateNames),
+      rowKeys: ['0'],
+    })
+
+    expect(result.success).toBe(false)
+  })
+})

apps/sim/app/api/table/row-secret-provenance.ts

@@ -29,24 +29,30 @@ type TableWriteProvenanceResult =
 interface TableWriteProvenanceTarget {
   selectionKey: string
   rowKey: string
-  columnId: string
+  /** Storage column id, or `null` when the wire translator drops this column. */
+  columnId: string | null
 }
 
-/** Maps tool-facing column names to the stable storage ids used by the sidecar. */
+/**
+ * Maps tool-facing column names to the stable storage ids used by the sidecar.
+ *
+ * The wire translator drops keys that name no column in the table schema, and the
+ * write path drops them identically, so such a column is simply never persisted.
+ * It still gets a target — callers key one provenance selection per column they
+ * sent, and the completeness check pairs the two — but with a `null` column id so
+ * no provenance is recorded for a value that was never stored.
+ */
 export function createTableWriteProvenanceTargets(
   rows: readonly RowData[],
   translate: (data: RowData) => RowData
 ): TableWriteProvenanceTarget[] {
   return rows.flatMap((row, rowIndex) =>
     Object.entries(row).map(([columnKey, value]) => {
       const translatedKeys = Object.keys(translate({ [columnKey]: value }))
-      if (translatedKeys.length !== 1) {
-        throw new Error('Table row secret provenance column translation is invalid')
-      }
       return {
         selectionKey: tableRowSecretProvenanceSelectionKey(rowIndex, columnKey),
         rowKey: String(rowIndex),
-        columnId: translatedKeys[0],
+        columnId: translatedKeys.length === 1 ? translatedKeys[0] : null,
       }
     })
   )
@@ -81,6 +87,7 @@ export function resolveTableWriteSecretProvenance(options: {
       provenanceByRowKey[rowKey] = { complete: true, columns: {} }
     }
     for (const target of options.targets) {
+      if (target.columnId === null) continue
       const row = provenanceByRowKey[target.rowKey] ?? { complete: true, columns: {} }
       row.columns[target.columnId] = {
         version: 1,
@@ -132,11 +139,14 @@ export function resolveTableWriteSecretProvenance(options: {
     if (
       !target ||
       selection.provenance.scope?.userId !== options.userId ||
-      selection.provenance.scope?.workspaceId !== options.workspaceId ||
-      Object.hasOwn(provenanceByRowKey[target.rowKey].columns, target.columnId)
+      selection.provenance.scope?.workspaceId !== options.workspaceId
     ) {
       return { success: false, response: invalidProvenanceResponse() }
     }
+    if (target.columnId === null) continue
+    if (Object.hasOwn(provenanceByRowKey[target.rowKey].columns, target.columnId)) {
+      return { success: false, response: invalidProvenanceResponse() }
+    }
     provenanceByRowKey[target.rowKey].columns[target.columnId] = selection.provenance
   }
   return { success: true, provenanceByRowKey }