Skip to content

Commit a36741f

Browse files
TheodoreSpeaksTheodoreSpeaks
committed
improvement(api): make multipart transfers stateless
1 parent 6614484 commit a36741f

41 files changed

Lines changed: 749 additions & 20254 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/docs/openapi-v2-files-audit.json‎

Lines changed: 14 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,7 @@
319319
"post": {
320320
"operationId": "createFileUpload",
321321
"summary": "Create File Upload",
322-
"description": "Create a durable multipart upload session. Every file uses this flow; a small file is a single part. The maximum file size is 5 GB.",
322+
"description": "Create a stateless multipart upload session and signed upload token. Every file uses this flow; a small file is a single part. The maximum file size is 5 GB.",
323323
"tags": ["Files"],
324324
"requestBody": {
325325
"required": true,
@@ -339,31 +339,6 @@
339339
}
340340
},
341341
"/api/v2/files/uploads/{uploadId}": {
342-
"get": {
343-
"operationId": "getFileUpload",
344-
"summary": "Get File Upload",
345-
"description": "Read the durable state of a file upload session.",
346-
"tags": ["Files"],
347-
"parameters": [
348-
{
349-
"name": "uploadId",
350-
"in": "path",
351-
"required": true,
352-
"schema": { "type": "string" }
353-
},
354-
{ "$ref": "#/components/parameters/WorkspaceIdQuery" }
355-
],
356-
"responses": {
357-
"200": {
358-
"description": "The upload session.",
359-
"content": { "application/json": { "schema": {} } }
360-
},
361-
"401": { "$ref": "#/components/responses/Unauthorized" },
362-
"404": { "$ref": "#/components/responses/NotFound" },
363-
"429": { "$ref": "#/components/responses/RateLimited" },
364-
"500": { "$ref": "#/components/responses/InternalError" }
365-
}
366-
},
367342
"delete": {
368343
"operationId": "abortFileUpload",
369344
"summary": "Abort File Upload",
@@ -376,7 +351,8 @@
376351
"required": true,
377352
"schema": { "type": "string" }
378353
},
379-
{ "$ref": "#/components/parameters/WorkspaceIdQuery" }
354+
{ "$ref": "#/components/parameters/WorkspaceIdQuery" },
355+
{ "$ref": "#/components/parameters/UploadTokenHeader" }
380356
],
381357
"responses": {
382358
"200": {
@@ -404,7 +380,8 @@
404380
"required": true,
405381
"schema": { "type": "string" }
406382
},
407-
{ "$ref": "#/components/parameters/WorkspaceIdQuery" }
383+
{ "$ref": "#/components/parameters/WorkspaceIdQuery" },
384+
{ "$ref": "#/components/parameters/UploadTokenHeader" }
408385
],
409386
"requestBody": {
410387
"required": true,
@@ -437,7 +414,8 @@
437414
"required": true,
438415
"schema": { "type": "string" }
439416
},
440-
{ "$ref": "#/components/parameters/WorkspaceIdQuery" }
417+
{ "$ref": "#/components/parameters/WorkspaceIdQuery" },
418+
{ "$ref": "#/components/parameters/UploadTokenHeader" }
441419
],
442420
"requestBody": {
443421
"required": true,
@@ -1699,6 +1677,13 @@
16991677
"example": "a91c4b2e-6d3f-4e8a-b5c7-0d9e2f1a8c64"
17001678
}
17011679
},
1680+
"UploadTokenHeader": {
1681+
"name": "upload-token",
1682+
"in": "header",
1683+
"required": true,
1684+
"description": "The signed token returned when the multipart upload was created.",
1685+
"schema": { "type": "string", "minLength": 1 }
1686+
},
17021687
"FileIdPath": {
17031688
"name": "fileId",
17041689
"in": "path",

‎apps/docs/openapi-v2-tables.json‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3494,7 +3494,7 @@
34943494
"post": {
34953495
"operationId": "createTableImport",
34963496
"summary": "Create Table Import",
3497-
"description": "Create one durable import resource for either a new or existing table. Upload sources return multipart details; workspace-file sources start immediately.",
3497+
"description": "Create a table import. Upload sources return a stateless multipart token; workspace-file sources start immediately and both use table jobs for processing state.",
34983498
"tags": ["Tables"],
34993499
"requestBody": {
35003500
"required": true,
@@ -3519,7 +3519,7 @@
35193519
"get": {
35203520
"operationId": "getTableImport",
35213521
"summary": "Get Table Import",
3522-
"description": "Read upload, processing, progress, and terminal state using the same import id.",
3522+
"description": "Read processing progress and terminal state from the table job using the same import id.",
35233523
"tags": ["Tables"],
35243524
"parameters": [
35253525
{
@@ -3553,7 +3553,8 @@
35533553
"required": true,
35543554
"schema": { "type": "string" }
35553555
},
3556-
{ "$ref": "#/components/parameters/WorkspaceIdQuery" }
3556+
{ "$ref": "#/components/parameters/WorkspaceIdQuery" },
3557+
{ "$ref": "#/components/parameters/OptionalUploadTokenHeader" }
35573558
],
35583559
"responses": {
35593560
"200": {
@@ -3581,7 +3582,8 @@
35813582
"required": true,
35823583
"schema": { "type": "string" }
35833584
},
3584-
{ "$ref": "#/components/parameters/WorkspaceIdQuery" }
3585+
{ "$ref": "#/components/parameters/WorkspaceIdQuery" },
3586+
{ "$ref": "#/components/parameters/UploadTokenHeader" }
35853587
],
35863588
"requestBody": {
35873589
"required": true,
@@ -3614,7 +3616,8 @@
36143616
"required": true,
36153617
"schema": { "type": "string" }
36163618
},
3617-
{ "$ref": "#/components/parameters/WorkspaceIdQuery" }
3619+
{ "$ref": "#/components/parameters/WorkspaceIdQuery" },
3620+
{ "$ref": "#/components/parameters/UploadTokenHeader" }
36183621
],
36193622
"requestBody": {
36203623
"required": true,
@@ -3884,6 +3887,20 @@
38843887
},
38853888
"description": "The unique identifier of the workspace that owns the table."
38863889
},
3890+
"UploadTokenHeader": {
3891+
"name": "upload-token",
3892+
"in": "header",
3893+
"required": true,
3894+
"description": "The signed token returned for an upload-backed table import.",
3895+
"schema": { "type": "string", "minLength": 1 }
3896+
},
3897+
"OptionalUploadTokenHeader": {
3898+
"name": "upload-token",
3899+
"in": "header",
3900+
"required": false,
3901+
"description": "Required when canceling before upload completion; omitted when canceling a running table job.",
3902+
"schema": { "type": "string", "minLength": 1 }
3903+
},
38873904
"LimitQuery": {
38883905
"name": "limit",
38893906
"in": "query",

‎apps/sim/app/api/cron/cleanup-stale-executions/route.ts‎

Lines changed: 1 addition & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,5 @@
11
import { asyncJobs, db } from '@sim/db'
2-
import {
3-
tableImports,
4-
tableJobs,
5-
uploadSessions,
6-
workflowDeploymentOperation,
7-
workflowExecutionLogs,
8-
} from '@sim/db/schema'
2+
import { tableJobs, workflowDeploymentOperation, workflowExecutionLogs } from '@sim/db/schema'
93
import { createLogger } from '@sim/logger'
104
import { toError } from '@sim/utils/errors'
115
import { and, eq, exists, gt, inArray, lt, sql } from 'drizzle-orm'
@@ -16,7 +10,6 @@ import { JOB_RETENTION_HOURS, JOB_STATUS } from '@/lib/core/async-jobs'
1610
import { getMaxExecutionTimeout } from '@/lib/core/execution-limits'
1711
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
1812
import { deleteFile } from '@/lib/uploads/core/storage-service'
19-
import { expireUploadSessions } from '@/lib/uploads/multipart-session/service'
2013

2114
const logger = createLogger('CleanupStaleExecutions')
2215

@@ -135,8 +128,6 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
135128
// place (no rollback); the user retries. Also prune long-settled terminal jobs so the table
136129
// doesn't grow unbounded (the latest job per table is what list/detail reads surface).
137130
let staleTableJobsMarkedFailed = 0
138-
let stalePreparingImportsMarkedFailed = 0
139-
let expiredUploadSessions = 0
140131
try {
141132
const now = new Date()
142133
const staleJobs = await db
@@ -151,80 +142,10 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
151142
.returning({ id: tableJobs.id })
152143

153144
staleTableJobsMarkedFailed = staleJobs.length
154-
if (staleJobs.length > 0) {
155-
const now = new Date()
156-
await db
157-
.update(tableImports)
158-
.set({
159-
status: 'failed',
160-
error: `Import terminated: no progress for more than ${STALE_THRESHOLD_MINUTES} minutes`,
161-
completedAt: now,
162-
updatedAt: now,
163-
})
164-
.where(
165-
inArray(
166-
tableImports.id,
167-
staleJobs.map((job) => job.id)
168-
)
169-
)
170-
}
171145
if (staleTableJobsMarkedFailed > 0) {
172146
logger.info(`Marked ${staleTableJobsMarkedFailed} stale table jobs as failed`)
173147
}
174148

175-
const stalePreparingImports = await db
176-
.select({ id: tableImports.id })
177-
.from(tableImports)
178-
.where(
179-
and(eq(tableImports.status, 'preparing'), lt(tableImports.updatedAt, staleThreshold))
180-
)
181-
.orderBy(tableImports.updatedAt)
182-
.limit(100)
183-
if (stalePreparingImports.length > 0) {
184-
const failedImports = await db
185-
.update(tableImports)
186-
.set({
187-
status: 'failed',
188-
error: `Import terminated: preparation did not finish within ${STALE_THRESHOLD_MINUTES} minutes`,
189-
completedAt: now,
190-
updatedAt: now,
191-
})
192-
.where(
193-
and(
194-
eq(tableImports.status, 'preparing'),
195-
inArray(
196-
tableImports.id,
197-
stalePreparingImports.map((record) => record.id)
198-
)
199-
)
200-
)
201-
.returning({ uploadSessionId: tableImports.uploadSessionId })
202-
stalePreparingImportsMarkedFailed = failedImports.length
203-
204-
const uploadSessionIds = failedImports.flatMap((record) =>
205-
record.uploadSessionId ? [record.uploadSessionId] : []
206-
)
207-
if (uploadSessionIds.length > 0) {
208-
const uploads = await db
209-
.select({ storageKey: uploadSessions.storageKey })
210-
.from(uploadSessions)
211-
.where(inArray(uploadSessions.id, uploadSessionIds))
212-
for (const upload of uploads) {
213-
await deleteFile({ key: upload.storageKey, context: 'table-import' }).catch((error) => {
214-
logger.warn('Failed to delete source for a stale table import', {
215-
storageKey: upload.storageKey,
216-
error: toError(error).message,
217-
})
218-
})
219-
}
220-
}
221-
}
222-
if (stalePreparingImportsMarkedFailed > 0) {
223-
logger.info(
224-
`Marked ${stalePreparingImportsMarkedFailed} stale preparing table imports as failed`
225-
)
226-
}
227-
228149
const terminalRetention = new Date(Date.now() - TABLE_JOB_RETENTION_HOURS * 60 * 60 * 1000)
229150
const pruned = await db
230151
.delete(tableJobs)
@@ -255,14 +176,6 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
255176
})
256177
}
257178

258-
try {
259-
expiredUploadSessions = await expireUploadSessions(new Date(), 100)
260-
} catch (error) {
261-
logger.error('Failed to expire multipart upload sessions:', {
262-
error: toError(error).message,
263-
})
264-
}
265-
266179
// Clean up stale pending jobs (never started, e.g., due to server crash before startJob())
267180
let stalePendingJobsMarkedFailed = 0
268181

@@ -393,12 +306,6 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
393306
tableJobs: {
394307
staleMarkedFailed: staleTableJobsMarkedFailed,
395308
},
396-
tableImports: {
397-
stalePreparingMarkedFailed: stalePreparingImportsMarkedFailed,
398-
},
399-
uploadSessions: {
400-
expired: expiredUploadSessions,
401-
},
402309
deploymentOperations: {
403310
pruned: deploymentOperationsPruned,
404311
retentionDays: DEPLOYMENT_OPERATION_RETENTION_DAYS,

‎apps/sim/app/api/files/uploads/[uploadId]/complete/route.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,11 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Uploa
2828
const access = await requireWorkspaceWrite(user, workspaceId)
2929
if (access) return access
3030
try {
31-
const upload = await getOwnedUploadSession({
31+
const upload = getOwnedUploadSession({
3232
uploadId: parsed.data.params.uploadId,
3333
workspaceId,
3434
userId: user,
35+
uploadToken: parsed.data.headers['upload-token'],
3536
})
3637
const metadata = upload.metadata as { folderId?: string | null }
3738
const completed = await completeUploadSession({
@@ -49,7 +50,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Uploa
4950
return { value: registered.file.id, completedFileId: registered.file.id }
5051
},
5152
})
52-
const fileId = completed.value ?? completed.session.completedFileId
53+
const fileId = completed.value
5354
if (!fileId) throw new Error('Completed upload is missing its workspace file id')
5455
const file = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true })
5556
if (!file) throw new Error(`Completed workspace file ${fileId} not found`)

‎apps/sim/app/api/files/uploads/[uploadId]/parts/route.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Uploa
2525
const access = await requireWorkspaceWrite(user, workspaceId)
2626
if (access) return access
2727
try {
28-
const upload = await getOwnedUploadSession({
28+
const upload = getOwnedUploadSession({
2929
uploadId: parsed.data.params.uploadId,
3030
workspaceId,
3131
userId: user,
32+
uploadToken: parsed.data.headers['upload-token'],
3233
})
3334
const parts = await createUploadPartUrls({
3435
session: upload,

‎apps/sim/app/api/files/uploads/[uploadId]/route.ts‎

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,7 @@
11
import { type NextRequest, NextResponse } from 'next/server'
2-
import {
3-
abortWorkspaceFileUploadContract,
4-
getWorkspaceFileUploadContract,
5-
} from '@/lib/api/contracts/upload-sessions'
2+
import { abortWorkspaceFileUploadContract } from '@/lib/api/contracts/upload-sessions'
63
import { parseRequest } from '@/lib/api/server'
74
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
8-
import { getWorkspaceFile } from '@/lib/uploads/contexts/workspace'
95
import { abortUploadSession, getOwnedUploadSession } from '@/lib/uploads/multipart-session/service'
106
import {
117
requireUploadUser,
@@ -18,31 +14,6 @@ interface UploadRouteParams {
1814
params: Promise<{ uploadId: string }>
1915
}
2016

21-
export const GET = withRouteHandler(async (request: NextRequest, context: UploadRouteParams) => {
22-
const user = await requireUploadUser()
23-
if (user instanceof NextResponse) return user
24-
const parsed = await parseRequest(getWorkspaceFileUploadContract, request, context)
25-
if (!parsed.success) return parsed.response
26-
const { workspaceId } = parsed.data.query
27-
const access = await requireWorkspaceWrite(user, workspaceId)
28-
if (access) return access
29-
try {
30-
const upload = await getOwnedUploadSession({
31-
uploadId: parsed.data.params.uploadId,
32-
workspaceId,
33-
userId: user,
34-
})
35-
const file = upload.completedFileId
36-
? await getWorkspaceFile(workspaceId, upload.completedFileId, { throwOnError: true })
37-
: null
38-
return NextResponse.json({ data: toV2FileUpload(upload, file) })
39-
} catch (error) {
40-
const classified = uploadSessionErrorResponse(error)
41-
if (classified) return classified
42-
throw error
43-
}
44-
})
45-
4617
export const DELETE = withRouteHandler(async (request: NextRequest, context: UploadRouteParams) => {
4718
const user = await requireUploadUser()
4819
if (user instanceof NextResponse) return user
@@ -52,10 +23,11 @@ export const DELETE = withRouteHandler(async (request: NextRequest, context: Upl
5223
const access = await requireWorkspaceWrite(user, workspaceId)
5324
if (access) return access
5425
try {
55-
const upload = await getOwnedUploadSession({
26+
const upload = getOwnedUploadSession({
5627
uploadId: parsed.data.params.uploadId,
5728
workspaceId,
5829
userId: user,
30+
uploadToken: parsed.data.headers['upload-token'],
5931
})
6032
return NextResponse.json({ data: toV2FileUpload(await abortUploadSession(upload), null) })
6133
} catch (error) {

0 commit comments

Comments
 (0)