fix: stream upload to S3

src/http/routes/s3/commands/put-object.ts

@@ -123,7 +123,8 @@ export default function PutObject(s3Router: S3Router) {
             fileStream,
             uploadRequest.mimeType,
             uploadRequest.cacheControl,
-            ctx.signals.body
+            ctx.signals.body,
+            uploadRequest.contentLength
           )
 
           return {
@@ -177,6 +178,7 @@ export default function PutObject(s3Router: S3Router) {
               Key: key,
               CacheControl: uploadRequest.cacheControl,
               ContentType: uploadRequest.mimeType,
+              ContentLength: uploadRequest.contentLength,
               Expires: req.Headers?.['expires'] ? new Date(req.Headers?.['expires']) : undefined,
               ContentEncoding: req.Headers?.['content-encoding'],
               Metadata: metadata,

src/storage/backend/adapter.ts

@@ -97,7 +97,8 @@ export abstract class StorageBackendAdapter {
     body: NodeJS.ReadableStream,
     contentType: string,
     cacheControl: string,
-    signal?: AbortSignal
+    signal?: AbortSignal,
+    contentLength?: number
   ): Promise<ObjectMetadata> {
     throw new Error('uploadObject not implemented')
   }

src/storage/backend/file.ts

@@ -180,7 +180,9 @@ export class FileBackend implements StorageBackendAdapter {
     version: string | undefined,
     body: NodeJS.ReadableStream,
     contentType: string,
-    cacheControl: string
+    cacheControl: string,
+    signal?: AbortSignal,
+    contentLength?: number
   ): Promise<ObjectMetadata> {
     try {
       const file = this.resolveSecurePath(withOptionalVersion(`${bucketName}/${key}`, version))

src/storage/backend/s3/adapter.ts

@@ -11,6 +11,7 @@ import {
   HeadObjectCommand,
   ListObjectsV2Command,
   ListPartsCommand,
+  PutObjectCommand,
   S3Client,
   S3ClientConfig,
   UploadPartCommand,
@@ -41,6 +42,8 @@ const {
   storageS3DisableChecksum,
 } = getConfig()
 
+export const MAX_PUT_OBJECT_SIZE = 5 * 1024 * 1024 * 1024 // 5GB
+
 export interface S3ClientOptions {
   endpoint?: string
   region?: string
@@ -126,13 +129,15 @@ export class S3Backend implements StorageBackendAdapter {
 
   /**
    * Uploads and store an object
+   * Max 5GB
    * @param bucketName
    * @param key
    * @param version
    * @param body
    * @param contentType
    * @param cacheControl
    * @param signal
+   * @param contentLength
    */
   async uploadObject(
     bucketName: string,
@@ -141,12 +146,94 @@ export class S3Backend implements StorageBackendAdapter {
     body: Readable,
     contentType: string,
     cacheControl: string,
-    signal?: AbortSignal
+    signal?: AbortSignal,
+    contentLength?: number
   ): Promise<ObjectMetadata> {
     if (signal?.aborted) {
       throw ERRORS.Aborted('Upload was aborted')
     }
 
+    if (typeof contentLength !== 'number' || contentLength > MAX_PUT_OBJECT_SIZE) {
+      // Use multipart when the length is unknown or exceeds S3's 5GB single-request limit.
+      return this.bufferedMultipartUpload(
+        bucketName,
+        key,
+        version,
+        body,
+        contentType,
+        cacheControl,
+        signal
+      )
+    }
+
+    // Use PutObject directly when content-length is known and within S3's single-object limit (5GB).
+    // This avoids the buffering overhead of the Upload class which buffers each part in memory.
+    return this.putObject(
+      bucketName,
+      key,
+      version,
+      body,
+      contentType,
+      cacheControl,
+      signal,
+      contentLength
+    )
+  }
+
+  protected async putObject(
+    bucketName: string,
+    key: string,
+    version: string | undefined,
+    body: Readable,
+    contentType: string,
+    cacheControl: string,
+    signal: AbortSignal | undefined,
+    contentLength: number
+  ): Promise<ObjectMetadata> {
+    const dataStream = tracingFeatures?.upload ? monitorStream(body) : body
+
+    const command = new PutObjectCommand({
+      Bucket: bucketName,
+      Key: withOptionalVersion(key, version),
+      Body: dataStream,
+      ContentType: contentType,
+      CacheControl: cacheControl,
+      ContentLength: contentLength,
+    })
+
+    try {
+      const data = await this.client.send(command, {
+        abortSignal: signal,
+      })
+
+      return {
+        httpStatusCode: data.$metadata.httpStatusCode || 200,
+        cacheControl,
+        eTag: data.ETag || '',
+        mimetype: contentType,
+        contentLength,
+        // PutObject does not return LastModified; keep the fast path single-request and use the local completion time.
+        lastModified: new Date(),
+        size: contentLength,
+        contentRange: undefined,
+      }
+    } catch (err) {
+      if (err instanceof Error && err.name === 'AbortError') {
+        throw ERRORS.AbortedTerminate('Upload was aborted', err)
+      }
+      throw StorageBackendError.fromError(err)
+    }
+  }
+
+  protected async bufferedMultipartUpload(
+    bucketName: string,
+    key: string,
+    version: string | undefined,
+    body: Readable,
+    contentType: string,
+    cacheControl: string,
+    signal?: AbortSignal
+  ): Promise<ObjectMetadata> {
     const dataStream = tracingFeatures?.upload ? monitorStream(body) : body
 
     const upload = new Upload({
@@ -177,15 +264,13 @@ export class S3Backend implements StorageBackendAdapter {
     try {
       const data = await upload.done()
 
-      // Remove event listener to allow GC of upload and dataStream references
       upload.off('httpUploadProgress', progressHandler)
 
-      // Only call head for objects that are > 0 bytes
-      // for some reason headObject can take a really long time to resolve on zero byte uploads, this was causing requests to timeout
-      const metadata = hasUploadedBytes
+      const metadata: ObjectMetadata = hasUploadedBytes
         ? await this.headObject(bucketName, key, version)
         : {
             httpStatusCode: 200,
+            cacheControl,
             eTag: data.ETag || '',
             mimetype: contentType,
             lastModified: new Date(),

src/storage/protocols/s3/s3-handler.ts

@@ -694,6 +694,7 @@ export class S3ProtocolHandler {
         body: command.Body as Readable,
         cacheControl: command.CacheControl!,
         mimeType: command.ContentType!,
+        contentLength: command.ContentLength,
         isTruncated: options.isTruncated,
         userMetadata: command.Metadata,
       },

src/storage/uploader.ts

@@ -18,6 +18,7 @@ interface FileUpload {
   body: Readable
   mimeType: string
   cacheControl: string
+  contentLength?: number
   isTruncated: () => boolean
   xRobotsTag?: string
   userMetadata?: Record<string, unknown>
@@ -110,7 +111,8 @@ export class Uploader {
         file.body,
         file.mimeType,
         file.cacheControl,
-        request.signal
+        request.signal,
+        file.contentLength
       )
 
       if (request.file.xRobotsTag) {
@@ -335,6 +337,7 @@ export async function fileUploadFromRequest(
   let userMetadata: Record<string, unknown> | undefined
   let mimeType: string
   let isTruncated: () => boolean
+  let fileContentLength: number | undefined
   let maxFileSize = 0
 
   // When is an empty folder we restrict it to 0 bytes
@@ -357,6 +360,8 @@ export async function fileUploadFromRequest(
 
       const file = formData.file
       body = file
+      // multipart/form-data content-length includes boundary overhead and cannot be trusted as file size,
+      // so we intentionally leave fileContentLength undefined and let the backend stream via multipart upload.
       /* @ts-expect-error: https://github.com/aws/aws-sdk-js-v3/issues/2085 */
       const customMd = formData.fields.metadata?.value ?? formData.fields.userMetadata?.value
       /* @ts-expect-error: https://github.com/aws/aws-sdk-js-v3/issues/2085 */
@@ -407,11 +412,15 @@ export async function fileUploadFromRequest(
       userMetadata = parseUserMetadata(customMd)
     }
 
-    const contentLength = getKnownRequestContentLength(request)
-    isTruncated = () => {
-      // @todo more secure to get this from the stream or from s3 in the next step
-      return typeof contentLength === 'number' && contentLength > maxFileSize
+    fileContentLength = getKnownRequestContentLength(request)
+    if (typeof fileContentLength === 'number' && fileContentLength > maxFileSize) {
+      throw ERRORS.EntityTooLarge()
     }
+
+    // Known-size binary uploads are rejected before
+    // reaching the backend when they exceed the limit.
+    // Unknown-size binary uploads do not have a later truncation signal.
+    isTruncated = () => false
   }
 
   // Detect if the request stream closed before we could pass it to the storage backend
@@ -425,6 +434,7 @@ export async function fileUploadFromRequest(
     body,
     mimeType,
     cacheControl,
+    contentLength: fileContentLength,
     isTruncated,
     userMetadata,
     maxFileSize,

src/test/object.test.ts

@@ -995,7 +995,8 @@ describe('testing POST object via binary upload', () => {
         message: 'The object exceeded the maximum allowed size',
       })
     )
-    expect(S3Backend.prototype.uploadObject).toHaveBeenCalledTimes(1)
+    // Early size check in fileUploadFromRequest rejects before reaching the backend
+    expect(S3Backend.prototype.uploadObject).not.toHaveBeenCalled()
   })
 
   test('return 400 when uploading to object with no file name', async () => {