fix: stream SQL database dumps

src/export/dump.test.ts

@@ -49,13 +49,15 @@ describe('Database Dump Module', () => {
                 { id: 1, name: 'Alice' },
                 { id: 2, name: 'Bob' },
             ])
+            .mockResolvedValueOnce([])
             .mockResolvedValueOnce([
                 { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' },
             ])
             .mockResolvedValueOnce([
                 { id: 1, total: 99.99 },
                 { id: 2, total: 49.5 },
             ])
+            .mockResolvedValueOnce([])
 
         const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
 
@@ -71,13 +73,13 @@ describe('Database Dump Module', () => {
         expect(dumpText).toContain(
             'CREATE TABLE users (id INTEGER, name TEXT);'
         )
-        expect(dumpText).toContain("INSERT INTO users VALUES (1, 'Alice');")
-        expect(dumpText).toContain("INSERT INTO users VALUES (2, 'Bob');")
+        expect(dumpText).toContain('INSERT INTO "users" VALUES (1, \'Alice\');')
+        expect(dumpText).toContain('INSERT INTO "users" VALUES (2, \'Bob\');')
         expect(dumpText).toContain(
             'CREATE TABLE orders (id INTEGER, total REAL);'
         )
-        expect(dumpText).toContain('INSERT INTO orders VALUES (1, 99.99);')
-        expect(dumpText).toContain('INSERT INTO orders VALUES (2, 49.5);')
+        expect(dumpText).toContain('INSERT INTO "orders" VALUES (1, 99.99);')
+        expect(dumpText).toContain('INSERT INTO "orders" VALUES (2, 49.5);')
     })
 
     it('should handle empty databases (no tables)', async () => {
@@ -118,14 +120,88 @@ describe('Database Dump Module', () => {
                 { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' },
             ])
             .mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }])
+            .mockResolvedValueOnce([])
 
         const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
 
         expect(response).toBeInstanceOf(Response)
         const dumpText = await response.text()
         expect(dumpText).toContain(
-            "INSERT INTO users VALUES (1, 'Alice''s adventure');"
+            "INSERT INTO \"users\" VALUES (1, 'Alice''s adventure');"
+        )
+    })
+
+    it('should read table rows in bounded pages', async () => {
+        vi.mocked(executeOperation)
+            .mockResolvedValueOnce([{ name: 'events' }])
+            .mockResolvedValueOnce([
+                { sql: 'CREATE TABLE events (id INTEGER, payload TEXT);' },
+            ])
+            .mockResolvedValueOnce([{ id: 1, payload: 'first' }])
+            .mockResolvedValueOnce([{ id: 2, payload: 'second' }])
+            .mockResolvedValueOnce([])
+
+        const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
+        const dumpText = await response.text()
+
+        expect(dumpText).toContain(
+            'INSERT INTO "events" VALUES (1, \'first\');'
+        )
+        expect(dumpText).toContain(
+            'INSERT INTO "events" VALUES (2, \'second\');'
         )
+        expect(executeOperation).toHaveBeenCalledWith(
+            [
+                {
+                    sql: 'SELECT * FROM "events" LIMIT ? OFFSET ?;',
+                    params: [500, 500],
+                },
+            ],
+            mockDataSource,
+            mockConfig
+        )
+    })
+
+    it('should request the next data page only after the prior chunk is consumed', async () => {
+        vi.mocked(executeOperation)
+            .mockResolvedValueOnce([{ name: 'events' }])
+            .mockResolvedValueOnce([
+                { sql: 'CREATE TABLE events (id INTEGER, payload TEXT);' },
+            ])
+            .mockResolvedValueOnce([{ id: 1, payload: 'first' }])
+            .mockResolvedValueOnce([{ id: 2, payload: 'second' }])
+            .mockResolvedValueOnce([])
+
+        const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
+        const reader = response.body!.getReader()
+        const decoder = new TextDecoder()
+
+        expect(executeOperation).toHaveBeenCalledTimes(1)
+
+        expect(decoder.decode((await reader.read()).value)).toBe(
+            'SQLite format 3\0'
+        )
+        expect(decoder.decode((await reader.read()).value)).toContain(
+            'CREATE TABLE events'
+        )
+
+        await Promise.resolve()
+        expect(executeOperation).not.toHaveBeenCalledWith(
+            [
+                {
+                    sql: 'SELECT * FROM "events" LIMIT ? OFFSET ?;',
+                    params: [500, 500],
+                },
+            ],
+            mockDataSource,
+            mockConfig
+        )
+
+        expect(decoder.decode((await reader.read()).value)).toContain('first')
+
+        await reader.read()
+        await reader.read()
+        expect((await reader.read()).done).toBe(true)
     })
 
     it('should return a 500 response when an error occurs', async () => {

src/export/dump.ts

@@ -3,6 +3,45 @@ import { StarbaseDBConfiguration } from '../handler'
 import { DataSource } from '../types'
 import { createResponse } from '../utils'
 
+const DEFAULT_DUMP_BATCH_SIZE = 500
+
+function quoteIdentifier(identifier: string): string {
+    return `"${identifier.replace(/"/g, '""')}"`
+}
+
+function toSqlHex(bytes: Uint8Array): string {
+    return Array.from(bytes)
+        .map((byte) => byte.toString(16).padStart(2, '0'))
+        .join('')
+}
+
+function formatSqlValue(value: unknown): string {
+    if (value === null || value === undefined) return 'NULL'
+    if (typeof value === 'number') {
+        return Number.isFinite(value) ? String(value) : 'NULL'
+    }
+    if (typeof value === 'bigint') return value.toString()
+    if (typeof value === 'boolean') return value ? '1' : '0'
+    if (value instanceof ArrayBuffer) {
+        return `X'${toSqlHex(new Uint8Array(value))}'`
+    }
+    if (ArrayBuffer.isView(value)) {
+        return `X'${toSqlHex(
+            new Uint8Array(value.buffer, value.byteOffset, value.byteLength)
+        )}'`
+    }
+
+    return `'${String(value).replace(/'/g, "''")}'`
+}
+
+function buildInsertStatement(
+    table: string,
+    row: Record<string, unknown>
+): string {
+    const values = Object.values(row).map(formatSqlValue)
+    return `INSERT INTO ${quoteIdentifier(table)} VALUES (${values.join(', ')});\n`
+}
+
 export async function dumpDatabaseRoute(
     dataSource: DataSource,
     config: StarbaseDBConfiguration
@@ -16,54 +55,89 @@ export async function dumpDatabaseRoute(
         )
 
         const tables = tablesResult.map((row: any) => row.name)
-        let dumpContent = 'SQLite format 3\0' // SQLite file header
-
-        // Iterate through all tables
-        for (const table of tables) {
-            // Get table schema
-            const schemaResult = await executeOperation(
-                [
-                    {
-                        sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`,
-                    },
-                ],
-                dataSource,
-                config
-            )
-
-            if (schemaResult.length) {
-                const schema = schemaResult[0].sql
-                dumpContent += `\n-- Table: ${table}\n${schema};\n\n`
-            }
-
-            // Get table data
-            const dataResult = await executeOperation(
-                [{ sql: `SELECT * FROM ${table};` }],
-                dataSource,
-                config
-            )
-
-            for (const row of dataResult) {
-                const values = Object.values(row).map((value) =>
-                    typeof value === 'string'
-                        ? `'${value.replace(/'/g, "''")}'`
-                        : value
-                )
-                dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n`
-            }
-
-            dumpContent += '\n'
-        }
-
-        // Create a Blob from the dump content
-        const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' })
+        const encoder = new TextEncoder()
+        let headerSent = false
+        let tableIndex = 0
+        let schemaSent = false
+        let offset = 0
+        const stream = new ReadableStream({
+            async pull(controller) {
+                try {
+                    if (!headerSent) {
+                        headerSent = true
+                        controller.enqueue(encoder.encode('SQLite format 3\0'))
+                        return
+                    }
+
+                    while (tableIndex < tables.length) {
+                        const table = tables[tableIndex]
+
+                        if (!schemaSent) {
+                            schemaSent = true
+                            const schemaResult = await executeOperation(
+                                [
+                                    {
+                                        sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name=?;`,
+                                        params: [table],
+                                    },
+                                ],
+                                dataSource,
+                                config
+                            )
+
+                            if (schemaResult.length) {
+                                const schema = schemaResult[0].sql
+                                controller.enqueue(
+                                    encoder.encode(
+                                        `\n-- Table: ${table}\n${schema};\n\n`
+                                    )
+                                )
+                                return
+                            }
+                        }
+
+                        const dataResult = await executeOperation(
+                            [
+                                {
+                                    sql: `SELECT * FROM ${quoteIdentifier(table)} LIMIT ? OFFSET ?;`,
+                                    params: [DEFAULT_DUMP_BATCH_SIZE, offset],
+                                },
+                            ],
+                            dataSource,
+                            config
+                        )
+
+                        if (dataResult.length) {
+                            offset += DEFAULT_DUMP_BATCH_SIZE
+                            const chunk = dataResult
+                                .map((row: Record<string, unknown>) =>
+                                    buildInsertStatement(table, row)
+                                )
+                                .join('')
+                            controller.enqueue(encoder.encode(chunk))
+                            return
+                        }
+
+                        tableIndex += 1
+                        schemaSent = false
+                        offset = 0
+                        controller.enqueue(encoder.encode('\n'))
+                        return
+                    }
+
+                    controller.close()
+                } catch (error) {
+                    controller.error(error)
+                }
+            },
+        })
 
         const headers = new Headers({
             'Content-Type': 'application/x-sqlite3',
             'Content-Disposition': 'attachment; filename="database_dump.sql"',
         })
 
-        return new Response(blob, { headers })
+        return new Response(stream, { headers })
     } catch (error: any) {
         console.error('Database Dump Error:', error)
         return createResponse(undefined, 'Failed to create database dump', 500)