diff --git a/src/export/dump.test.ts b/src/export/dump.test.ts index ca65b43..3c960d5 100644 --- a/src/export/dump.test.ts +++ b/src/export/dump.test.ts @@ -49,6 +49,7 @@ describe('Database Dump Module', () => { { id: 1, name: 'Alice' }, { id: 2, name: 'Bob' }, ]) + .mockResolvedValueOnce([]) .mockResolvedValueOnce([ { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' }, ]) @@ -56,6 +57,7 @@ describe('Database Dump Module', () => { { id: 1, total: 99.99 }, { id: 2, total: 49.5 }, ]) + .mockResolvedValueOnce([]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) @@ -71,13 +73,13 @@ describe('Database Dump Module', () => { expect(dumpText).toContain( 'CREATE TABLE users (id INTEGER, name TEXT);' ) - expect(dumpText).toContain("INSERT INTO users VALUES (1, 'Alice');") - expect(dumpText).toContain("INSERT INTO users VALUES (2, 'Bob');") + expect(dumpText).toContain('INSERT INTO "users" VALUES (1, \'Alice\');') + expect(dumpText).toContain('INSERT INTO "users" VALUES (2, \'Bob\');') expect(dumpText).toContain( 'CREATE TABLE orders (id INTEGER, total REAL);' ) - expect(dumpText).toContain('INSERT INTO orders VALUES (1, 99.99);') - expect(dumpText).toContain('INSERT INTO orders VALUES (2, 49.5);') + expect(dumpText).toContain('INSERT INTO "orders" VALUES (1, 99.99);') + expect(dumpText).toContain('INSERT INTO "orders" VALUES (2, 49.5);') }) it('should handle empty databases (no tables)', async () => { @@ -118,14 +120,88 @@ describe('Database Dump Module', () => { { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' }, ]) .mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }]) + .mockResolvedValueOnce([]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) expect(response).toBeInstanceOf(Response) const dumpText = await response.text() expect(dumpText).toContain( - "INSERT INTO users VALUES (1, 'Alice''s adventure');" + "INSERT INTO \"users\" VALUES (1, 'Alice''s adventure');" + ) + }) + + it('should read table rows in bounded pages', async () => { + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'events' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE events (id INTEGER, payload TEXT);' }, + ]) + .mockResolvedValueOnce([{ id: 1, payload: 'first' }]) + .mockResolvedValueOnce([{ id: 2, payload: 'second' }]) + .mockResolvedValueOnce([]) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + const dumpText = await response.text() + + expect(dumpText).toContain( + 'INSERT INTO "events" VALUES (1, \'first\');' + ) + expect(dumpText).toContain( + 'INSERT INTO "events" VALUES (2, \'second\');' ) + expect(executeOperation).toHaveBeenCalledWith( + [ + { + sql: 'SELECT * FROM "events" LIMIT ? OFFSET ?;', + params: [500, 500], + }, + ], + mockDataSource, + mockConfig + ) + }) + + it('should request the next data page only after the prior chunk is consumed', async () => { + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'events' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE events (id INTEGER, payload TEXT);' }, + ]) + .mockResolvedValueOnce([{ id: 1, payload: 'first' }]) + .mockResolvedValueOnce([{ id: 2, payload: 'second' }]) + .mockResolvedValueOnce([]) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + const reader = response.body!.getReader() + const decoder = new TextDecoder() + + expect(executeOperation).toHaveBeenCalledTimes(1) + + expect(decoder.decode((await reader.read()).value)).toBe( + 'SQLite format 3\0' + ) + expect(decoder.decode((await reader.read()).value)).toContain( + 'CREATE TABLE events' + ) + + await Promise.resolve() + expect(executeOperation).not.toHaveBeenCalledWith( + [ + { + sql: 'SELECT * FROM "events" LIMIT ? OFFSET ?;', + params: [500, 500], + }, + ], + mockDataSource, + mockConfig + ) + + expect(decoder.decode((await reader.read()).value)).toContain('first') + + await reader.read() + await reader.read() + expect((await reader.read()).done).toBe(true) }) it('should return a 500 response when an error occurs', async () => { diff --git a/src/export/dump.ts b/src/export/dump.ts index 91a2e89..1ab5865 100644 --- a/src/export/dump.ts +++ b/src/export/dump.ts @@ -3,6 +3,45 @@ import { StarbaseDBConfiguration } from '../handler' import { DataSource } from '../types' import { createResponse } from '../utils' +const DEFAULT_DUMP_BATCH_SIZE = 500 + +function quoteIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"` +} + +function toSqlHex(bytes: Uint8Array): string { + return Array.from(bytes) + .map((byte) => byte.toString(16).padStart(2, '0')) + .join('') +} + +function formatSqlValue(value: unknown): string { + if (value === null || value === undefined) return 'NULL' + if (typeof value === 'number') { + return Number.isFinite(value) ? String(value) : 'NULL' + } + if (typeof value === 'bigint') return value.toString() + if (typeof value === 'boolean') return value ? '1' : '0' + if (value instanceof ArrayBuffer) { + return `X'${toSqlHex(new Uint8Array(value))}'` + } + if (ArrayBuffer.isView(value)) { + return `X'${toSqlHex( + new Uint8Array(value.buffer, value.byteOffset, value.byteLength) + )}'` + } + + return `'${String(value).replace(/'/g, "''")}'` +} + +function buildInsertStatement( + table: string, + row: Record +): string { + const values = Object.values(row).map(formatSqlValue) + return `INSERT INTO ${quoteIdentifier(table)} VALUES (${values.join(', ')});\n` +} + export async function dumpDatabaseRoute( dataSource: DataSource, config: StarbaseDBConfiguration @@ -16,54 +55,89 @@ export async function dumpDatabaseRoute( ) const tables = tablesResult.map((row: any) => row.name) - let dumpContent = 'SQLite format 3\0' // SQLite file header - - // Iterate through all tables - for (const table of tables) { - // Get table schema - const schemaResult = await executeOperation( - [ - { - sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, - }, - ], - dataSource, - config - ) - - if (schemaResult.length) { - const schema = schemaResult[0].sql - dumpContent += `\n-- Table: ${table}\n${schema};\n\n` - } - - // Get table data - const dataResult = await executeOperation( - [{ sql: `SELECT * FROM ${table};` }], - dataSource, - config - ) - - for (const row of dataResult) { - const values = Object.values(row).map((value) => - typeof value === 'string' - ? `'${value.replace(/'/g, "''")}'` - : value - ) - dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` - } - - dumpContent += '\n' - } - - // Create a Blob from the dump content - const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' }) + const encoder = new TextEncoder() + let headerSent = false + let tableIndex = 0 + let schemaSent = false + let offset = 0 + const stream = new ReadableStream({ + async pull(controller) { + try { + if (!headerSent) { + headerSent = true + controller.enqueue(encoder.encode('SQLite format 3\0')) + return + } + + while (tableIndex < tables.length) { + const table = tables[tableIndex] + + if (!schemaSent) { + schemaSent = true + const schemaResult = await executeOperation( + [ + { + sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name=?;`, + params: [table], + }, + ], + dataSource, + config + ) + + if (schemaResult.length) { + const schema = schemaResult[0].sql + controller.enqueue( + encoder.encode( + `\n-- Table: ${table}\n${schema};\n\n` + ) + ) + return + } + } + + const dataResult = await executeOperation( + [ + { + sql: `SELECT * FROM ${quoteIdentifier(table)} LIMIT ? OFFSET ?;`, + params: [DEFAULT_DUMP_BATCH_SIZE, offset], + }, + ], + dataSource, + config + ) + + if (dataResult.length) { + offset += DEFAULT_DUMP_BATCH_SIZE + const chunk = dataResult + .map((row: Record) => + buildInsertStatement(table, row) + ) + .join('') + controller.enqueue(encoder.encode(chunk)) + return + } + + tableIndex += 1 + schemaSent = false + offset = 0 + controller.enqueue(encoder.encode('\n')) + return + } + + controller.close() + } catch (error) { + controller.error(error) + } + }, + }) const headers = new Headers({ 'Content-Type': 'application/x-sqlite3', 'Content-Disposition': 'attachment; filename="database_dump.sql"', }) - return new Response(blob, { headers }) + return new Response(stream, { headers }) } catch (error: any) { console.error('Database Dump Error:', error) return createResponse(undefined, 'Failed to create database dump', 500)