From 22474ff21b880b8be3e173a4298329423cdad8e5 Mon Sep 17 00:00:00 2001 From: Treasure520520 <203431274+Treasure520520@users.noreply.github.com> Date: Wed, 13 May 2026 20:16:09 +0800 Subject: [PATCH 1/2] fix: stream SQL database dumps --- src/export/dump.test.ts | 44 ++++++++++-- src/export/dump.ts | 145 ++++++++++++++++++++++++++++------------ 2 files changed, 142 insertions(+), 47 deletions(-) diff --git a/src/export/dump.test.ts b/src/export/dump.test.ts index ca65b43..e517270 100644 --- a/src/export/dump.test.ts +++ b/src/export/dump.test.ts @@ -49,6 +49,7 @@ describe('Database Dump Module', () => { { id: 1, name: 'Alice' }, { id: 2, name: 'Bob' }, ]) + .mockResolvedValueOnce([]) .mockResolvedValueOnce([ { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' }, ]) @@ -56,6 +57,7 @@ describe('Database Dump Module', () => { { id: 1, total: 99.99 }, { id: 2, total: 49.5 }, ]) + .mockResolvedValueOnce([]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) @@ -71,13 +73,13 @@ describe('Database Dump Module', () => { expect(dumpText).toContain( 'CREATE TABLE users (id INTEGER, name TEXT);' ) - expect(dumpText).toContain("INSERT INTO users VALUES (1, 'Alice');") - expect(dumpText).toContain("INSERT INTO users VALUES (2, 'Bob');") + expect(dumpText).toContain('INSERT INTO "users" VALUES (1, \'Alice\');') + expect(dumpText).toContain('INSERT INTO "users" VALUES (2, \'Bob\');') expect(dumpText).toContain( 'CREATE TABLE orders (id INTEGER, total REAL);' ) - expect(dumpText).toContain('INSERT INTO orders VALUES (1, 99.99);') - expect(dumpText).toContain('INSERT INTO orders VALUES (2, 49.5);') + expect(dumpText).toContain('INSERT INTO "orders" VALUES (1, 99.99);') + expect(dumpText).toContain('INSERT INTO "orders" VALUES (2, 49.5);') }) it('should handle empty databases (no tables)', async () => { @@ -118,13 +120,45 @@ describe('Database Dump Module', () => { { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' }, ]) .mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }]) + .mockResolvedValueOnce([]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) expect(response).toBeInstanceOf(Response) const dumpText = await response.text() expect(dumpText).toContain( - "INSERT INTO users VALUES (1, 'Alice''s adventure');" + "INSERT INTO \"users\" VALUES (1, 'Alice''s adventure');" + ) + }) + + it('should read table rows in bounded pages', async () => { + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'events' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE events (id INTEGER, payload TEXT);' }, + ]) + .mockResolvedValueOnce([{ id: 1, payload: 'first' }]) + .mockResolvedValueOnce([{ id: 2, payload: 'second' }]) + .mockResolvedValueOnce([]) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + const dumpText = await response.text() + + expect(dumpText).toContain( + 'INSERT INTO "events" VALUES (1, \'first\');' + ) + expect(dumpText).toContain( + 'INSERT INTO "events" VALUES (2, \'second\');' + ) + expect(executeOperation).toHaveBeenCalledWith( + [ + { + sql: 'SELECT * FROM "events" LIMIT ? OFFSET ?;', + params: [500, 500], + }, + ], + mockDataSource, + mockConfig ) }) diff --git a/src/export/dump.ts b/src/export/dump.ts index 91a2e89..a3fd070 100644 --- a/src/export/dump.ts +++ b/src/export/dump.ts @@ -3,6 +3,45 @@ import { StarbaseDBConfiguration } from '../handler' import { DataSource } from '../types' import { createResponse } from '../utils' +const DEFAULT_DUMP_BATCH_SIZE = 500 + +function quoteIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"` +} + +function toSqlHex(bytes: Uint8Array): string { + return Array.from(bytes) + .map((byte) => byte.toString(16).padStart(2, '0')) + .join('') +} + +function formatSqlValue(value: unknown): string { + if (value === null || value === undefined) return 'NULL' + if (typeof value === 'number') { + return Number.isFinite(value) ? String(value) : 'NULL' + } + if (typeof value === 'bigint') return value.toString() + if (typeof value === 'boolean') return value ? '1' : '0' + if (value instanceof ArrayBuffer) { + return `X'${toSqlHex(new Uint8Array(value))}'` + } + if (ArrayBuffer.isView(value)) { + return `X'${toSqlHex( + new Uint8Array(value.buffer, value.byteOffset, value.byteLength) + )}'` + } + + return `'${String(value).replace(/'/g, "''")}'` +} + +function buildInsertStatement( + table: string, + row: Record +): string { + const values = Object.values(row).map(formatSqlValue) + return `INSERT INTO ${quoteIdentifier(table)} VALUES (${values.join(', ')});\n` +} + export async function dumpDatabaseRoute( dataSource: DataSource, config: StarbaseDBConfiguration @@ -16,54 +55,76 @@ export async function dumpDatabaseRoute( ) const tables = tablesResult.map((row: any) => row.name) - let dumpContent = 'SQLite format 3\0' // SQLite file header - - // Iterate through all tables - for (const table of tables) { - // Get table schema - const schemaResult = await executeOperation( - [ - { - sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, - }, - ], - dataSource, - config - ) - - if (schemaResult.length) { - const schema = schemaResult[0].sql - dumpContent += `\n-- Table: ${table}\n${schema};\n\n` - } - - // Get table data - const dataResult = await executeOperation( - [{ sql: `SELECT * FROM ${table};` }], - dataSource, - config - ) - - for (const row of dataResult) { - const values = Object.values(row).map((value) => - typeof value === 'string' - ? `'${value.replace(/'/g, "''")}'` - : value - ) - dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` - } - - dumpContent += '\n' - } - - // Create a Blob from the dump content - const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' }) + const encoder = new TextEncoder() + const stream = new ReadableStream({ + async start(controller) { + try { + controller.enqueue(encoder.encode('SQLite format 3\0')) + + for (const table of tables) { + const schemaResult = await executeOperation( + [ + { + sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name=?;`, + params: [table], + }, + ], + dataSource, + config + ) + + if (schemaResult.length) { + const schema = schemaResult[0].sql + controller.enqueue( + encoder.encode( + `\n-- Table: ${table}\n${schema};\n\n` + ) + ) + } + + let offset = 0 + while (true) { + const dataResult = await executeOperation( + [ + { + sql: `SELECT * FROM ${quoteIdentifier(table)} LIMIT ? OFFSET ?;`, + params: [ + DEFAULT_DUMP_BATCH_SIZE, + offset, + ], + }, + ], + dataSource, + config + ) + + if (!dataResult.length) break + + const chunk = dataResult + .map((row: Record) => + buildInsertStatement(table, row) + ) + .join('') + controller.enqueue(encoder.encode(chunk)) + offset += DEFAULT_DUMP_BATCH_SIZE + } + + controller.enqueue(encoder.encode('\n')) + } + + controller.close() + } catch (error) { + controller.error(error) + } + }, + }) const headers = new Headers({ 'Content-Type': 'application/x-sqlite3', 'Content-Disposition': 'attachment; filename="database_dump.sql"', }) - return new Response(blob, { headers }) + return new Response(stream, { headers }) } catch (error: any) { console.error('Database Dump Error:', error) return createResponse(undefined, 'Failed to create database dump', 500) From 5fb75a94096e8568572b84ea9fda230cb92b26b4 Mon Sep 17 00:00:00 2001 From: Treasure520520 <203431274+Treasure520520@users.noreply.github.com> Date: Thu, 14 May 2026 14:10:15 +0800 Subject: [PATCH 2/2] fix: make SQL dump stream pull driven --- src/export/dump.test.ts | 42 ++++++++++++++++++++++ src/export/dump.ts | 77 ++++++++++++++++++++++++----------------- 2 files changed, 87 insertions(+), 32 deletions(-) diff --git a/src/export/dump.test.ts b/src/export/dump.test.ts index e517270..3c960d5 100644 --- a/src/export/dump.test.ts +++ b/src/export/dump.test.ts @@ -162,6 +162,48 @@ describe('Database Dump Module', () => { ) }) + it('should request the next data page only after the prior chunk is consumed', async () => { + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'events' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE events (id INTEGER, payload TEXT);' }, + ]) + .mockResolvedValueOnce([{ id: 1, payload: 'first' }]) + .mockResolvedValueOnce([{ id: 2, payload: 'second' }]) + .mockResolvedValueOnce([]) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + const reader = response.body!.getReader() + const decoder = new TextDecoder() + + expect(executeOperation).toHaveBeenCalledTimes(1) + + expect(decoder.decode((await reader.read()).value)).toBe( + 'SQLite format 3\0' + ) + expect(decoder.decode((await reader.read()).value)).toContain( + 'CREATE TABLE events' + ) + + await Promise.resolve() + expect(executeOperation).not.toHaveBeenCalledWith( + [ + { + sql: 'SELECT * FROM "events" LIMIT ? OFFSET ?;', + params: [500, 500], + }, + ], + mockDataSource, + mockConfig + ) + + expect(decoder.decode((await reader.read()).value)).toContain('first') + + await reader.read() + await reader.read() + expect((await reader.read()).done).toBe(true) + }) + it('should return a 500 response when an error occurs', async () => { const consoleErrorMock = vi .spyOn(console, 'error') diff --git a/src/export/dump.ts b/src/export/dump.ts index a3fd070..1ab5865 100644 --- a/src/export/dump.ts +++ b/src/export/dump.ts @@ -56,60 +56,73 @@ export async function dumpDatabaseRoute( const tables = tablesResult.map((row: any) => row.name) const encoder = new TextEncoder() + let headerSent = false + let tableIndex = 0 + let schemaSent = false + let offset = 0 const stream = new ReadableStream({ - async start(controller) { + async pull(controller) { try { - controller.enqueue(encoder.encode('SQLite format 3\0')) - - for (const table of tables) { - const schemaResult = await executeOperation( - [ - { - sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name=?;`, - params: [table], - }, - ], - dataSource, - config - ) + if (!headerSent) { + headerSent = true + controller.enqueue(encoder.encode('SQLite format 3\0')) + return + } - if (schemaResult.length) { - const schema = schemaResult[0].sql - controller.enqueue( - encoder.encode( - `\n-- Table: ${table}\n${schema};\n\n` - ) - ) - } + while (tableIndex < tables.length) { + const table = tables[tableIndex] - let offset = 0 - while (true) { - const dataResult = await executeOperation( + if (!schemaSent) { + schemaSent = true + const schemaResult = await executeOperation( [ { - sql: `SELECT * FROM ${quoteIdentifier(table)} LIMIT ? OFFSET ?;`, - params: [ - DEFAULT_DUMP_BATCH_SIZE, - offset, - ], + sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name=?;`, + params: [table], }, ], dataSource, config ) - if (!dataResult.length) break + if (schemaResult.length) { + const schema = schemaResult[0].sql + controller.enqueue( + encoder.encode( + `\n-- Table: ${table}\n${schema};\n\n` + ) + ) + return + } + } + + const dataResult = await executeOperation( + [ + { + sql: `SELECT * FROM ${quoteIdentifier(table)} LIMIT ? OFFSET ?;`, + params: [DEFAULT_DUMP_BATCH_SIZE, offset], + }, + ], + dataSource, + config + ) + if (dataResult.length) { + offset += DEFAULT_DUMP_BATCH_SIZE const chunk = dataResult .map((row: Record) => buildInsertStatement(table, row) ) .join('') controller.enqueue(encoder.encode(chunk)) - offset += DEFAULT_DUMP_BATCH_SIZE + return } + tableIndex += 1 + schemaSent = false + offset = 0 controller.enqueue(encoder.encode('\n')) + return } controller.close()