From 6c5161fe782a9e8e343b06fdfbd4eb0c2a701f55 Mon Sep 17 00:00:00 2001 From: thoa100m <207382254+thoa100m@users.noreply.github.com> Date: Wed, 13 May 2026 09:34:51 +0700 Subject: [PATCH] Stream export responses in pages --- src/export/csv.test.ts | 106 +++++++++++++++--------------- src/export/csv.ts | 103 +++++++++++++++++++---------- src/export/dump.test.ts | 67 +++++++++++++++---- src/export/dump.ts | 142 ++++++++++++++++++++++++++++------------ src/export/index.ts | 61 ++++++++++++----- src/export/json.test.ts | 93 +++++++++++++------------- src/export/json.ts | 57 ++++++++++++---- 7 files changed, 417 insertions(+), 212 deletions(-) diff --git a/src/export/csv.test.ts b/src/export/csv.test.ts index b186aeb..9e27a9c 100644 --- a/src/export/csv.test.ts +++ b/src/export/csv.test.ts @@ -1,13 +1,14 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { exportTableToCsvRoute } from './csv' -import { getTableData, createExportResponse } from './index' +import { getTableDataPage, tableExists } from './index' import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' vi.mock('./index', () => ({ - getTableData: vi.fn(), - createExportResponse: vi.fn(), + EXPORT_PAGE_SIZE: 500, + getTableDataPage: vi.fn(), + tableExists: vi.fn(), })) vi.mock('../utils', () => ({ @@ -43,38 +44,29 @@ beforeEach(() => { describe('CSV Export Module', () => { it('should return a CSV file when table data exists', async () => { - vi.mocked(getTableData).mockResolvedValue([ + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataPage).mockResolvedValueOnce([ { id: 1, name: 'Alice', age: 30 }, { id: 2, name: 'Bob', age: 25 }, ]) - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) - const response = await exportTableToCsvRoute( 'users', mockDataSource, mockConfig ) - expect(getTableData).toHaveBeenCalledWith( - 'users', - mockDataSource, - mockConfig + expect(response.headers.get('Content-Type')).toBe('text/csv') + expect(response.headers.get('Content-Disposition')).toBe( + 'attachment; filename="users_export.csv"' ) - expect(createExportResponse).toHaveBeenCalledWith( - 'id,name,age\n1,Alice,30\n2,Bob,25\n', - 'users_export.csv', - 'text/csv' + await expect(response.text()).resolves.toBe( + 'id,name,age\n1,Alice,30\n2,Bob,25\n' ) - expect(response.headers.get('Content-Type')).toBe('text/csv') }) it('should return 404 if table does not exist', async () => { - vi.mocked(getTableData).mockResolvedValue(null) + vi.mocked(tableExists).mockResolvedValue(false) const response = await exportTableToCsvRoute( 'non_existent_table', @@ -82,11 +74,6 @@ describe('CSV Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( - 'non_existent_table', - mockDataSource, - mockConfig - ) expect(response.status).toBe(404) const jsonResponse: { error: string } = await response.json() @@ -96,13 +83,8 @@ describe('CSV Export Module', () => { }) it('should handle empty table (return only headers)', async () => { - vi.mocked(getTableData).mockResolvedValue([]) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataPage).mockResolvedValueOnce([]) const response = await exportTableToCsvRoute( 'empty_table', @@ -110,49 +92,65 @@ describe('CSV Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( - 'empty_table', - mockDataSource, - mockConfig - ) - expect(createExportResponse).toHaveBeenCalledWith( - '', - 'empty_table_export.csv', - 'text/csv' - ) expect(response.headers.get('Content-Type')).toBe('text/csv') + await expect(response.text()).resolves.toBe('') }) it('should escape commas and quotes in CSV values', async () => { - vi.mocked(getTableData).mockResolvedValue([ + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataPage).mockResolvedValueOnce([ { id: 1, name: 'Sahithi, is', bio: 'my forever "penguin"' }, ]) - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) + const response = await exportTableToCsvRoute( + 'special_chars', + mockDataSource, + mockConfig ) + expect(response.headers.get('Content-Type')).toBe('text/csv') + await expect(response.text()).resolves.toBe( + 'id,name,bio\n1,"Sahithi, is","my forever ""penguin"""\n' + ) + }) + + it('should stream table data in pages instead of loading a full table at once', async () => { + const firstPage = Array.from({ length: 500 }, (_, index) => ({ + id: index + 1, + })) + const secondPage = [{ id: 501 }] + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataPage) + .mockResolvedValueOnce(firstPage) + .mockResolvedValueOnce(secondPage) + const response = await exportTableToCsvRoute( - 'special_chars', + 'users', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - 'id,name,bio\n1,"Sahithi, is","my forever ""penguin"""\n', - 'special_chars_export.csv', - 'text/csv' + const csv = await response.text() + expect(csv).toContain('501') + expect(getTableDataPage).toHaveBeenCalledWith( + 'users', + 0, + mockDataSource, + mockConfig + ) + expect(getTableDataPage).toHaveBeenCalledWith( + 'users', + 500, + mockDataSource, + mockConfig ) - expect(response.headers.get('Content-Type')).toBe('text/csv') }) it('should return 500 on an unexpected error', async () => { const consoleErrorMock = vi .spyOn(console, 'error') .mockImplementation(() => {}) - vi.mocked(getTableData).mockRejectedValue(new Error('Database Error')) + vi.mocked(tableExists).mockRejectedValue(new Error('Database Error')) const response = await exportTableToCsvRoute( 'users', diff --git a/src/export/csv.ts b/src/export/csv.ts index 22a4591..a87c612 100644 --- a/src/export/csv.ts +++ b/src/export/csv.ts @@ -1,17 +1,32 @@ -import { getTableData, createExportResponse } from './index' +import { EXPORT_PAGE_SIZE, getTableDataPage, tableExists } from './index' import { createResponse } from '../utils' import { DataSource } from '../types' import { StarbaseDBConfiguration } from '../handler' +function serializeCsvValue(value: unknown): string { + if (value === null || value === undefined) { + return '' + } + + const stringValue = String(value) + if ( + stringValue.includes(',') || + stringValue.includes('"') || + stringValue.includes('\n') + ) { + return `"${stringValue.replace(/"/g, '""')}"` + } + + return stringValue +} + export async function exportTableToCsvRoute( tableName: string, dataSource: DataSource, config: StarbaseDBConfiguration ): Promise { try { - const data = await getTableData(tableName, dataSource, config) - - if (data === null) { + if (!(await tableExists(tableName, dataSource, config))) { return createResponse( undefined, `Table '${tableName}' does not exist.`, @@ -19,36 +34,58 @@ export async function exportTableToCsvRoute( ) } - // Convert the result to CSV - let csvContent = '' - if (data.length > 0) { - // Add headers - csvContent += Object.keys(data[0]).join(',') + '\n' - - // Add data rows - data.forEach((row: any) => { - csvContent += - Object.values(row) - .map((value) => { - if ( - typeof value === 'string' && - (value.includes(',') || - value.includes('"') || - value.includes('\n')) - ) { - return `"${value.replace(/"/g, '""')}"` - } - return value - }) - .join(',') + '\n' - }) - } + const encoder = new TextEncoder() + const stream = new ReadableStream({ + async start(controller) { + let offset = 0 + let wroteHeader = false + let hasMoreRows = true + + while (hasMoreRows) { + const dataResult = await getTableDataPage( + tableName, + offset, + dataSource, + config + ) + + if (!wroteHeader && dataResult.length > 0) { + controller.enqueue( + encoder.encode( + Object.keys(dataResult[0]).join(',') + '\n' + ) + ) + wroteHeader = true + } + + for (const row of dataResult) { + controller.enqueue( + encoder.encode( + Object.values(row) + .map(serializeCsvValue) + .join(',') + '\n' + ) + ) + } + + hasMoreRows = dataResult.length === EXPORT_PAGE_SIZE + offset += EXPORT_PAGE_SIZE + + if (hasMoreRows) { + await new Promise((resolve) => setTimeout(resolve, 0)) + } + } + + controller.close() + }, + }) - return createExportResponse( - csvContent, - `${tableName}_export.csv`, - 'text/csv' - ) + return new Response(stream, { + headers: { + 'Content-Type': 'text/csv', + 'Content-Disposition': `attachment; filename="${tableName}_export.csv"`, + }, + }) } catch (error: any) { console.error('CSV Export Error:', error) return createResponse(undefined, 'Failed to export table to CSV', 500) diff --git a/src/export/dump.test.ts b/src/export/dump.test.ts index ca65b43..35833af 100644 --- a/src/export/dump.test.ts +++ b/src/export/dump.test.ts @@ -1,12 +1,16 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { dumpDatabaseRoute } from './dump' -import { executeOperation } from '.' +import { executeOperation, getTableDataPage } from '.' import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' vi.mock('.', () => ({ + EXPORT_PAGE_SIZE: 500, executeOperation: vi.fn(), + getTableDataPage: vi.fn(), + quoteIdentifier: (identifier: string) => + `"${identifier.replace(/"/g, '""')}"`, })) vi.mock('../utils', () => ({ @@ -46,11 +50,13 @@ describe('Database Dump Module', () => { { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, ]) .mockResolvedValueOnce([ - { id: 1, name: 'Alice' }, - { id: 2, name: 'Bob' }, + { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' }, ]) + + vi.mocked(getTableDataPage) .mockResolvedValueOnce([ - { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' }, + { id: 1, name: 'Alice' }, + { id: 2, name: 'Bob' }, ]) .mockResolvedValueOnce([ { id: 1, total: 99.99 }, @@ -71,13 +77,13 @@ describe('Database Dump Module', () => { expect(dumpText).toContain( 'CREATE TABLE users (id INTEGER, name TEXT);' ) - expect(dumpText).toContain("INSERT INTO users VALUES (1, 'Alice');") - expect(dumpText).toContain("INSERT INTO users VALUES (2, 'Bob');") + expect(dumpText).toContain('INSERT INTO "users" VALUES (1, \'Alice\');') + expect(dumpText).toContain('INSERT INTO "users" VALUES (2, \'Bob\');') expect(dumpText).toContain( 'CREATE TABLE orders (id INTEGER, total REAL);' ) - expect(dumpText).toContain('INSERT INTO orders VALUES (1, 99.99);') - expect(dumpText).toContain('INSERT INTO orders VALUES (2, 49.5);') + expect(dumpText).toContain('INSERT INTO "orders" VALUES (1, 99.99);') + expect(dumpText).toContain('INSERT INTO "orders" VALUES (2, 49.5);') }) it('should handle empty databases (no tables)', async () => { @@ -99,7 +105,7 @@ describe('Database Dump Module', () => { .mockResolvedValueOnce([ { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, ]) - .mockResolvedValueOnce([]) + vi.mocked(getTableDataPage).mockResolvedValueOnce([]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) @@ -117,14 +123,53 @@ describe('Database Dump Module', () => { .mockResolvedValueOnce([ { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' }, ]) - .mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }]) + vi.mocked(getTableDataPage).mockResolvedValueOnce([ + { id: 1, bio: "Alice's adventure" }, + ]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) expect(response).toBeInstanceOf(Response) const dumpText = await response.text() expect(dumpText).toContain( - "INSERT INTO users VALUES (1, 'Alice''s adventure');" + "INSERT INTO \"users\" VALUES (1, 'Alice''s adventure');" + ) + }) + + it('should stream table data in pages instead of loading a full table at once', async () => { + const firstPage = Array.from({ length: 500 }, (_, index) => ({ + id: index + 1, + name: `User ${index + 1}`, + })) + const secondPage = [{ id: 501, name: 'User 501' }] + + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, + ]) + + vi.mocked(getTableDataPage) + .mockResolvedValueOnce(firstPage) + .mockResolvedValueOnce(secondPage) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + const dumpText = await response.text() + + expect(getTableDataPage).toHaveBeenCalledWith( + 'users', + 0, + mockDataSource, + mockConfig + ) + expect(getTableDataPage).toHaveBeenCalledWith( + 'users', + 500, + mockDataSource, + mockConfig + ) + expect(dumpText).toContain( + 'INSERT INTO "users" VALUES (501, \'User 501\');' ) }) diff --git a/src/export/dump.ts b/src/export/dump.ts index 91a2e89..a008764 100644 --- a/src/export/dump.ts +++ b/src/export/dump.ts @@ -1,8 +1,35 @@ -import { executeOperation } from '.' +import { + executeOperation, + EXPORT_PAGE_SIZE, + getTableDataPage, + quoteIdentifier, +} from '.' import { StarbaseDBConfiguration } from '../handler' import { DataSource } from '../types' import { createResponse } from '../utils' +function serializeSqlValue(value: unknown): string { + if (value === null || value === undefined) { + return 'NULL' + } + + if (typeof value === 'string') { + return `'${value.replace(/'/g, "''")}'` + } + + if (typeof value === 'boolean') { + return value ? '1' : '0' + } + + if (value instanceof Uint8Array) { + return `X'${Array.from(value) + .map((byte) => byte.toString(16).padStart(2, '0')) + .join('')}'` + } + + return String(value) +} + export async function dumpDatabaseRoute( dataSource: DataSource, config: StarbaseDBConfiguration @@ -16,54 +43,83 @@ export async function dumpDatabaseRoute( ) const tables = tablesResult.map((row: any) => row.name) - let dumpContent = 'SQLite format 3\0' // SQLite file header - - // Iterate through all tables - for (const table of tables) { - // Get table schema - const schemaResult = await executeOperation( - [ - { - sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, - }, - ], - dataSource, - config - ) - - if (schemaResult.length) { - const schema = schemaResult[0].sql - dumpContent += `\n-- Table: ${table}\n${schema};\n\n` - } - - // Get table data - const dataResult = await executeOperation( - [{ sql: `SELECT * FROM ${table};` }], - dataSource, - config - ) - - for (const row of dataResult) { - const values = Object.values(row).map((value) => - typeof value === 'string' - ? `'${value.replace(/'/g, "''")}'` - : value - ) - dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` - } - - dumpContent += '\n' - } - - // Create a Blob from the dump content - const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' }) + const encoder = new TextEncoder() + + const stream = new ReadableStream({ + async start(controller) { + controller.enqueue(encoder.encode('SQLite format 3\0')) + + for (const table of tables) { + const quotedTable = quoteIdentifier(table) + + const schemaResult = await executeOperation( + [ + { + sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name=?;`, + params: [table], + }, + ], + dataSource, + config + ) + + if (schemaResult.length) { + const schema = schemaResult[0].sql + controller.enqueue( + encoder.encode( + `\n-- Table: ${table}\n${schema};\n\n` + ) + ) + } + + let offset = 0 + let hasMoreRows = true + + while (hasMoreRows) { + const dataResult = await getTableDataPage( + table, + offset, + dataSource, + config + ) + + for (const row of dataResult) { + const values = Object.values(row).map((value) => + serializeSqlValue(value) + ) + controller.enqueue( + encoder.encode( + `INSERT INTO ${quotedTable} VALUES (${values.join(', ')});\n` + ) + ) + } + + hasMoreRows = dataResult.length === EXPORT_PAGE_SIZE + offset += EXPORT_PAGE_SIZE + + if (hasMoreRows) { + await new Promise((resolve) => + setTimeout(resolve, 0) + ) + } + } + + controller.enqueue(encoder.encode('\n')) + } + + controller.close() + }, + cancel() { + // Client disconnected; stop producing dump chunks. + }, + }) const headers = new Headers({ 'Content-Type': 'application/x-sqlite3', 'Content-Disposition': 'attachment; filename="database_dump.sql"', }) - return new Response(blob, { headers }) + return new Response(stream, { headers }) } catch (error: any) { console.error('Database Dump Error:', error) return createResponse(undefined, 'Failed to create database dump', 500) diff --git a/src/export/index.ts b/src/export/index.ts index 9c40119..fdc9807 100644 --- a/src/export/index.ts +++ b/src/export/index.ts @@ -2,6 +2,12 @@ import { DataSource } from '../types' import { executeTransaction } from '../operation' import { StarbaseDBConfiguration } from '../handler' +export const EXPORT_PAGE_SIZE = 500 + +export function quoteIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"` +} + export async function executeOperation( queries: { sql: string; params?: any[] }[], dataSource: DataSource, @@ -19,6 +25,43 @@ export async function executeOperation( : results } +export async function tableExists( + tableName: string, + dataSource: DataSource, + config: StarbaseDBConfiguration +): Promise { + const tableExistsResult = await executeOperation( + [ + { + sql: `SELECT name FROM sqlite_master WHERE type='table' AND name=?;`, + params: [tableName], + }, + ], + dataSource, + config + ) + + return tableExistsResult.length > 0 +} + +export async function getTableDataPage( + tableName: string, + offset: number, + dataSource: DataSource, + config: StarbaseDBConfiguration, + limit = EXPORT_PAGE_SIZE +): Promise { + return executeOperation( + [ + { + sql: `SELECT * FROM ${quoteIdentifier(tableName)} LIMIT ${limit} OFFSET ${offset};`, + }, + ], + dataSource, + config + ) +} + export async function getTableData( tableName: string, dataSource: DataSource, @@ -26,28 +69,16 @@ export async function getTableData( ): Promise { try { // Verify if the table exists - const tableExistsResult = await executeOperation( - [ - { - sql: `SELECT name FROM sqlite_master WHERE type='table' AND name=?;`, - params: [tableName], - }, - ], - dataSource, - config - ) - - if (!tableExistsResult || tableExistsResult.length === 0) { + if (!(await tableExists(tableName, dataSource, config))) { return null } // Get table data - const dataResult = await executeOperation( - [{ sql: `SELECT * FROM ${tableName};` }], + return executeOperation( + [{ sql: `SELECT * FROM ${quoteIdentifier(tableName)};` }], dataSource, config ) - return dataResult } catch (error: any) { console.error('Table Data Fetch Error:', error) throw error diff --git a/src/export/json.test.ts b/src/export/json.test.ts index 3fe4a8c..8865c51 100644 --- a/src/export/json.test.ts +++ b/src/export/json.test.ts @@ -1,13 +1,14 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { exportTableToJsonRoute } from './json' -import { getTableData, createExportResponse } from './index' +import { getTableDataPage, tableExists } from './index' import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' vi.mock('./index', () => ({ - getTableData: vi.fn(), - createExportResponse: vi.fn(), + EXPORT_PAGE_SIZE: 500, + getTableDataPage: vi.fn(), + tableExists: vi.fn(), })) vi.mock('../utils', () => ({ @@ -41,7 +42,7 @@ beforeEach(() => { describe('JSON Export Module', () => { it('should return a 404 response if table does not exist', async () => { - vi.mocked(getTableData).mockResolvedValue(null) + vi.mocked(tableExists).mockResolvedValue(false) const response = await exportTableToJsonRoute( 'missing_table', @@ -59,13 +60,8 @@ describe('JSON Export Module', () => { { id: 1, name: 'Alice' }, { id: 2, name: 'Bob' }, ] - vi.mocked(getTableData).mockResolvedValue(mockData) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataPage).mockResolvedValueOnce(mockData) const response = await exportTableToJsonRoute( 'users', @@ -73,27 +69,16 @@ describe('JSON Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( - 'users', - mockDataSource, - mockConfig - ) - expect(createExportResponse).toHaveBeenCalledWith( - JSON.stringify(mockData, null, 4), - 'users_export.json', - 'application/json' - ) expect(response.headers.get('Content-Type')).toBe('application/json') + expect(response.headers.get('Content-Disposition')).toBe( + 'attachment; filename="users_export.json"' + ) + await expect(response.json()).resolves.toEqual(mockData) }) it('should return an empty JSON array when table has no data', async () => { - vi.mocked(getTableData).mockResolvedValue([]) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataPage).mockResolvedValueOnce([]) const response = await exportTableToJsonRoute( 'empty_table', @@ -101,12 +86,8 @@ describe('JSON Export Module', () => { mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - '[]', - 'empty_table_export.json', - 'application/json' - ) expect(response.headers.get('Content-Type')).toBe('application/json') + await expect(response.json()).resolves.toEqual([]) }) it('should escape special characters in JSON properly', async () => { @@ -114,33 +95,55 @@ describe('JSON Export Module', () => { { id: 1, name: 'Sahithi "The Best"' }, { id: 2, description: 'New\nLine' }, ] - vi.mocked(getTableData).mockResolvedValue(specialCharsData) + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataPage).mockResolvedValueOnce(specialCharsData) - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) + const response = await exportTableToJsonRoute( + 'special_chars', + mockDataSource, + mockConfig ) + expect(response.headers.get('Content-Type')).toBe('application/json') + await expect(response.json()).resolves.toEqual(specialCharsData) + }) + + it('should stream table data in pages instead of loading a full table at once', async () => { + const firstPage = Array.from({ length: 500 }, (_, index) => ({ + id: index + 1, + })) + const secondPage = [{ id: 501 }] + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataPage) + .mockResolvedValueOnce(firstPage) + .mockResolvedValueOnce(secondPage) + const response = await exportTableToJsonRoute( - 'special_chars', + 'users', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - JSON.stringify(specialCharsData, null, 4), - 'special_chars_export.json', - 'application/json' + await expect(response.json()).resolves.toHaveLength(501) + expect(getTableDataPage).toHaveBeenCalledWith( + 'users', + 0, + mockDataSource, + mockConfig + ) + expect(getTableDataPage).toHaveBeenCalledWith( + 'users', + 500, + mockDataSource, + mockConfig ) - expect(response.headers.get('Content-Type')).toBe('application/json') }) it('should return a 500 response when an error occurs', async () => { const consoleErrorMock = vi .spyOn(console, 'error') .mockImplementation(() => {}) - vi.mocked(getTableData).mockRejectedValue(new Error('Database Error')) + vi.mocked(tableExists).mockRejectedValue(new Error('Database Error')) const response = await exportTableToJsonRoute( 'users', diff --git a/src/export/json.ts b/src/export/json.ts index c0ab811..4087e86 100644 --- a/src/export/json.ts +++ b/src/export/json.ts @@ -1,4 +1,4 @@ -import { getTableData, createExportResponse } from './index' +import { EXPORT_PAGE_SIZE, getTableDataPage, tableExists } from './index' import { createResponse } from '../utils' import { DataSource } from '../types' import { StarbaseDBConfiguration } from '../handler' @@ -9,9 +9,7 @@ export async function exportTableToJsonRoute( config: StarbaseDBConfiguration ): Promise { try { - const data = await getTableData(tableName, dataSource, config) - - if (data === null) { + if (!(await tableExists(tableName, dataSource, config))) { return createResponse( undefined, `Table '${tableName}' does not exist.`, @@ -19,14 +17,51 @@ export async function exportTableToJsonRoute( ) } - // Convert the result to JSON - const jsonData = JSON.stringify(data, null, 4) + const encoder = new TextEncoder() + const stream = new ReadableStream({ + async start(controller) { + controller.enqueue(encoder.encode('[')) + + let offset = 0 + let isFirstRow = true + let hasMoreRows = true + + while (hasMoreRows) { + const dataResult = await getTableDataPage( + tableName, + offset, + dataSource, + config + ) + + for (const row of dataResult) { + controller.enqueue( + encoder.encode( + `${isFirstRow ? '' : ','}${JSON.stringify(row)}` + ) + ) + isFirstRow = false + } + + hasMoreRows = dataResult.length === EXPORT_PAGE_SIZE + offset += EXPORT_PAGE_SIZE + + if (hasMoreRows) { + await new Promise((resolve) => setTimeout(resolve, 0)) + } + } + + controller.enqueue(encoder.encode(']')) + controller.close() + }, + }) - return createExportResponse( - jsonData, - `${tableName}_export.json`, - 'application/json' - ) + return new Response(stream, { + headers: { + 'Content-Type': 'application/json', + 'Content-Disposition': `attachment; filename="${tableName}_export.json"`, + }, + }) } catch (error: any) { console.error('JSON Export Error:', error) return createResponse(undefined, 'Failed to export table to JSON', 500)