Stream SQL, CSV, and JSON exports with paged reads

src/export/csv.test.ts

@@ -1,13 +1,12 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest'
 import { exportTableToCsvRoute } from './csv'
-import { getTableData, createExportResponse } from './index'
+import { executeOperation } from './index'
 import { createResponse } from '../utils'
 import type { DataSource } from '../types'
 import type { StarbaseDBConfiguration } from '../handler'
 
 vi.mock('./index', () => ({
-    getTableData: vi.fn(),
-    createExportResponse: vi.fn(),
+    executeOperation: vi.fn(),
 }))
 
 vi.mock('../utils', () => ({
@@ -23,6 +22,16 @@ vi.mock('../utils', () => ({
 let mockDataSource: DataSource
 let mockConfig: StarbaseDBConfiguration
 
+const tableColumns = (names: string[]) =>
+    names.map((name, index) => ({
+        cid: index,
+        name,
+        type: '',
+        notnull: 0,
+        dflt_value: null,
+        pk: name === 'id' ? 1 : 0,
+    }))
+
 beforeEach(() => {
     vi.clearAllMocks()
 
@@ -43,50 +52,38 @@ beforeEach(() => {
 
 describe('CSV Export Module', () => {
     it('should return a CSV file when table data exists', async () => {
-        vi.mocked(getTableData).mockResolvedValue([
-            { id: 1, name: 'Alice', age: 30 },
-            { id: 2, name: 'Bob', age: 25 },
-        ])
-
-        vi.mocked(createExportResponse).mockReturnValue(
-            new Response('mocked-csv-content', {
-                headers: { 'Content-Type': 'text/csv' },
-            })
-        )
+        vi.mocked(executeOperation)
+            .mockResolvedValueOnce([{ name: 'users' }])
+            .mockResolvedValueOnce(tableColumns(['id', 'name', 'age']))
+            .mockResolvedValueOnce([
+                { id: 1, name: 'Alice', age: 30 },
+                { id: 2, name: 'Bob', age: 25 },
+            ])
 
         const response = await exportTableToCsvRoute(
             'users',
             mockDataSource,
             mockConfig
         )
 
-        expect(getTableData).toHaveBeenCalledWith(
-            'users',
-            mockDataSource,
-            mockConfig
+        expect(response.headers.get('Content-Type')).toBe('text/csv')
+        expect(response.headers.get('Content-Disposition')).toBe(
+            'attachment; filename="users_export.csv"'
         )
-        expect(createExportResponse).toHaveBeenCalledWith(
-            'id,name,age\n1,Alice,30\n2,Bob,25\n',
-            'users_export.csv',
-            'text/csv'
+        await expect(response.text()).resolves.toBe(
+            'id,name,age\n1,Alice,30\n2,Bob,25\n'
         )
-        expect(response.headers.get('Content-Type')).toBe('text/csv')
     })
 
     it('should return 404 if table does not exist', async () => {
-        vi.mocked(getTableData).mockResolvedValue(null)
+        vi.mocked(executeOperation).mockResolvedValueOnce([])
 
         const response = await exportTableToCsvRoute(
             'non_existent_table',
             mockDataSource,
             mockConfig
         )
 
-        expect(getTableData).toHaveBeenCalledWith(
-            'non_existent_table',
-            mockDataSource,
-            mockConfig
-        )
         expect(response.status).toBe(404)
 
         const jsonResponse: { error: string } = await response.json()
@@ -96,54 +93,37 @@ describe('CSV Export Module', () => {
     })
 
     it('should handle empty table (return only headers)', async () => {
-        vi.mocked(getTableData).mockResolvedValue([])
-
-        vi.mocked(createExportResponse).mockReturnValue(
-            new Response('mocked-csv-content', {
-                headers: { 'Content-Type': 'text/csv' },
-            })
-        )
+        vi.mocked(executeOperation)
+            .mockResolvedValueOnce([{ name: 'empty_table' }])
+            .mockResolvedValueOnce(tableColumns(['id', 'name']))
+            .mockResolvedValueOnce([])
 
         const response = await exportTableToCsvRoute(
             'empty_table',
             mockDataSource,
             mockConfig
         )
 
-        expect(getTableData).toHaveBeenCalledWith(
-            'empty_table',
-            mockDataSource,
-            mockConfig
-        )
-        expect(createExportResponse).toHaveBeenCalledWith(
-            '',
-            'empty_table_export.csv',
-            'text/csv'
-        )
         expect(response.headers.get('Content-Type')).toBe('text/csv')
+        await expect(response.text()).resolves.toBe('id,name\n')
     })
 
     it('should escape commas and quotes in CSV values', async () => {
-        vi.mocked(getTableData).mockResolvedValue([
-            { id: 1, name: 'Sahithi, is', bio: 'my forever "penguin"' },
-        ])
-
-        vi.mocked(createExportResponse).mockReturnValue(
-            new Response('mocked-csv-content', {
-                headers: { 'Content-Type': 'text/csv' },
-            })
-        )
+        vi.mocked(executeOperation)
+            .mockResolvedValueOnce([{ name: 'special_chars' }])
+            .mockResolvedValueOnce(tableColumns(['id', 'name', 'bio']))
+            .mockResolvedValueOnce([
+                { id: 1, name: 'Sahithi, is', bio: 'my forever "penguin"' },
+            ])
 
         const response = await exportTableToCsvRoute(
             'special_chars',
             mockDataSource,
             mockConfig
         )
 
-        expect(createExportResponse).toHaveBeenCalledWith(
-            'id,name,bio\n1,"Sahithi, is","my forever ""penguin"""\n',
-            'special_chars_export.csv',
-            'text/csv'
+        await expect(response.text()).resolves.toBe(
+            'id,name,bio\n1,"Sahithi, is","my forever ""penguin"""\n'
         )
         expect(response.headers.get('Content-Type')).toBe('text/csv')
     })
@@ -152,7 +132,9 @@ describe('CSV Export Module', () => {
         const consoleErrorMock = vi
             .spyOn(console, 'error')
             .mockImplementation(() => {})
-        vi.mocked(getTableData).mockRejectedValue(new Error('Database Error'))
+        vi.mocked(executeOperation).mockRejectedValue(
+            new Error('Database Error')
+        )
 
         const response = await exportTableToCsvRoute(
             'users',

src/export/csv.ts

@@ -1,51 +1,30 @@
-import { getTableData, createExportResponse } from './index'
 import { createResponse } from '../utils'
 import { DataSource } from '../types'
 import { StarbaseDBConfiguration } from '../handler'
+import {
+    createStreamingExportResponse,
+    csvTableChunks,
+    getTablePagePlan,
+} from './streaming'
 
 export async function exportTableToCsvRoute(
     tableName: string,
     dataSource: DataSource,
     config: StarbaseDBConfiguration
 ): Promise<Response> {
     try {
-        const data = await getTableData(tableName, dataSource, config)
+        const pagePlan = await getTablePagePlan(tableName, dataSource, config)
 
-        if (data === null) {
+        if (!pagePlan) {
             return createResponse(
                 undefined,
                 `Table '${tableName}' does not exist.`,
                 404
             )
         }
 
-        // Convert the result to CSV
-        let csvContent = ''
-        if (data.length > 0) {
-            // Add headers
-            csvContent += Object.keys(data[0]).join(',') + '\n'
-
-            // Add data rows
-            data.forEach((row: any) => {
-                csvContent +=
-                    Object.values(row)
-                        .map((value) => {
-                            if (
-                                typeof value === 'string' &&
-                                (value.includes(',') ||
-                                    value.includes('"') ||
-                                    value.includes('\n'))
-                            ) {
-                                return `"${value.replace(/"/g, '""')}"`
-                            }
-                            return value
-                        })
-                        .join(',') + '\n'
-            })
-        }
-
-        return createExportResponse(
-            csvContent,
+        return createStreamingExportResponse(
+            csvTableChunks(tableName, dataSource, config, pagePlan),
             `${tableName}_export.csv`,
             'text/csv'
         )

src/export/dump.test.ts

@@ -22,6 +22,16 @@ vi.mock('../utils', () => ({
 let mockDataSource: DataSource
 let mockConfig: StarbaseDBConfiguration
 
+const tableColumns = (names: string[]) =>
+    names.map((name, index) => ({
+        cid: index,
+        name,
+        type: '',
+        notnull: 0,
+        dflt_value: null,
+        pk: name === 'id' ? 1 : 0,
+    }))
+
 beforeEach(() => {
     vi.clearAllMocks()
 
@@ -45,13 +55,17 @@ describe('Database Dump Module', () => {
             .mockResolvedValueOnce([
                 { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' },
             ])
+            .mockResolvedValueOnce([{ name: 'users' }])
+            .mockResolvedValueOnce(tableColumns(['id', 'name']))
             .mockResolvedValueOnce([
                 { id: 1, name: 'Alice' },
                 { id: 2, name: 'Bob' },
             ])
             .mockResolvedValueOnce([
                 { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' },
             ])
+            .mockResolvedValueOnce([{ name: 'orders' }])
+            .mockResolvedValueOnce(tableColumns(['id', 'total']))
             .mockResolvedValueOnce([
                 { id: 1, total: 99.99 },
                 { id: 2, total: 49.5 },
@@ -71,13 +85,13 @@ describe('Database Dump Module', () => {
         expect(dumpText).toContain(
             'CREATE TABLE users (id INTEGER, name TEXT);'
         )
-        expect(dumpText).toContain("INSERT INTO users VALUES (1, 'Alice');")
-        expect(dumpText).toContain("INSERT INTO users VALUES (2, 'Bob');")
+        expect(dumpText).toContain('INSERT INTO "users" VALUES (1, \'Alice\');')
+        expect(dumpText).toContain('INSERT INTO "users" VALUES (2, \'Bob\');')
         expect(dumpText).toContain(
             'CREATE TABLE orders (id INTEGER, total REAL);'
         )
-        expect(dumpText).toContain('INSERT INTO orders VALUES (1, 99.99);')
-        expect(dumpText).toContain('INSERT INTO orders VALUES (2, 49.5);')
+        expect(dumpText).toContain('INSERT INTO "orders" VALUES (1, 99.99);')
+        expect(dumpText).toContain('INSERT INTO "orders" VALUES (2, 49.5);')
     })
 
     it('should handle empty databases (no tables)', async () => {
@@ -99,6 +113,8 @@ describe('Database Dump Module', () => {
             .mockResolvedValueOnce([
                 { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' },
             ])
+            .mockResolvedValueOnce([{ name: 'users' }])
+            .mockResolvedValueOnce(tableColumns(['id', 'name']))
             .mockResolvedValueOnce([])
 
         const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
@@ -117,14 +133,16 @@ describe('Database Dump Module', () => {
             .mockResolvedValueOnce([
                 { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' },
             ])
+            .mockResolvedValueOnce([{ name: 'users' }])
+            .mockResolvedValueOnce(tableColumns(['id', 'bio']))
             .mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }])
 
         const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
 
         expect(response).toBeInstanceOf(Response)
         const dumpText = await response.text()
         expect(dumpText).toContain(
-            "INSERT INTO users VALUES (1, 'Alice''s adventure');"
+            "INSERT INTO \"users\" VALUES (1, 'Alice''s adventure');"
         )
     })
 

src/export/dump.ts

@@ -1,69 +1,24 @@
-import { executeOperation } from '.'
 import { StarbaseDBConfiguration } from '../handler'
 import { DataSource } from '../types'
 import { createResponse } from '../utils'
+import {
+    createStreamingExportResponse,
+    listExportableTables,
+    sqlDumpChunks,
+} from './streaming'
 
 export async function dumpDatabaseRoute(
     dataSource: DataSource,
     config: StarbaseDBConfiguration
 ): Promise<Response> {
     try {
-        // Get all table names
-        const tablesResult = await executeOperation(
-            [{ sql: "SELECT name FROM sqlite_master WHERE type='table';" }],
-            dataSource,
-            config
-        )
-
-        const tables = tablesResult.map((row: any) => row.name)
-        let dumpContent = 'SQLite format 3\0' // SQLite file header
-
-        // Iterate through all tables
-        for (const table of tables) {
-            // Get table schema
-            const schemaResult = await executeOperation(
-                [
-                    {
-                        sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`,
-                    },
-                ],
-                dataSource,
-                config
-            )
-
-            if (schemaResult.length) {
-                const schema = schemaResult[0].sql
-                dumpContent += `\n-- Table: ${table}\n${schema};\n\n`
-            }
-
-            // Get table data
-            const dataResult = await executeOperation(
-                [{ sql: `SELECT * FROM ${table};` }],
-                dataSource,
-                config
-            )
+        const tables = await listExportableTables(dataSource, config)
 
-            for (const row of dataResult) {
-                const values = Object.values(row).map((value) =>
-                    typeof value === 'string'
-                        ? `'${value.replace(/'/g, "''")}'`
-                        : value
-                )
-                dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n`
-            }
-
-            dumpContent += '\n'
-        }
-
-        // Create a Blob from the dump content
-        const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' })
-
-        const headers = new Headers({
-            'Content-Type': 'application/x-sqlite3',
-            'Content-Disposition': 'attachment; filename="database_dump.sql"',
-        })
-
-        return new Response(blob, { headers })
+        return createStreamingExportResponse(
+            sqlDumpChunks(tables, dataSource, config),
+            'database_dump.sql',
+            'application/x-sqlite3'
+        )
     } catch (error: any) {
         console.error('Database Dump Error:', error)
         return createResponse(undefined, 'Failed to create database dump', 500)