diff --git a/packages/cli/src/commands/export/all.ts b/packages/cli/src/commands/export/all.ts index cf0e3cf4f1..ed93a4d8ac 100644 --- a/packages/cli/src/commands/export/all.ts +++ b/packages/cli/src/commands/export/all.ts @@ -38,32 +38,42 @@ export class ExportAllCommand extends BaseCommand { await fs.promises.mkdir(backupPath, { recursive: true }); for (const { name: tableName, columns } of tables) { - // TODO: implement batching - - const rows = await connection.query(`SELECT * from ${tableName}`); + const totalRowsCount = await connection + .query(`SELECT COUNT(*) AS count FROM ${tableName}`) + .then((rows: Array<{ count: number }>) => rows[0].count); const stream = fs.createWriteStream(join(backupPath, `${tableName}.jsonl`)); - for (const row of rows) { - // Our sqlite setup has some quirks. The following code normalizes the exported data so that it can be imported into a new postgres or sqlite database. - if (this.globalConfig.database.type === 'sqlite') { - for (const { type: columnType, propertyName } of columns) { - if (propertyName in row) { - // Our sqlite setup used `simple-json` for JSON columns, which is stored as strings. - // This is because when we wrote this code, sqlite did not support native JSON column types. - if (columnType === jsonColumnType) { - row[propertyName] = JSON.parse(row[propertyName]); - } - // Sqlite does not have a separate Boolean data type, and uses integers 0/1 to mark values as boolean - else if (columnType === Boolean) { - row[propertyName] = Boolean(row[propertyName]); + let cursor = 0; + const batchSize = 10; + while (cursor < totalRowsCount) { + const rows = await connection.query( + `SELECT * from ${tableName} LIMIT ${cursor}, ${batchSize}`, + ); + + for (const row of rows) { + // Our sqlite setup has some quirks. The following code normalizes the exported data so that it can be imported into a new postgres or sqlite database. + if (this.globalConfig.database.type === 'sqlite') { + for (const { type: columnType, propertyName } of columns) { + if (propertyName in row) { + // Our sqlite setup used `simple-json` for JSON columns, which is stored as strings. + // This is because when we wrote this code, sqlite did not support native JSON column types. + if (columnType === jsonColumnType) { + row[propertyName] = JSON.parse(row[propertyName]); + } + // Sqlite does not have a separate Boolean data type, and uses integers 0/1 to mark values as boolean + else if (columnType === Boolean) { + row[propertyName] = Boolean(row[propertyName]); + } } } } + + stream.write(JSON.stringify(row)); + stream.write('\n'); } - stream.write(JSON.stringify(row)); - stream.write('\n'); + cursor += batchSize; } stream.end();