From bffb802ba2055b5a7d66630703e7e7e714aa53a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?=
 =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?=
 =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?=
 <aditya@netroy.in>
Date: Mon, 23 Sep 2024 20:42:29 +0200
Subject: [PATCH] implement batching

---
 packages/cli/src/commands/export/all.ts | 46 +++++++++++++++----------
 1 file changed, 28 insertions(+), 18 deletions(-)

diff --git a/packages/cli/src/commands/export/all.ts b/packages/cli/src/commands/export/all.ts
index cf0e3cf4f1..ed93a4d8ac 100644
--- a/packages/cli/src/commands/export/all.ts
+++ b/packages/cli/src/commands/export/all.ts
@@ -38,32 +38,42 @@ export class ExportAllCommand extends BaseCommand {
 		await fs.promises.mkdir(backupPath, { recursive: true });
 
 		for (const { name: tableName, columns } of tables) {
-			// TODO: implement batching
-
-			const rows = await connection.query(`SELECT * from ${tableName}`);
+			const totalRowsCount = await connection
+				.query(`SELECT COUNT(*) AS count FROM ${tableName}`)
+				.then((rows: Array<{ count: number }>) => rows[0].count);
 
 			const stream = fs.createWriteStream(join(backupPath, `${tableName}.jsonl`));
 
-			for (const row of rows) {
-				// Our sqlite setup has some quirks. The following code normalizes the exported data so that it can be imported into a new postgres or sqlite database.
-				if (this.globalConfig.database.type === 'sqlite') {
-					for (const { type: columnType, propertyName } of columns) {
-						if (propertyName in row) {
-							// Our sqlite setup used `simple-json` for JSON columns, which is stored as strings.
-							// This is because when we wrote this code, sqlite did not support native JSON column types.
-							if (columnType === jsonColumnType) {
-								row[propertyName] = JSON.parse(row[propertyName]);
-							}
-							// Sqlite does not have a separate Boolean data type, and uses integers 0/1 to mark values as boolean
-							else if (columnType === Boolean) {
-								row[propertyName] = Boolean(row[propertyName]);
+			let cursor = 0;
+			const batchSize = 10;
+			while (cursor < totalRowsCount) {
+				const rows = await connection.query(
+					`SELECT * from ${tableName} LIMIT ${cursor}, ${batchSize}`,
+				);
+
+				for (const row of rows) {
+					// Our sqlite setup has some quirks. The following code normalizes the exported data so that it can be imported into a new postgres or sqlite database.
+					if (this.globalConfig.database.type === 'sqlite') {
+						for (const { type: columnType, propertyName } of columns) {
+							if (propertyName in row) {
+								// Our sqlite setup used `simple-json` for JSON columns, which is stored as strings.
+								// This is because when we wrote this code, sqlite did not support native JSON column types.
+								if (columnType === jsonColumnType) {
+									row[propertyName] = JSON.parse(row[propertyName]);
+								}
+								// Sqlite does not have a separate Boolean data type, and uses integers 0/1 to mark values as boolean
+								else if (columnType === Boolean) {
+									row[propertyName] = Boolean(row[propertyName]);
+								}
 							}
 						}
 					}
+
+					stream.write(JSON.stringify(row));
+					stream.write('\n');
 				}
 
-				stream.write(JSON.stringify(row));
-				stream.write('\n');
+				cursor += batchSize;
 			}
 
 			stream.end();