mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-03-05 20:50:17 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

implement batching

Browse source
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™ 2024-09-23 20:42:29 +02:00
parent 0a541a91e8
commit bffb802ba2
No known key found for this signature in database
1 changed files with 28 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
packages/cli/src/commands/export/all.ts
View file

@ -38,12 +38,19 @@ export class ExportAllCommand extends BaseCommand {
await fs.promises.mkdir(backupPath, { recursive: true });
for (const { name: tableName, columns } of tables) {
// TODO: implement batching
const rows = await connection.query(`SELECT * from ${tableName}`);
const totalRowsCount = await connection
.query(`SELECT COUNT(*) AS count FROM ${tableName}`)
.then((rows: Array<{ count: number }>) => rows[0].count);
const stream = fs.createWriteStream(join(backupPath, `${tableName}.jsonl`));
let cursor = 0;
const batchSize = 10;
while (cursor < totalRowsCount) {
const rows = await connection.query(
`SELECT * from ${tableName} LIMIT ${cursor}, ${batchSize}`,
);
for (const row of rows) {
// Our sqlite setup has some quirks. The following code normalizes the exported data so that it can be imported into a new postgres or sqlite database.
if (this.globalConfig.database.type === 'sqlite') {
@ -66,6 +73,9 @@ export class ExportAllCommand extends BaseCommand {
stream.write('\n');
}
cursor += batchSize;
}
stream.end();
}