mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-03-05 20:50:17 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

implement batching

Browse source
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™ 2024-09-23 20:42:29 +02:00
parent 0a541a91e8
commit bffb802ba2
No known key found for this signature in database
1 changed files with 28 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
packages/cli/src/commands/export/all.ts
View file

@ -38,32 +38,42 @@ export class ExportAllCommand extends BaseCommand {
await fs.promises.mkdir(backupPath, { recursive: true });
for (const { name: tableName, columns } of tables) {
// TODO: implement batching
const rows = await connection.query(`SELECT * from ${tableName}`);
const totalRowsCount = await connection
.query(`SELECT COUNT(*) AS count FROM ${tableName}`)
.then((rows: Array<{ count: number }>) => rows[0].count);
const stream = fs.createWriteStream(join(backupPath, `${tableName}.jsonl`));
for (const row of rows) {
// Our sqlite setup has some quirks. The following code normalizes the exported data so that it can be imported into a new postgres or sqlite database.
if (this.globalConfig.database.type === 'sqlite') {
for (const { type: columnType, propertyName } of columns) {
if (propertyName in row) {
// Our sqlite setup used `simple-json` for JSON columns, which is stored as strings.
// This is because when we wrote this code, sqlite did not support native JSON column types.
if (columnType === jsonColumnType) {
row[propertyName] = JSON.parse(row[propertyName]);
}
// Sqlite does not have a separate Boolean data type, and uses integers 0/1 to mark values as boolean
else if (columnType === Boolean) {
row[propertyName] = Boolean(row[propertyName]);
let cursor = 0;
const batchSize = 10;
while (cursor < totalRowsCount) {
const rows = await connection.query(
`SELECT * from ${tableName} LIMIT ${cursor}, ${batchSize}`,
);
for (const row of rows) {
// Our sqlite setup has some quirks. The following code normalizes the exported data so that it can be imported into a new postgres or sqlite database.
if (this.globalConfig.database.type === 'sqlite') {
for (const { type: columnType, propertyName } of columns) {
if (propertyName in row) {
// Our sqlite setup used `simple-json` for JSON columns, which is stored as strings.
// This is because when we wrote this code, sqlite did not support native JSON column types.
if (columnType === jsonColumnType) {
row[propertyName] = JSON.parse(row[propertyName]);
}
// Sqlite does not have a separate Boolean data type, and uses integers 0/1 to mark values as boolean
else if (columnType === Boolean) {
row[propertyName] = Boolean(row[propertyName]);
}
}
}
}
stream.write(JSON.stringify(row));
stream.write('\n');
}
stream.write(JSON.stringify(row));
stream.write('\n');
cursor += batchSize;
}
stream.end();