mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-03-05 20:50:17 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Go back to write intermediary files

Browse source
This commit is contained in:
Iván Ovejero 2024-10-11 12:44:14 +02:00
parent 879362a5b9
commit b2e0c9e4c9
No known key found for this signature in database
1 changed files with 19 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
packages/cli/src/databases/import-export/database-export.service.ts
View file

@ -45,7 +45,7 @@ export class DatabaseExportService {
const month = String(now.getMonth() + 1).padStart(2, '0'); const month = String(now.getMonth() + 1).padStart(2, '0');
const day = String(now.getDate()).padStart(2, '0'); const day = String(now.getDate()).padStart(2, '0');
const tarballFileName = `${ZIP_BASE_FILE_NAME}-${year}-${month}-${day}.tar.gz`; const tarballFileName = `${ZIP_BASE_FILE_NAME}-${year}-${month}-${day}.tar.zip`;
return path.join(this.config.outDir, tarballFileName); return path.join(this.config.outDir, tarballFileName);
} }
@ -73,6 +73,8 @@ export class DatabaseExportService {
private async writeTarball() { private async writeTarball() {
const pack = tar.pack(); const pack = tar.pack();
pack.pipe(createGzip()).pipe(fs.createWriteStream(this.tarballPath));
// DB row -> entryStream -> tarStream -> gzipStream -> writeStream // DB row -> entryStream -> tarStream -> gzipStream -> writeStream
const tables = const tables =
@ -81,7 +83,15 @@ export class DatabaseExportService {
: this.schemaService.getTables().filter((t) => !EXCLUDE_LIST.includes(t.tableName)); : this.schemaService.getTables().filter((t) => !EXCLUDE_LIST.includes(t.tableName));
for (const { tableName, columns } of tables) { for (const { tableName, columns } of tables) {
const entry = pack.entry({ name: `${tableName}.jsonl` }); const totalRowsCount = await this.schemaService
.getDataSource()
.query(`SELECT COUNT(*) AS count FROM ${tableName}`)
.then((rows: Array<{ count: number }>) => rows[0].count);
if (totalRowsCount === 0) continue;
const tableFilePath = path.join(this.config.outDir, `${tableName}.jsonl`);
const writeStream = fs.createWriteStream(tableFilePath);
let offset = 0; let offset = 0;
let totalRows = 0; let totalRows = 0;
@ -98,7 +108,9 @@ export class DatabaseExportService {
for (const column of columns) { for (const column of columns) {
this.normalizeRow(row, { column, tableName }); this.normalizeRow(row, { column, tableName });
} }
entry.write(JSON.stringify(row) + '\n'); const json = JSON.stringify(row);
writeStream.write(json);
writeStream.write('\n');
} }
totalRows += batch.length; totalRows += batch.length;
@ -109,6 +121,10 @@ export class DatabaseExportService {
this.rowCounts[tableName] = totalRows; this.rowCounts[tableName] = totalRows;
this.logger.info(`[ExportService] Exported ${totalRows} rows from ${tableName}`); this.logger.info(`[ExportService] Exported ${totalRows} rows from ${tableName}`);
writeStream.end();
pack.entry({ name: `${tableName}.jsonl` }, await fs.promises.readFile(tableFilePath));
await fs.promises.rm(tableFilePath);
} }
const manifest: Manifest = { const manifest: Manifest = {
@ -124,9 +140,6 @@ export class DatabaseExportService {
pack.entry({ name: MANIFEST_FILENAME }, manifestBuffer); pack.entry({ name: MANIFEST_FILENAME }, manifestBuffer);
pack.finalize(); pack.finalize();
// pack.pipe(process.stdout);
pack.pipe(createGzip()).pipe(fs.createWriteStream(this.tarballPath));
} }
private normalizeRow( private normalizeRow(