diff --git a/packages/cli/package.json b/packages/cli/package.json index 9b50da508a..35a2b889d0 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -71,6 +71,7 @@ "@types/superagent": "^8.1.7", "@types/swagger-ui-express": "^4.1.6", "@types/syslog-client": "^1.1.2", + "@types/unzip-stream": "^0.3.4", "@types/uuid": "catalog:", "@types/validator": "^13.7.0", "@types/ws": "^8.5.4", @@ -170,6 +171,7 @@ "swagger-ui-express": "5.0.0", "syslog-client": "1.1.1", "typedi": "catalog:", + "unzip-stream": "0.3.4", "uuid": "catalog:", "validator": "13.7.0", "winston": "3.8.2", diff --git a/packages/cli/src/commands/import/all.ts b/packages/cli/src/commands/import/all.ts index 65064f4154..b7024a564f 100644 --- a/packages/cli/src/commands/import/all.ts +++ b/packages/cli/src/commands/import/all.ts @@ -2,35 +2,31 @@ import { Flags } from '@oclif/core'; import { DataSource, MigrationExecutor } from '@n8n/typeorm'; import * as assert from 'assert/strict'; import fs from 'fs'; +import readline from 'readline'; import { join } from 'path'; import Container from 'typedi'; import { tmpdir } from 'node:os'; +import { pipeline } from 'node:stream/promises'; +import { Extract } from 'unzip-stream'; import { BaseCommand } from '../base-command'; import { ApplicationError } from 'n8n-workflow'; -// TODO: do this -//const fs = require('fs'); -//const readline = require('readline'); -// -//(async () => { -// const fileStream = fs.createReadStream(__dirname + '/test.jsonl'); -// const lineStream = readline.createInterface({ -// input: fileStream, -// crlfDelay: Infinity, -// }); -// -// for await (const line of lineStream) { -// console.log(JSON.parse(line)); -// } -//})(); +const excludeList = [ + 'execution_annotation_tags', + 'execution_annotations', + 'execution_data', + 'execution_entity', + 'execution_metadata', + 'annotation_tag_entity', +]; export class ImportAllCommand extends BaseCommand { static description = 'Import Everything'; static examples = ['$ n8n import:all', '$ n8n import:all --input=backup.zip']; - // TODO: add `clean` flag, or add a prompt to confirm + // TODO: add `clean` flag, or add a prompt to confirm DB truncation static flags = { input: Flags.string({ char: 'o', @@ -41,6 +37,7 @@ export class ImportAllCommand extends BaseCommand { // TODO: do batching async run() { + const { flags } = await this.parse(ImportAllCommand); // TODO: // 1. check last migrations const connection = Container.get(DataSource); @@ -50,10 +47,17 @@ export class ImportAllCommand extends BaseCommand { assert.ok(lastExecutedMigration, 'should have been run by db.ts'); + const zipPath = join(flags.input, 'n8n-backup.zip'); + if (!fs.existsSync(zipPath)) { + throw new ApplicationError('Backup zip file not count'); + } + + // TODO: instead of extracting to the filesystem, stream the files directly const backupPath = '/tmp/backup'; + await pipeline(fs.createReadStream(zipPath), Extract({ path: backupPath })); const lastMigrationInBackup = ( - await fs.promises.readFile(join(backupPath, 'lastMigration'), 'utf8') + await fs.promises.readFile(join(backupPath, '.lastMigration'), 'utf8') ).trim(); if (lastMigrationInBackup !== lastExecutedMigration.name) { @@ -65,14 +69,6 @@ export class ImportAllCommand extends BaseCommand { // 3. disable foreign keys // 4. import each jsonl - const excludeList = [ - 'execution_annotation_tags', - 'execution_annotations', - 'execution_data', - 'execution_entity', - 'execution_metadata', - 'annotation_tag_entity', - ]; const tables = connection.entityMetadatas .filter((v) => !excludeList.includes(v.tableName)) .map((v) => ({ name: v.tableName, target: v.target })); @@ -81,13 +77,20 @@ export class ImportAllCommand extends BaseCommand { const repo = connection.getRepository(target); await repo.delete({}); - const rows = (await fs.promises.readFile(`${join(backupPath, name)}.jsonl`, 'utf8')) - .split('\n') - .filter((row) => row !== ''); + const filePath = join(backupPath, `${name}.jsonl`); + if (!fs.existsSync(filePath)) continue; - for (const row of rows) { - await repo.insert(JSON.parse(row)); + const fileStream = fs.createReadStream(filePath); + const lineStream = readline.createInterface({ + input: fileStream, + }); + + for await (const line of lineStream) { + // TODO: insert in batches to reduce DB load + await repo.insert(JSON.parse(line)); } + + fileStream.close(); } // 5. enable foreign keys diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8815431150..bfe8578f9a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -935,6 +935,9 @@ importers: typedi: specifier: 'catalog:' version: 0.10.0(patch_hash=sk6omkefrosihg7lmqbzh7vfxe) + unzip-stream: + specifier: 0.3.4 + version: 0.3.4 uuid: specifier: 'catalog:' version: 10.0.0 @@ -1023,6 +1026,9 @@ importers: '@types/syslog-client': specifier: ^1.1.2 version: 1.1.2 + '@types/unzip-stream': + specifier: ^0.3.4 + version: 0.3.4 '@types/uuid': specifier: 'catalog:' version: 10.0.0 @@ -5192,6 +5198,9 @@ packages: '@types/unist@3.0.2': resolution: {integrity: sha512-dqId9J8K/vGi5Zr7oo212BGii5m3q5Hxlkwy3WpYuKPklmBEvsbMYYyLxAQpSffdLl/gdW0XUpKWFvYmyoWCoQ==} + '@types/unzip-stream@0.3.4': + resolution: {integrity: sha512-ud0vtsNRF+joUCyvNMyo0j5DKX2Lh/im+xVgRzBEsfHhQYZ+i4fKTveova9XxLzt6Jl6G0e/0mM4aC0gqZYSnA==} + '@types/utf8@3.0.3': resolution: {integrity: sha512-+lqLGxWZsEe4Z6OrzBI7Ym4SMUTaMS5yOrHZ0/IL0bpIye1Qbs4PpobJL2mLDbftUXlPFZR7fu6d1yM+bHLX1w==} @@ -5948,6 +5957,9 @@ packages: resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} engines: {node: '>=8'} + binary@0.3.0: + resolution: {integrity: sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==} + binascii@0.0.2: resolution: {integrity: sha512-rA2CrUl1+6yKrn+XgLs8Hdy18OER1UW146nM+ixzhQXDY+Bd3ySkyIJGwF2a4I45JwbvF1mDL/nWkqBwpOcdBA==} @@ -6046,6 +6058,10 @@ packages: buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + buffers@0.1.1: + resolution: {integrity: sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==} + engines: {node: '>=0.2.0'} + buildcheck@0.0.6: resolution: {integrity: sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==} engines: {node: '>=10.0.0'} @@ -6137,6 +6153,9 @@ packages: resolution: {integrity: sha512-pT1ZgP8rPNqUgieVaEY+ryQr6Q4HXNg8Ei9UnLUrjN4IA7dvQC5JB+/kxVcPNDHyBcc/26CXPkbNzq3qwrOEKA==} engines: {node: '>=12'} + chainsaw@0.1.0: + resolution: {integrity: sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==} + chalk@2.4.2: resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} engines: {node: '>=4'} @@ -11725,6 +11744,9 @@ packages: engines: {node: '>=6.0.0'} hasBin: true + traverse@0.3.9: + resolution: {integrity: sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==} + tree-kill@1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true @@ -12091,6 +12113,9 @@ packages: resolution: {integrity: sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==} engines: {node: '>=8'} + unzip-stream@0.3.4: + resolution: {integrity: sha512-PyofABPVv+d7fL7GOpusx7eRT9YETY2X04PhwbSipdj6bMxVCFJrr+nm0Mxqbf9hUiTin/UsnuFWBXlDZFy0Cw==} + update-browserslist-db@1.0.13: resolution: {integrity: sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==} hasBin: true @@ -17494,6 +17519,10 @@ snapshots: '@types/unist@3.0.2': {} + '@types/unzip-stream@0.3.4': + dependencies: + '@types/node': 18.16.16 + '@types/utf8@3.0.3': {} '@types/uuencode@0.0.3(patch_hash=3i7wecddkama6vhpu5o37g24u4)': @@ -18439,6 +18468,11 @@ snapshots: binary-extensions@2.2.0: {} + binary@0.3.0: + dependencies: + buffers: 0.1.1 + chainsaw: 0.1.0 + binascii@0.0.2: {} bindings@1.5.0: @@ -18547,6 +18581,8 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + buffers@0.1.1: {} + buildcheck@0.0.6: optional: true @@ -18668,6 +18704,10 @@ snapshots: loupe: 3.1.1 pathval: 2.0.0 + chainsaw@0.1.0: + dependencies: + traverse: 0.3.9 + chalk@2.4.2: dependencies: ansi-styles: 3.2.1 @@ -25551,6 +25591,8 @@ snapshots: dependencies: yargs: 17.7.2 + traverse@0.3.9: {} + tree-kill@1.2.2: {} triple-beam@1.3.0: {} @@ -25905,6 +25947,11 @@ snapshots: untildify@4.0.0: {} + unzip-stream@0.3.4: + dependencies: + binary: 0.3.0 + mkdirp: 0.5.6 + update-browserslist-db@1.0.13(browserslist@4.23.0): dependencies: browserslist: 4.23.0