working import

packages/cli/package.json

@@ -71,6 +71,7 @@
     "@types/superagent": "^8.1.7",
     "@types/swagger-ui-express": "^4.1.6",
     "@types/syslog-client": "^1.1.2",
+    "@types/unzip-stream": "^0.3.4",
     "@types/uuid": "catalog:",
     "@types/validator": "^13.7.0",
     "@types/ws": "^8.5.4",
@@ -170,6 +171,7 @@
     "swagger-ui-express": "5.0.0",
     "syslog-client": "1.1.1",
     "typedi": "catalog:",
+    "unzip-stream": "0.3.4",
     "uuid": "catalog:",
     "validator": "13.7.0",
     "winston": "3.8.2",

packages/cli/src/commands/import/all.ts

@@ -2,35 +2,31 @@ import { Flags } from '@oclif/core';
 import { DataSource, MigrationExecutor } from '@n8n/typeorm';
 import * as assert from 'assert/strict';
 import fs from 'fs';
+import readline from 'readline';
 import { join } from 'path';
 import Container from 'typedi';
 import { tmpdir } from 'node:os';
+import { pipeline } from 'node:stream/promises';
+import { Extract } from 'unzip-stream';
 
 import { BaseCommand } from '../base-command';
 import { ApplicationError } from 'n8n-workflow';
 
-// TODO: do this
+const excludeList = [
-//const fs = require('fs');
+	'execution_annotation_tags',
-//const readline = require('readline');
+	'execution_annotations',
-//
+	'execution_data',
-//(async () => {
+	'execution_entity',
-//	const fileStream = fs.createReadStream(__dirname + '/test.jsonl');
+	'execution_metadata',
-//	const lineStream = readline.createInterface({
+	'annotation_tag_entity',
-//		input: fileStream,
+];
-//		crlfDelay: Infinity,
-//	});
-//
-//	for await (const line of lineStream) {
-//		console.log(JSON.parse(line));
-//	}
-//})();
 
 export class ImportAllCommand extends BaseCommand {
 	static description = 'Import Everything';
 
 	static examples = ['$ n8n import:all', '$ n8n import:all --input=backup.zip'];
 
-	// TODO: add `clean` flag, or add a prompt to confirm
+	// TODO: add `clean` flag, or add a prompt to confirm DB truncation
 	static flags = {
 		input: Flags.string({
 			char: 'o',
@@ -41,6 +37,7 @@ export class ImportAllCommand extends BaseCommand {
 
 	// TODO: do batching
 	async run() {
+		const { flags } = await this.parse(ImportAllCommand);
 		// TODO:
 		// 1. check last migrations
 		const connection = Container.get(DataSource);
@@ -50,10 +47,17 @@ export class ImportAllCommand extends BaseCommand {
 
 		assert.ok(lastExecutedMigration, 'should have been run by db.ts');
 
+		const zipPath = join(flags.input, 'n8n-backup.zip');
+		if (!fs.existsSync(zipPath)) {
+			throw new ApplicationError('Backup zip file not count');
+		}
+
+		// TODO: instead of extracting to the filesystem, stream the files directly
 		const backupPath = '/tmp/backup';
+		await pipeline(fs.createReadStream(zipPath), Extract({ path: backupPath }));
 
 		const lastMigrationInBackup = (
-			await fs.promises.readFile(join(backupPath, 'lastMigration'), 'utf8')
+			await fs.promises.readFile(join(backupPath, '.lastMigration'), 'utf8')
 		).trim();
 
 		if (lastMigrationInBackup !== lastExecutedMigration.name) {
@@ -65,14 +69,6 @@ export class ImportAllCommand extends BaseCommand {
 		// 3. disable foreign keys
 
 		// 4. import each jsonl
-		const excludeList = [
-			'execution_annotation_tags',
-			'execution_annotations',
-			'execution_data',
-			'execution_entity',
-			'execution_metadata',
-			'annotation_tag_entity',
-		];
 		const tables = connection.entityMetadatas
 			.filter((v) => !excludeList.includes(v.tableName))
 			.map((v) => ({ name: v.tableName, target: v.target }));
@@ -81,13 +77,20 @@ export class ImportAllCommand extends BaseCommand {
 			const repo = connection.getRepository(target);
 			await repo.delete({});
 
-			const rows = (await fs.promises.readFile(`${join(backupPath, name)}.jsonl`, 'utf8'))
+			const filePath = join(backupPath, `${name}.jsonl`);
-				.split('\n')
+			if (!fs.existsSync(filePath)) continue;
-				.filter((row) => row !== '');
 
-			for (const row of rows) {
+			const fileStream = fs.createReadStream(filePath);
-				await repo.insert(JSON.parse(row));
+			const lineStream = readline.createInterface({
+				input: fileStream,
+			});
+
+			for await (const line of lineStream) {
+				// TODO: insert in batches to reduce DB load
+				await repo.insert(JSON.parse(line));
 			}
+
+			fileStream.close();
 		}
 
 		// 5. enable foreign keys

pnpm-lock.yaml

@@ -935,6 +935,9 @@ importers:
       typedi:
         specifier: 'catalog:'
         version: 0.10.0(patch_hash=sk6omkefrosihg7lmqbzh7vfxe)
+      unzip-stream:
+        specifier: 0.3.4
+        version: 0.3.4
       uuid:
         specifier: 'catalog:'
         version: 10.0.0
@@ -1023,6 +1026,9 @@ importers:
       '@types/syslog-client':
         specifier: ^1.1.2
         version: 1.1.2
+      '@types/unzip-stream':
+        specifier: ^0.3.4
+        version: 0.3.4
       '@types/uuid':
         specifier: 'catalog:'
         version: 10.0.0
@@ -5192,6 +5198,9 @@ packages:
   '@types/unist@3.0.2':
     resolution: {integrity: sha512-dqId9J8K/vGi5Zr7oo212BGii5m3q5Hxlkwy3WpYuKPklmBEvsbMYYyLxAQpSffdLl/gdW0XUpKWFvYmyoWCoQ==}
 
+  '@types/unzip-stream@0.3.4':
+    resolution: {integrity: sha512-ud0vtsNRF+joUCyvNMyo0j5DKX2Lh/im+xVgRzBEsfHhQYZ+i4fKTveova9XxLzt6Jl6G0e/0mM4aC0gqZYSnA==}
+
   '@types/utf8@3.0.3':
     resolution: {integrity: sha512-+lqLGxWZsEe4Z6OrzBI7Ym4SMUTaMS5yOrHZ0/IL0bpIye1Qbs4PpobJL2mLDbftUXlPFZR7fu6d1yM+bHLX1w==}
 
@@ -5948,6 +5957,9 @@ packages:
     resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==}
     engines: {node: '>=8'}
 
+  binary@0.3.0:
+    resolution: {integrity: sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==}
+
   binascii@0.0.2:
     resolution: {integrity: sha512-rA2CrUl1+6yKrn+XgLs8Hdy18OER1UW146nM+ixzhQXDY+Bd3ySkyIJGwF2a4I45JwbvF1mDL/nWkqBwpOcdBA==}
 
@@ -6046,6 +6058,10 @@ packages:
   buffer@6.0.3:
     resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
 
+  buffers@0.1.1:
+    resolution: {integrity: sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==}
+    engines: {node: '>=0.2.0'}
+
   buildcheck@0.0.6:
     resolution: {integrity: sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==}
     engines: {node: '>=10.0.0'}
@@ -6137,6 +6153,9 @@ packages:
     resolution: {integrity: sha512-pT1ZgP8rPNqUgieVaEY+ryQr6Q4HXNg8Ei9UnLUrjN4IA7dvQC5JB+/kxVcPNDHyBcc/26CXPkbNzq3qwrOEKA==}
     engines: {node: '>=12'}
 
+  chainsaw@0.1.0:
+    resolution: {integrity: sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==}
+
   chalk@2.4.2:
     resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==}
     engines: {node: '>=4'}
@@ -11725,6 +11744,9 @@ packages:
     engines: {node: '>=6.0.0'}
     hasBin: true
 
+  traverse@0.3.9:
+    resolution: {integrity: sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==}
+
   tree-kill@1.2.2:
     resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==}
     hasBin: true
@@ -12091,6 +12113,9 @@ packages:
     resolution: {integrity: sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==}
     engines: {node: '>=8'}
 
+  unzip-stream@0.3.4:
+    resolution: {integrity: sha512-PyofABPVv+d7fL7GOpusx7eRT9YETY2X04PhwbSipdj6bMxVCFJrr+nm0Mxqbf9hUiTin/UsnuFWBXlDZFy0Cw==}
+
   update-browserslist-db@1.0.13:
     resolution: {integrity: sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==}
     hasBin: true
@@ -17494,6 +17519,10 @@ snapshots:
 
   '@types/unist@3.0.2': {}
 
+  '@types/unzip-stream@0.3.4':
+    dependencies:
+      '@types/node': 18.16.16
+
   '@types/utf8@3.0.3': {}
 
   '@types/uuencode@0.0.3(patch_hash=3i7wecddkama6vhpu5o37g24u4)':
@@ -18439,6 +18468,11 @@ snapshots:
 
   binary-extensions@2.2.0: {}
 
+  binary@0.3.0:
+    dependencies:
+      buffers: 0.1.1
+      chainsaw: 0.1.0
+
   binascii@0.0.2: {}
 
   bindings@1.5.0:
@@ -18547,6 +18581,8 @@ snapshots:
       base64-js: 1.5.1
       ieee754: 1.2.1
 
+  buffers@0.1.1: {}
+
   buildcheck@0.0.6:
     optional: true
 
@@ -18668,6 +18704,10 @@ snapshots:
       loupe: 3.1.1
       pathval: 2.0.0
 
+  chainsaw@0.1.0:
+    dependencies:
+      traverse: 0.3.9
+
   chalk@2.4.2:
     dependencies:
       ansi-styles: 3.2.1
@@ -25551,6 +25591,8 @@ snapshots:
     dependencies:
       yargs: 17.7.2
 
+  traverse@0.3.9: {}
+
   tree-kill@1.2.2: {}
 
   triple-beam@1.3.0: {}
@@ -25905,6 +25947,11 @@ snapshots:
 
   untildify@4.0.0: {}
 
+  unzip-stream@0.3.4:
+    dependencies:
+      binary: 0.3.0
+      mkdirp: 0.5.6
+
   update-browserslist-db@1.0.13(browserslist@4.23.0):
     dependencies:
       browserslist: 4.23.0