mirror of
https://github.com/n8n-io/n8n.git
synced 2025-03-05 20:50:17 -08:00
working import
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™
parent
9ca333735f
commit
e537dfd13d
|
|
@ -71,6 +71,7 @@
|
|||
"@types/superagent": "^8.1.7",
|
||||
"@types/swagger-ui-express": "^4.1.6",
|
||||
"@types/syslog-client": "^1.1.2",
|
||||
"@types/unzip-stream": "^0.3.4",
|
||||
"@types/uuid": "catalog:",
|
||||
"@types/validator": "^13.7.0",
|
||||
"@types/ws": "^8.5.4",
|
||||
|
|
@ -170,6 +171,7 @@
|
|||
"swagger-ui-express": "5.0.0",
|
||||
"syslog-client": "1.1.1",
|
||||
"typedi": "catalog:",
|
||||
"unzip-stream": "0.3.4",
|
||||
"uuid": "catalog:",
|
||||
"validator": "13.7.0",
|
||||
"winston": "3.8.2",
|
||||
|
|
|
|||
|
|
@ -2,35 +2,31 @@ import { Flags } from '@oclif/core';
|
|||
import { DataSource, MigrationExecutor } from '@n8n/typeorm';
|
||||
import * as assert from 'assert/strict';
|
||||
import fs from 'fs';
|
||||
import readline from 'readline';
|
||||
import { join } from 'path';
|
||||
import Container from 'typedi';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { Extract } from 'unzip-stream';
|
||||
|
||||
import { BaseCommand } from '../base-command';
|
||||
import { ApplicationError } from 'n8n-workflow';
|
||||
|
||||
// TODO: do this
|
||||
//const fs = require('fs');
|
||||
//const readline = require('readline');
|
||||
//
|
||||
//(async () => {
|
||||
// const fileStream = fs.createReadStream(__dirname + '/test.jsonl');
|
||||
// const lineStream = readline.createInterface({
|
||||
// input: fileStream,
|
||||
// crlfDelay: Infinity,
|
||||
// });
|
||||
//
|
||||
// for await (const line of lineStream) {
|
||||
// console.log(JSON.parse(line));
|
||||
// }
|
||||
//})();
|
||||
const excludeList = [
|
||||
'execution_annotation_tags',
|
||||
'execution_annotations',
|
||||
'execution_data',
|
||||
'execution_entity',
|
||||
'execution_metadata',
|
||||
'annotation_tag_entity',
|
||||
];
|
||||
|
||||
export class ImportAllCommand extends BaseCommand {
|
||||
static description = 'Import Everything';
|
||||
|
||||
static examples = ['$ n8n import:all', '$ n8n import:all --input=backup.zip'];
|
||||
|
||||
// TODO: add `clean` flag, or add a prompt to confirm
|
||||
// TODO: add `clean` flag, or add a prompt to confirm DB truncation
|
||||
static flags = {
|
||||
input: Flags.string({
|
||||
char: 'o',
|
||||
|
|
@ -41,6 +37,7 @@ export class ImportAllCommand extends BaseCommand {
|
|||
|
||||
// TODO: do batching
|
||||
async run() {
|
||||
const { flags } = await this.parse(ImportAllCommand);
|
||||
// TODO:
|
||||
// 1. check last migrations
|
||||
const connection = Container.get(DataSource);
|
||||
|
|
@ -50,10 +47,17 @@ export class ImportAllCommand extends BaseCommand {
|
|||
|
||||
assert.ok(lastExecutedMigration, 'should have been run by db.ts');
|
||||
|
||||
const zipPath = join(flags.input, 'n8n-backup.zip');
|
||||
if (!fs.existsSync(zipPath)) {
|
||||
throw new ApplicationError('Backup zip file not count');
|
||||
}
|
||||
|
||||
// TODO: instead of extracting to the filesystem, stream the files directly
|
||||
const backupPath = '/tmp/backup';
|
||||
await pipeline(fs.createReadStream(zipPath), Extract({ path: backupPath }));
|
||||
|
||||
const lastMigrationInBackup = (
|
||||
await fs.promises.readFile(join(backupPath, 'lastMigration'), 'utf8')
|
||||
await fs.promises.readFile(join(backupPath, '.lastMigration'), 'utf8')
|
||||
).trim();
|
||||
|
||||
if (lastMigrationInBackup !== lastExecutedMigration.name) {
|
||||
|
|
@ -65,14 +69,6 @@ export class ImportAllCommand extends BaseCommand {
|
|||
// 3. disable foreign keys
|
||||
|
||||
// 4. import each jsonl
|
||||
const excludeList = [
|
||||
'execution_annotation_tags',
|
||||
'execution_annotations',
|
||||
'execution_data',
|
||||
'execution_entity',
|
||||
'execution_metadata',
|
||||
'annotation_tag_entity',
|
||||
];
|
||||
const tables = connection.entityMetadatas
|
||||
.filter((v) => !excludeList.includes(v.tableName))
|
||||
.map((v) => ({ name: v.tableName, target: v.target }));
|
||||
|
|
@ -81,13 +77,20 @@ export class ImportAllCommand extends BaseCommand {
|
|||
const repo = connection.getRepository(target);
|
||||
await repo.delete({});
|
||||
|
||||
const rows = (await fs.promises.readFile(`${join(backupPath, name)}.jsonl`, 'utf8'))
|
||||
.split('\n')
|
||||
.filter((row) => row !== '');
|
||||
const filePath = join(backupPath, `${name}.jsonl`);
|
||||
if (!fs.existsSync(filePath)) continue;
|
||||
|
||||
for (const row of rows) {
|
||||
await repo.insert(JSON.parse(row));
|
||||
const fileStream = fs.createReadStream(filePath);
|
||||
const lineStream = readline.createInterface({
|
||||
input: fileStream,
|
||||
});
|
||||
|
||||
for await (const line of lineStream) {
|
||||
// TODO: insert in batches to reduce DB load
|
||||
await repo.insert(JSON.parse(line));
|
||||
}
|
||||
|
||||
fileStream.close();
|
||||
}
|
||||
|
||||
// 5. enable foreign keys
|
||||
|
|
|
|||
|
|
@ -935,6 +935,9 @@ importers:
|
|||
typedi:
|
||||
specifier: 'catalog:'
|
||||
version: 0.10.0(patch_hash=sk6omkefrosihg7lmqbzh7vfxe)
|
||||
unzip-stream:
|
||||
specifier: 0.3.4
|
||||
version: 0.3.4
|
||||
uuid:
|
||||
specifier: 'catalog:'
|
||||
version: 10.0.0
|
||||
|
|
@ -1023,6 +1026,9 @@ importers:
|
|||
'@types/syslog-client':
|
||||
specifier: ^1.1.2
|
||||
version: 1.1.2
|
||||
'@types/unzip-stream':
|
||||
specifier: ^0.3.4
|
||||
version: 0.3.4
|
||||
'@types/uuid':
|
||||
specifier: 'catalog:'
|
||||
version: 10.0.0
|
||||
|
|
@ -5192,6 +5198,9 @@ packages:
|
|||
'@types/unist@3.0.2':
|
||||
resolution: {integrity: sha512-dqId9J8K/vGi5Zr7oo212BGii5m3q5Hxlkwy3WpYuKPklmBEvsbMYYyLxAQpSffdLl/gdW0XUpKWFvYmyoWCoQ==}
|
||||
|
||||
'@types/unzip-stream@0.3.4':
|
||||
resolution: {integrity: sha512-ud0vtsNRF+joUCyvNMyo0j5DKX2Lh/im+xVgRzBEsfHhQYZ+i4fKTveova9XxLzt6Jl6G0e/0mM4aC0gqZYSnA==}
|
||||
|
||||
'@types/utf8@3.0.3':
|
||||
resolution: {integrity: sha512-+lqLGxWZsEe4Z6OrzBI7Ym4SMUTaMS5yOrHZ0/IL0bpIye1Qbs4PpobJL2mLDbftUXlPFZR7fu6d1yM+bHLX1w==}
|
||||
|
||||
|
|
@ -5948,6 +5957,9 @@ packages:
|
|||
resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
binary@0.3.0:
|
||||
resolution: {integrity: sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==}
|
||||
|
||||
binascii@0.0.2:
|
||||
resolution: {integrity: sha512-rA2CrUl1+6yKrn+XgLs8Hdy18OER1UW146nM+ixzhQXDY+Bd3ySkyIJGwF2a4I45JwbvF1mDL/nWkqBwpOcdBA==}
|
||||
|
||||
|
|
@ -6046,6 +6058,10 @@ packages:
|
|||
buffer@6.0.3:
|
||||
resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
|
||||
|
||||
buffers@0.1.1:
|
||||
resolution: {integrity: sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==}
|
||||
engines: {node: '>=0.2.0'}
|
||||
|
||||
buildcheck@0.0.6:
|
||||
resolution: {integrity: sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==}
|
||||
engines: {node: '>=10.0.0'}
|
||||
|
|
@ -6137,6 +6153,9 @@ packages:
|
|||
resolution: {integrity: sha512-pT1ZgP8rPNqUgieVaEY+ryQr6Q4HXNg8Ei9UnLUrjN4IA7dvQC5JB+/kxVcPNDHyBcc/26CXPkbNzq3qwrOEKA==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
chainsaw@0.1.0:
|
||||
resolution: {integrity: sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==}
|
||||
|
||||
chalk@2.4.2:
|
||||
resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==}
|
||||
engines: {node: '>=4'}
|
||||
|
|
@ -11725,6 +11744,9 @@ packages:
|
|||
engines: {node: '>=6.0.0'}
|
||||
hasBin: true
|
||||
|
||||
traverse@0.3.9:
|
||||
resolution: {integrity: sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==}
|
||||
|
||||
tree-kill@1.2.2:
|
||||
resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==}
|
||||
hasBin: true
|
||||
|
|
@ -12091,6 +12113,9 @@ packages:
|
|||
resolution: {integrity: sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
unzip-stream@0.3.4:
|
||||
resolution: {integrity: sha512-PyofABPVv+d7fL7GOpusx7eRT9YETY2X04PhwbSipdj6bMxVCFJrr+nm0Mxqbf9hUiTin/UsnuFWBXlDZFy0Cw==}
|
||||
|
||||
update-browserslist-db@1.0.13:
|
||||
resolution: {integrity: sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==}
|
||||
hasBin: true
|
||||
|
|
@ -17494,6 +17519,10 @@ snapshots:
|
|||
|
||||
'@types/unist@3.0.2': {}
|
||||
|
||||
'@types/unzip-stream@0.3.4':
|
||||
dependencies:
|
||||
'@types/node': 18.16.16
|
||||
|
||||
'@types/utf8@3.0.3': {}
|
||||
|
||||
'@types/uuencode@0.0.3(patch_hash=3i7wecddkama6vhpu5o37g24u4)':
|
||||
|
|
@ -18439,6 +18468,11 @@ snapshots:
|
|||
|
||||
binary-extensions@2.2.0: {}
|
||||
|
||||
binary@0.3.0:
|
||||
dependencies:
|
||||
buffers: 0.1.1
|
||||
chainsaw: 0.1.0
|
||||
|
||||
binascii@0.0.2: {}
|
||||
|
||||
bindings@1.5.0:
|
||||
|
|
@ -18547,6 +18581,8 @@ snapshots:
|
|||
base64-js: 1.5.1
|
||||
ieee754: 1.2.1
|
||||
|
||||
buffers@0.1.1: {}
|
||||
|
||||
buildcheck@0.0.6:
|
||||
optional: true
|
||||
|
||||
|
|
@ -18668,6 +18704,10 @@ snapshots:
|
|||
loupe: 3.1.1
|
||||
pathval: 2.0.0
|
||||
|
||||
chainsaw@0.1.0:
|
||||
dependencies:
|
||||
traverse: 0.3.9
|
||||
|
||||
chalk@2.4.2:
|
||||
dependencies:
|
||||
ansi-styles: 3.2.1
|
||||
|
|
@ -25551,6 +25591,8 @@ snapshots:
|
|||
dependencies:
|
||||
yargs: 17.7.2
|
||||
|
||||
traverse@0.3.9: {}
|
||||
|
||||
tree-kill@1.2.2: {}
|
||||
|
||||
triple-beam@1.3.0: {}
|
||||
|
|
@ -25905,6 +25947,11 @@ snapshots:
|
|||
|
||||
untildify@4.0.0: {}
|
||||
|
||||
unzip-stream@0.3.4:
|
||||
dependencies:
|
||||
binary: 0.3.0
|
||||
mkdirp: 0.5.6
|
||||
|
||||
update-browserslist-db@1.0.13(browserslist@4.23.0):
|
||||
dependencies:
|
||||
browserslist: 4.23.0
|
||||
|
|
|
|||
Loading…
Reference in a new issue