Merge branch 'n8n-io:master' into Add-schema-registry-into-kafka

.eslintrc.js

@@ -122,6 +122,8 @@ module.exports = {
 			'undefined',
 		],
 
+		'no-void': ['error', { 'allowAsStatement': true }],
+
 		// ----------------------------------
 		//        @typescript-eslint
 		// ----------------------------------
@@ -250,6 +252,11 @@ module.exports = {
 		 */
 		'@typescript-eslint/no-misused-promises': ['error', { checksVoidReturn: false }],
 
+		/**
+		 * https://github.com/typescript-eslint/typescript-eslint/blob/v4.30.0/packages/eslint-plugin/docs/rules/no-floating-promises.md
+		 */
+		'@typescript-eslint/no-floating-promises': ['error', { ignoreVoid: true }],
+
 		/**
 		 * https://eslint.org/docs/1.0.0/rules/no-throw-literal
 		 */

.gitignore

@@ -5,7 +5,6 @@ tmp
 dist
 npm-debug.log*
 lerna-debug.log
-package-lock.json
 yarn.lock
 google-generated-credentials.json
 _START_PACKAGE
@@ -15,3 +14,4 @@ _START_PACKAGE
 .idea
 vetur.config.js
 nodelinter.config.json
+packages/*/package-lock.json

.npmrc

@@ -0,0 +1 @@
+legacy-peer-deps=true

.prettierignore

@@ -1,3 +1,4 @@
 packages/nodes-base
 packages/editor-ui
 packages/design-system
+*package.json

CONTRIBUTING.md

@@ -49,6 +49,10 @@ dependencies are installed and the packages get linked correctly. Here a short g
 
 ### Requirements
 
+#### Node.js
+
+We suggest using the current [Node.js](https://nodejs.org/en/) LTS version (14.18.0 which includes npm 6.14.15) for development purposes.
+
 #### Build tools
 
 The packages which n8n uses depend on a few build tools:

SECURITY.md

@@ -0,0 +1,4 @@
+## Reporting a Vulnerability
+
+Please report (suspected) security vulnerabilities to **[security@n8n.io](mailto:security@n8n.io)**. You will receive a response from
+us within 48 hours. If the issue is confirmed, we will release a patch as soon as possible depending on complexity but historically within a few days.

docker/images/n8n-custom/Dockerfile

@@ -20,7 +20,7 @@ COPY packages/nodes-base/ ./packages/nodes-base/
 COPY packages/workflow/ ./packages/workflow/
 RUN rm -rf node_modules packages/*/node_modules packages/*/dist
 
-RUN npm install --production --loglevel notice
+RUN npm ci --production --loglevel notice
 RUN lerna bootstrap --hoist -- --production
 RUN npm run build
 

docker/images/n8n-custom/docker-entrypoint.sh

@@ -6,6 +6,8 @@ if [ -d /root/.n8n ] ; then
   ln -s /root/.n8n /home/node/
 fi
 
+chown -R node /home/node
+
 if [ "$#" -gt 0 ]; then
   # Got started with arguments
   COMMAND=$1;

packages/cli/commands/db/revert.ts

@@ -0,0 +1,61 @@
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+/* eslint-disable no-console */
+import { Command, flags } from '@oclif/command';
+import { Connection, ConnectionOptions, createConnection } from 'typeorm';
+import { LoggerProxy } from 'n8n-workflow';
+
+import { getLogger } from '../../src/Logger';
+
+import { Db } from '../../src';
+
+export class DbRevertMigrationCommand extends Command {
+	static description = 'Revert last database migration';
+
+	static examples = ['$ n8n db:revert'];
+
+	static flags = {
+		help: flags.help({ char: 'h' }),
+	};
+
+	// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+	async run() {
+		const logger = getLogger();
+		LoggerProxy.init(logger);
+
+		// eslint-disable-next-line @typescript-eslint/no-shadow, @typescript-eslint/no-unused-vars
+		const { flags } = this.parse(DbRevertMigrationCommand);
+
+		let connection: Connection | undefined;
+		try {
+			await Db.init();
+			connection = Db.collections.Credentials?.manager.connection;
+
+			if (!connection) {
+				throw new Error(`No database connection available.`);
+			}
+
+			const connectionOptions: ConnectionOptions = Object.assign(connection.options, {
+				subscribers: [],
+				synchronize: false,
+				migrationsRun: false,
+				dropSchema: false,
+				logging: ['query', 'error', 'schema'],
+			});
+
+			// close connection in order to reconnect with updated options
+			await connection.close();
+			connection = await createConnection(connectionOptions);
+
+			await connection.undoLastMigration();
+			await connection.close();
+		} catch (error) {
+			if (connection) await connection.close();
+
+			console.error('Error reverting last migration. See log messages for details.');
+			logger.error(error.message);
+			this.exit(1);
+		}
+
+		this.exit();
+	}
+}

packages/cli/commands/execute.ts

@@ -11,12 +11,11 @@ import {
 	CredentialTypes,
 	Db,
 	ExternalHooks,
+	InternalHooksManager,
 	IWorkflowBase,
 	IWorkflowExecutionDataProcess,
 	LoadNodesAndCredentials,
 	NodeTypes,
-	// eslint-disable-next-line @typescript-eslint/no-unused-vars
-	WorkflowCredentials,
 	WorkflowHelpers,
 	WorkflowRunner,
 } from '../src';
@@ -125,6 +124,9 @@ export class Execute extends Command {
 		const externalHooks = ExternalHooks();
 		await externalHooks.init();
 
+		const instanceId = await UserSettings.getInstanceId();
+		InternalHooksManager.init(instanceId);
+
 		// Add the found types to an instance other parts of the application can use
 		const nodeTypes = NodeTypes();
 		await nodeTypes.init(loadNodesAndCredentials.nodeTypes);

packages/cli/commands/executeBatch.ts

@@ -12,7 +12,7 @@ import { Command, flags } from '@oclif/command';
 import { UserSettings } from 'n8n-core';
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-import { INode, INodeExecutionData, ITaskData, LoggerProxy } from 'n8n-workflow';
+import { INode, ITaskData, LoggerProxy } from 'n8n-workflow';
 
 import { sep } from 'path';
 
@@ -28,14 +28,11 @@ import {
 	CredentialTypes,
 	Db,
 	ExternalHooks,
-	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	InternalHooksManager,
-	IExecutionsCurrentSummary,
 	IWorkflowDb,
 	IWorkflowExecutionDataProcess,
 	LoadNodesAndCredentials,
 	NodeTypes,
-	// eslint-disable-next-line @typescript-eslint/no-unused-vars
-	WorkflowCredentials,
 	WorkflowRunner,
 } from '../src';
 
@@ -59,12 +56,12 @@ export class ExecuteBatch extends Command {
 	static executionTimeout = 3 * 60 * 1000;
 
 	static examples = [
-		`$ n8n executeAll`,
+		`$ n8n executeBatch`,
-		`$ n8n executeAll --concurrency=10 --skipList=/data/skipList.txt`,
+		`$ n8n executeBatch --concurrency=10 --skipList=/data/skipList.txt`,
-		`$ n8n executeAll --debug --output=/data/output.json`,
+		`$ n8n executeBatch --debug --output=/data/output.json`,
-		`$ n8n executeAll --ids=10,13,15 --shortOutput`,
+		`$ n8n executeBatch --ids=10,13,15 --shortOutput`,
-		`$ n8n executeAll --snapshot=/data/snapshots --shallow`,
+		`$ n8n executeBatch --snapshot=/data/snapshots --shallow`,
-		`$ n8n executeAll --compare=/data/previousExecutionData --retries=2`,
+		`$ n8n executeBatch --compare=/data/previousExecutionData --retries=2`,
 	];
 
 	static flags = {
@@ -307,6 +304,9 @@ export class ExecuteBatch extends Command {
 		const externalHooks = ExternalHooks();
 		await externalHooks.init();
 
+		const instanceId = await UserSettings.getInstanceId();
+		InternalHooksManager.init(instanceId);
+
 		// Add the found types to an instance other parts of the application can use
 		const nodeTypes = NodeTypes();
 		await nodeTypes.init(loadNodesAndCredentials.nodeTypes);
@@ -817,10 +817,22 @@ export class ExecuteBatch extends Command {
 								const changes = diff(JSON.parse(contents), data, { keysOnly: true });
 
 								if (changes !== undefined) {
-									// we have structural changes. Report them.
+									// If we had only additions with no removals
-									executionResult.error = `Workflow may contain breaking changes`;
+									// Then we treat as a warning and not an error.
-									executionResult.changes = changes;
+									// To find this, we convert the object to JSON
-									executionResult.executionStatus = 'error';
+									// and search for the `__deleted` string
+									const changesJson = JSON.stringify(changes);
+									if (changesJson.includes('__deleted')) {
+										// we have structural changes. Report them.
+										executionResult.error = 'Workflow may contain breaking changes';
+										executionResult.changes = changes;
+										executionResult.executionStatus = 'error';
+									} else {
+										executionResult.error =
+											'Workflow contains new data that previously did not exist.';
+										executionResult.changes = changes;
+										executionResult.executionStatus = 'warning';
+									}
 								} else {
 									executionResult.executionStatus = 'success';
 								}

packages/cli/commands/export/credentials.ts

@@ -129,7 +129,8 @@ export class ExportCredentialsCommand extends Command {
 
 				for (let i = 0; i < credentials.length; i++) {
 					const { name, type, nodesAccess, data } = credentials[i];
-					const credential = new Credentials(name, type, nodesAccess, data);
+					const id = credentials[i].id as string;
+					const credential = new Credentials({ id, name }, type, nodesAccess, data);
 					const plainData = credential.getData(encryptionKey);
 					(credentials[i] as ICredentialsDecryptedDb).data = plainData;
 				}

packages/cli/commands/import/workflow.ts

@@ -2,14 +2,13 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import { Command, flags } from '@oclif/command';
 
-import { LoggerProxy } from 'n8n-workflow';
+import { INode, INodeCredentialsDetails, LoggerProxy } from 'n8n-workflow';
 
 import * as fs from 'fs';
 import * as glob from 'fast-glob';
-import * as path from 'path';
 import { UserSettings } from 'n8n-core';
 import { getLogger } from '../../src/Logger';
-import { Db } from '../../src';
+import { Db, ICredentialsDb } from '../../src';
 
 export class ImportWorkflowsCommand extends Command {
 	static description = 'Import workflows';
@@ -30,6 +29,32 @@ export class ImportWorkflowsCommand extends Command {
 		}),
 	};
 
+	private transformCredentials(node: INode, credentialsEntities: ICredentialsDb[]) {
+		if (node.credentials) {
+			const allNodeCredentials = Object.entries(node.credentials);
+			// eslint-disable-next-line no-restricted-syntax
+			for (const [type, name] of allNodeCredentials) {
+				if (typeof name === 'string') {
+					const nodeCredentials: INodeCredentialsDetails = {
+						id: null,
+						name,
+					};
+
+					const matchingCredentials = credentialsEntities.filter(
+						(credentials) => credentials.name === name && credentials.type === type,
+					);
+
+					if (matchingCredentials.length === 1) {
+						nodeCredentials.id = matchingCredentials[0].id.toString();
+					}
+
+					// eslint-disable-next-line no-param-reassign
+					node.credentials[type] = nodeCredentials;
+				}
+			}
+		}
+	}
+
 	// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
 	async run() {
 		const logger = getLogger();
@@ -57,13 +82,23 @@ export class ImportWorkflowsCommand extends Command {
 
 			// Make sure the settings exist
 			await UserSettings.prepareUserSettings();
+			const credentialsEntities = (await Db.collections.Credentials?.find()) ?? [];
 			let i;
 			if (flags.separate) {
-				const files = await glob(
+				let inputPath = flags.input;
-					`${flags.input.endsWith(path.sep) ? flags.input : flags.input + path.sep}*.json`,
+				if (process.platform === 'win32') {
-				);
+					inputPath = inputPath.replace(/\\/g, '/');
+				}
+				inputPath = inputPath.replace(/\/$/g, '');
+				const files = await glob(`${inputPath}/*.json`);
 				for (i = 0; i < files.length; i++) {
 					const workflow = JSON.parse(fs.readFileSync(files[i], { encoding: 'utf8' }));
+					if (credentialsEntities.length > 0) {
+						// eslint-disable-next-line
+						workflow.nodes.forEach((node: INode) => {
+							this.transformCredentials(node, credentialsEntities);
+						});
+					}
 					// eslint-disable-next-line no-await-in-loop, @typescript-eslint/no-non-null-assertion
 					await Db.collections.Workflow!.save(workflow);
 				}
@@ -75,6 +110,12 @@ export class ImportWorkflowsCommand extends Command {
 				}
 
 				for (i = 0; i < fileContents.length; i++) {
+					if (credentialsEntities.length > 0) {
+						// eslint-disable-next-line
+						fileContents[i].nodes.forEach((node: INode) => {
+							this.transformCredentials(node, credentialsEntities);
+						});
+					}
 					// eslint-disable-next-line no-await-in-loop, @typescript-eslint/no-non-null-assertion
 					await Db.collections.Workflow!.save(fileContents[i]);
 				}

packages/cli/commands/start.ts

@@ -22,8 +22,7 @@ import {
 	Db,
 	ExternalHooks,
 	GenericHelpers,
-	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	InternalHooksManager,
-	IExecutionsCurrentSummary,
 	LoadNodesAndCredentials,
 	NodeTypes,
 	Server,
@@ -37,7 +36,7 @@ import { getLogger } from '../src/Logger';
 const open = require('open');
 
 let activeWorkflowRunner: ActiveWorkflowRunner.ActiveWorkflowRunner | undefined;
-let processExistCode = 0;
+let processExitCode = 0;
 
 export class Start extends Command {
 	static description = 'Starts n8n. Makes Web-UI available and starts active workflows';
@@ -92,9 +91,12 @@ export class Start extends Command {
 			setTimeout(() => {
 				// In case that something goes wrong with shutdown we
 				// kill after max. 30 seconds no matter what
-				process.exit(processExistCode);
+				console.log(`process exited after 30s`);
+				process.exit(processExitCode);
 			}, 30000);
 
+			await InternalHooksManager.getInstance().onN8nStop();
+
 			const skipWebhookDeregistration = config.get(
 				'endpoints.skipWebhoooksDeregistrationOnShutdown',
 			) as boolean;
@@ -133,7 +135,7 @@ export class Start extends Command {
 			console.error('There was an error shutting down n8n.', error);
 		}
 
-		process.exit(processExistCode);
+		process.exit(processExitCode);
 	}
 
 	async run() {
@@ -151,16 +153,22 @@ export class Start extends Command {
 				LoggerProxy.init(logger);
 				logger.info('Initializing n8n process');
 
-				// todo remove a few versions after release
 				logger.info(
-					'\nn8n now checks for new versions and security updates. You can turn this off using the environment variable N8N_VERSION_NOTIFICATIONS_ENABLED to "false"\nFor more information, please refer to https://docs.n8n.io/getting-started/installation/advanced/configuration.html\n',
+					'\n' +
+						'****************************************************\n' +
+						'*                                                  *\n' +
+						'*   n8n now sends selected, anonymous telemetry.   *\n' +
+						'*      For more details (and how to opt out):      *\n' +
+						'*   https://docs.n8n.io/reference/telemetry.html   *\n' +
+						'*                                                  *\n' +
+						'****************************************************\n',
 				);
 
 				// Start directly with the init of the database to improve startup time
 				const startDbInitPromise = Db.init().catch((error: Error) => {
 					logger.error(`There was an error initializing DB: "${error.message}"`);
 
-					processExistCode = 1;
+					processExitCode = 1;
 					// @ts-ignore
 					process.emit('SIGINT');
 					process.exit(1);
@@ -173,10 +181,6 @@ export class Start extends Command {
 				const loadNodesAndCredentials = LoadNodesAndCredentials();
 				await loadNodesAndCredentials.init();
 
-				// Load the credentials overwrites if any exist
-				const credentialsOverwrites = CredentialsOverwrites();
-				await credentialsOverwrites.init();
-
 				// Load all external hooks
 				const externalHooks = ExternalHooks();
 				await externalHooks.init();
@@ -187,6 +191,10 @@ export class Start extends Command {
 				const credentialTypes = CredentialTypes();
 				await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
 
+				// Load the credentials overwrites if any exist
+				const credentialsOverwrites = CredentialsOverwrites();
+				await credentialsOverwrites.init();
+
 				// Wait till the database is ready
 				await startDbInitPromise;
 
@@ -304,14 +312,16 @@ export class Start extends Command {
 					);
 				}
 
+				const instanceId = await UserSettings.getInstanceId();
+				InternalHooksManager.init(instanceId);
+
 				await Server.start();
 
 				// Start to get active workflows and run their triggers
 				activeWorkflowRunner = ActiveWorkflowRunner.getInstance();
 				await activeWorkflowRunner.init();
 
-				// eslint-disable-next-line @typescript-eslint/no-unused-vars
+				WaitTracker();
-				const waitTracker = WaitTracker();
 
 				const editorUrl = GenericHelpers.getBaseUrl();
 				this.log(`\nEditor is now accessible via:\n${editorUrl}`);
@@ -355,7 +365,7 @@ export class Start extends Command {
 				// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
 				this.error(`There was an error: ${error.message}`);
 
-				processExistCode = 1;
+				processExitCode = 1;
 				// @ts-ignore
 				process.emit('SIGINT');
 			}

packages/cli/commands/update/workflow.ts

@@ -4,8 +4,7 @@ import { Command, flags } from '@oclif/command';
 
 import { IDataObject, LoggerProxy } from 'n8n-workflow';
 
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
+import { Db } from '../../src';
-import { Db, GenericHelpers } from '../../src';
 
 import { getLogger } from '../../src/Logger';
 

packages/cli/commands/webhook.ts

@@ -18,10 +18,9 @@ import {
 	Db,
 	ExternalHooks,
 	GenericHelpers,
+	InternalHooksManager,
 	LoadNodesAndCredentials,
 	NodeTypes,
-	// eslint-disable-next-line @typescript-eslint/no-unused-vars
-	TestWebhooks,
 	WebhookServer,
 } from '../src';
 
@@ -149,6 +148,9 @@ export class Webhook extends Command {
 				// Wait till the database is ready
 				await startDbInitPromise;
 
+				const instanceId = await UserSettings.getInstanceId();
+				InternalHooksManager.init(instanceId);
+
 				if (config.get('executions.mode') === 'queue') {
 					const redisHost = config.get('queue.bull.redis.host');
 					const redisPassword = config.get('queue.bull.redis.password');

packages/cli/commands/worker.ts

@@ -12,21 +12,12 @@ import * as PCancelable from 'p-cancelable';
 import { Command, flags } from '@oclif/command';
 import { UserSettings, WorkflowExecute } from 'n8n-core';
 
-import {
+import { IExecuteResponsePromiseData, INodeTypes, IRun, Workflow, LoggerProxy } from 'n8n-workflow';
-	IDataObject,
-	INodeTypes,
-	IRun,
-	IWorkflowExecuteHooks,
-	Workflow,
-	WorkflowHooks,
-	LoggerProxy,
-} from 'n8n-workflow';
 
 import { FindOneOptions } from 'typeorm';
 
 import * as Bull from 'bull';
 import {
-	ActiveExecutions,
 	CredentialsOverwrites,
 	CredentialTypes,
 	Db,
@@ -34,12 +25,13 @@ import {
 	GenericHelpers,
 	IBullJobData,
 	IBullJobResponse,
+	IBullWebhookResponse,
 	IExecutionFlattedDb,
-	IExecutionResponse,
+	InternalHooksManager,
 	LoadNodesAndCredentials,
 	NodeTypes,
 	ResponseHelper,
-	WorkflowCredentials,
+	WebhookHelpers,
 	WorkflowExecuteAdditionalData,
 } from '../src';
 
@@ -182,6 +174,16 @@ export class Worker extends Command {
 			currentExecutionDb.workflowData,
 			{ retryOf: currentExecutionDb.retryOf as string },
 		);
+
+		additionalData.hooks.hookFunctions.sendResponse = [
+			async (response: IExecuteResponsePromiseData): Promise<void> => {
+				await job.progress({
+					executionId: job.data.executionId as string,
+					response: WebhookHelpers.encodeWebhookResponse(response),
+				} as IBullWebhookResponse);
+			},
+		];
+
 		additionalData.executionId = jobData.executionId;
 
 		let workflowExecute: WorkflowExecute;
@@ -203,7 +205,7 @@ export class Worker extends Command {
 		Worker.runningJobs[job.id] = workflowRun;
 
 		// Wait till the execution is finished
-		const runData = await workflowRun;
+		await workflowRun;
 
 		delete Worker.runningJobs[job.id];
 
@@ -269,6 +271,9 @@ export class Worker extends Command {
 				// eslint-disable-next-line @typescript-eslint/no-floating-promises
 				Worker.jobQueue.process(flags.concurrency, async (job) => this.runJob(job, nodeTypes));
 
+				const instanceId = await UserSettings.getInstanceId();
+				InternalHooksManager.init(instanceId);
+
 				const versions = await GenericHelpers.getVersions();
 
 				console.info('\nn8n worker is now ready');

packages/cli/config/index.ts

@@ -649,6 +649,46 @@ const config = convict({
 			env: 'N8N_VERSION_NOTIFICATIONS_INFO_URL',
 		},
 	},
+
+	deployment: {
+		type: {
+			format: String,
+			default: 'default',
+			env: 'N8N_DEPLOYMENT_TYPE',
+		},
+	},
+
+	personalization: {
+		enabled: {
+			doc: 'Whether personalization is enabled.',
+			format: Boolean,
+			default: true,
+			env: 'N8N_PERSONALIZATION_ENABLED',
+		},
+	},
+
+	diagnostics: {
+		enabled: {
+			doc: 'Whether diagnostic mode is enabled.',
+			format: Boolean,
+			default: true,
+			env: 'N8N_DIAGNOSTICS_ENABLED',
+		},
+		config: {
+			frontend: {
+				doc: 'Diagnostics config for frontend.',
+				format: String,
+				default: '1zPn9bgWPzlQc0p8Gj1uiK6DOTn;https://telemetry.n8n.io',
+				env: 'N8N_DIAGNOSTICS_CONFIG_FRONTEND',
+			},
+			backend: {
+				doc: 'Diagnostics config for backend.',
+				format: String,
+				default: '1zPn7YoGC3ZXE9zLeTKLuQCB4F6;https://telemetry.n8n.io/v1/batch',
+				env: 'N8N_DIAGNOSTICS_CONFIG_BACKEND',
+			},
+		},
+	},
 });
 
 // Overwrite default configuration with settings which got defined in

packages/cli/migrations/ormconfig.ts

@@ -9,7 +9,7 @@ module.exports = [
 		logging: true,
 		entities: Object.values(entities),
 		database: path.join(UserSettings.getUserN8nFolderPath(), 'database.sqlite'),
-		migrations: ['./src/databases/sqlite/migrations/*.ts'],
+		migrations: ['./src/databases/sqlite/migrations/index.ts'],
 		subscribers: ['./src/databases/sqlite/subscribers/*.ts'],
 		cli: {
 			entitiesDir: './src/databases/entities',
@@ -28,7 +28,7 @@ module.exports = [
 		database: 'n8n',
 		schema: 'public',
 		entities: Object.values(entities),
-		migrations: ['./src/databases/postgresdb/migrations/*.ts'],
+		migrations: ['./src/databases/postgresdb/migrations/index.ts'],
 		subscribers: ['src/subscriber/**/*.ts'],
 		cli: {
 			entitiesDir: './src/databases/entities',
@@ -46,7 +46,7 @@ module.exports = [
 		port: '3306',
 		logging: false,
 		entities: Object.values(entities),
-		migrations: ['./src/databases/mysqldb/migrations/*.ts'],
+		migrations: ['./src/databases/mysqldb/migrations/index.ts'],
 		subscribers: ['src/subscriber/**/*.ts'],
 		cli: {
 			entitiesDir: './src/databases/entities',

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "n8n",
-  "version": "0.139.1",
+  "version": "0.151.0",
   "description": "n8n Workflow Automation Tool",
   "license": "SEE LICENSE IN LICENSE.md",
   "homepage": "https://n8n.io",
@@ -31,7 +31,7 @@
     "start:windows": "cd bin && n8n",
     "test": "jest",
     "watch": "tsc --watch",
-    "typeorm": "ts-node ./node_modules/typeorm/cli.js"
+    "typeorm": "ts-node ../../node_modules/typeorm/cli.js"
   },
   "bin": {
     "n8n": "./bin/n8n"
@@ -66,10 +66,11 @@
     "@types/jest": "^26.0.13",
     "@types/localtunnel": "^1.9.0",
     "@types/lodash.get": "^4.4.6",
-    "@types/node": "^14.14.40",
+    "@types/node": "14.17.27",
     "@types/open": "^6.1.0",
     "@types/parseurl": "^1.3.1",
     "@types/request-promise-native": "~1.0.15",
+    "@types/validator": "^13.7.0",
     "concurrently": "^5.1.0",
     "jest": "^26.4.2",
     "nodemon": "^2.0.2",
@@ -83,6 +84,7 @@
   "dependencies": {
     "@oclif/command": "^1.5.18",
     "@oclif/errors": "^1.2.2",
+    "@rudderstack/rudder-sdk-node": "1.0.6",
     "@types/json-diff": "^0.5.1",
     "@types/jsonwebtoken": "^8.5.2",
     "basic-auth": "^2.0.1",
@@ -109,10 +111,10 @@
     "localtunnel": "^2.0.0",
     "lodash.get": "^4.4.2",
     "mysql2": "~2.3.0",
-    "n8n-core": "~0.84.0",
+    "n8n-core": "~0.95.0",
-    "n8n-editor-ui": "~0.107.1",
+    "n8n-editor-ui": "~0.118.0",
-    "n8n-nodes-base": "~0.136.0",
+    "n8n-nodes-base": "~0.148.0",
-    "n8n-workflow": "~0.70.0",
+    "n8n-workflow": "~0.78.0",
     "oauth-1.0a": "^2.2.6",
     "open": "^7.0.0",
     "pg": "^8.3.0",

packages/cli/src/ActiveExecutions.ts

@@ -5,9 +5,12 @@
 /* eslint-disable @typescript-eslint/no-unsafe-call */
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import { IRun } from 'n8n-workflow';
+import {
-
+	createDeferredPromise,
-import { createDeferredPromise } from 'n8n-core';
+	IDeferredPromise,
+	IExecuteResponsePromiseData,
+	IRun,
+} from 'n8n-workflow';
 
 import { ChildProcess } from 'child_process';
 // eslint-disable-next-line import/no-extraneous-dependencies
@@ -116,6 +119,28 @@ export class ActiveExecutions {
 		this.activeExecutions[executionId].workflowExecution = workflowExecution;
 	}
 
+	attachResponsePromise(
+		executionId: string,
+		responsePromise: IDeferredPromise<IExecuteResponsePromiseData>,
+	): void {
+		if (this.activeExecutions[executionId] === undefined) {
+			throw new Error(
+				`No active execution with id "${executionId}" got found to attach to workflowExecution to!`,
+			);
+		}
+
+		this.activeExecutions[executionId].responsePromise = responsePromise;
+	}
+
+	resolveResponsePromise(executionId: string, response: IExecuteResponsePromiseData): void {
+		if (this.activeExecutions[executionId] === undefined) {
+			return;
+		}
+
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+		this.activeExecutions[executionId].responsePromise?.resolve(response);
+	}
+
 	/**
 	 * Remove an active execution
 	 *
@@ -193,6 +218,7 @@ export class ActiveExecutions {
 
 		this.activeExecutions[executionId].postExecutePromises.push(waitPromise);
 
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access
 		return waitPromise.promise();
 	}
 

packages/cli/src/ActiveWorkflowRunner.ts

@@ -12,7 +12,9 @@
 import { ActiveWorkflows, NodeExecuteFunctions } from 'n8n-core';
 
 import {
+	IDeferredPromise,
 	IExecuteData,
+	IExecuteResponsePromiseData,
 	IGetExecutePollFunctions,
 	IGetExecuteTriggerFunctions,
 	INode,
@@ -40,8 +42,6 @@ import {
 	NodeTypes,
 	ResponseHelper,
 	WebhookHelpers,
-	// eslint-disable-next-line @typescript-eslint/no-unused-vars
-	WorkflowCredentials,
 	WorkflowExecuteAdditionalData,
 	WorkflowHelpers,
 	WorkflowRunner,
@@ -550,6 +550,7 @@ export class ActiveWorkflowRunner {
 		data: INodeExecutionData[][],
 		additionalData: IWorkflowExecuteAdditionalDataWorkflow,
 		mode: WorkflowExecuteMode,
+		responsePromise?: IDeferredPromise<IExecuteResponsePromiseData>,
 	) {
 		const nodeExecutionStack: IExecuteData[] = [
 			{
@@ -580,7 +581,7 @@ export class ActiveWorkflowRunner {
 		};
 
 		const workflowRunner = new WorkflowRunner();
-		return workflowRunner.run(runData, true);
+		return workflowRunner.run(runData, true, undefined, undefined, responsePromise);
 	}
 
 	/**
@@ -641,13 +642,16 @@ export class ActiveWorkflowRunner {
 				mode,
 				activation,
 			);
-			returnFunctions.emit = (data: INodeExecutionData[][]): void => {
+			returnFunctions.emit = (
+				data: INodeExecutionData[][],
+				responsePromise?: IDeferredPromise<IExecuteResponsePromiseData>,
+			): void => {
 				// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
 				Logger.debug(`Received trigger for workflow "${workflow.name}"`);
 				WorkflowHelpers.saveStaticData(workflow);
 				// eslint-disable-next-line id-denylist
-				this.runWorkflow(workflowData, node, data, additionalData, mode).catch((err) =>
+				this.runWorkflow(workflowData, node, data, additionalData, mode, responsePromise).catch(
-					console.error(err),
+					(error) => console.error(error),
 				);
 			};
 			return returnFunctions;

packages/cli/src/CredentialTypes.ts

@@ -1,31 +1,13 @@
 import { ICredentialType, ICredentialTypes as ICredentialTypesInterface } from 'n8n-workflow';
 
 // eslint-disable-next-line import/no-cycle
-import { CredentialsOverwrites, ICredentialsTypeData } from '.';
+import { ICredentialsTypeData } from '.';
 
 class CredentialTypesClass implements ICredentialTypesInterface {
 	credentialTypes: ICredentialsTypeData = {};
 
 	async init(credentialTypes: ICredentialsTypeData): Promise<void> {
 		this.credentialTypes = credentialTypes;
-
-		// Load the credentials overwrites if any exist
-		const credentialsOverwrites = CredentialsOverwrites().getAll();
-
-		// eslint-disable-next-line no-restricted-syntax
-		for (const credentialType of Object.keys(credentialsOverwrites)) {
-			if (credentialTypes[credentialType] === undefined) {
-				// eslint-disable-next-line no-continue
-				continue;
-			}
-
-			// Add which properties got overwritten that the Editor-UI knows
-			// which properties it should hide
-			// eslint-disable-next-line no-underscore-dangle, no-param-reassign
-			credentialTypes[credentialType].__overwrittenProperties = Object.keys(
-				credentialsOverwrites[credentialType],
-			);
-		}
 	}
 
 	getAll(): ICredentialType[] {

packages/cli/src/CredentialsHelper.ts

@@ -5,6 +5,7 @@ import {
 	ICredentialsExpressionResolveValues,
 	ICredentialsHelper,
 	INode,
+	INodeCredentialsDetails,
 	INodeParameters,
 	INodeProperties,
 	INodeType,
@@ -39,30 +40,32 @@ export class CredentialsHelper extends ICredentialsHelper {
 	/**
 	 * Returns the credentials instance
 	 *
-	 * @param {string} name Name of the credentials to return instance of
+	 * @param {INodeCredentialsDetails} nodeCredentials id and name to return instance of
 	 * @param {string} type Type of the credentials to return instance of
 	 * @returns {Credentials}
 	 * @memberof CredentialsHelper
 	 */
-	async getCredentials(name: string, type: string): Promise<Credentials> {
+	async getCredentials(
-		const credentialsDb = await Db.collections.Credentials?.find({ type });
+		nodeCredentials: INodeCredentialsDetails,
-
+		type: string,
-		if (credentialsDb === undefined || credentialsDb.length === 0) {
+	): Promise<Credentials> {
-			throw new Error(`No credentials of type "${type}" exist.`);
+		if (!nodeCredentials.id) {
+			throw new Error(`Credentials "${nodeCredentials.name}" for type "${type}" don't have an ID.`);
 		}
 
-		// eslint-disable-next-line @typescript-eslint/no-shadow
+		const credentials = await Db.collections.Credentials?.findOne({ id: nodeCredentials.id, type });
-		const credential = credentialsDb.find((credential) => credential.name === name);
 
-		if (credential === undefined) {
+		if (!credentials) {
-			throw new Error(`No credentials with name "${name}" exist for type "${type}".`);
+			throw new Error(
+				`Credentials with ID "${nodeCredentials.id}" don't exist for type "${type}".`,
+			);
 		}
 
 		return new Credentials(
-			credential.name,
+			{ id: credentials.id.toString(), name: credentials.name },
-			credential.type,
+			credentials.type,
-			credential.nodesAccess,
+			credentials.nodesAccess,
-			credential.data,
+			credentials.data,
 		);
 	}
 
@@ -101,21 +104,20 @@ export class CredentialsHelper extends ICredentialsHelper {
 	/**
 	 * Returns the decrypted credential data with applied overwrites
 	 *
-	 * @param {string} name Name of the credentials to return data of
+	 * @param {INodeCredentialsDetails} nodeCredentials id and name to return instance of
 	 * @param {string} type Type of the credentials to return data of
 	 * @param {boolean} [raw] Return the data as supplied without defaults or overwrites
 	 * @returns {ICredentialDataDecryptedObject}
 	 * @memberof CredentialsHelper
 	 */
 	async getDecrypted(
-		name: string,
+		nodeCredentials: INodeCredentialsDetails,
 		type: string,
 		mode: WorkflowExecuteMode,
 		raw?: boolean,
 		expressionResolveValues?: ICredentialsExpressionResolveValues,
 	): Promise<ICredentialDataDecryptedObject> {
-		const credentials = await this.getCredentials(name, type);
+		const credentials = await this.getCredentials(nodeCredentials, type);
-
 		const decryptedDataOriginal = credentials.getData(this.encryptionKey);
 
 		if (raw === true) {
@@ -228,12 +230,12 @@ export class CredentialsHelper extends ICredentialsHelper {
 	 * @memberof CredentialsHelper
 	 */
 	async updateCredentials(
-		name: string,
+		nodeCredentials: INodeCredentialsDetails,
 		type: string,
 		data: ICredentialDataDecryptedObject,
 	): Promise<void> {
 		// eslint-disable-next-line @typescript-eslint/await-thenable
-		const credentials = await this.getCredentials(name, type);
+		const credentials = await this.getCredentials(nodeCredentials, type);
 
 		if (Db.collections.Credentials === null) {
 			// The first time executeWorkflow gets called the Database has
@@ -251,7 +253,7 @@ export class CredentialsHelper extends ICredentialsHelper {
 
 		// Save the credentials in DB
 		const findQuery = {
-			name,
+			id: credentials.id,
 			type,
 		};
 

packages/cli/src/CredentialsOverwrites.ts

@@ -12,6 +12,9 @@ class CredentialsOverwritesClass {
 	private resolvedTypes: string[] = [];
 
 	async init(overwriteData?: ICredentialsOverwrite) {
+		// If data gets reinitialized reset the resolved types cache
+		this.resolvedTypes.length = 0;
+
 		if (overwriteData !== undefined) {
 			// If data is already given it can directly be set instead of
 			// loaded from environment
@@ -41,6 +44,7 @@ class CredentialsOverwritesClass {
 
 			if (overwrites && Object.keys(overwrites).length) {
 				this.overwriteData[type] = overwrites;
+				credentialTypeData.__overwrittenProperties = Object.keys(overwrites);
 			}
 		}
 	}

packages/cli/src/GenericHelpers.ts

@@ -7,7 +7,6 @@
 import * as express from 'express';
 import { join as pathJoin } from 'path';
 import { readFile as fsReadFile } from 'fs/promises';
-import { readFileSync as fsReadFileSync } from 'fs';
 import { IDataObject } from 'n8n-workflow';
 import * as config from '../config';
 
@@ -137,45 +136,6 @@ export async function getConfigValue(
 	return data;
 }
 
-/**
- * Gets value from config with support for "_FILE" environment variables synchronously
- *
- * @export
- * @param {string} configKey The key of the config data to get
- * @returns {(string | boolean | number | undefined)}
- */
-export function getConfigValueSync(configKey: string): string | boolean | number | undefined {
-	// Get the environment variable
-	const configSchema = config.getSchema();
-	// @ts-ignore
-	const currentSchema = extractSchemaForKey(configKey, configSchema._cvtProperties as IDataObject);
-	// Check if environment variable is defined for config key
-	if (currentSchema.env === undefined) {
-		// No environment variable defined, so return value from config
-		return config.get(configKey);
-	}
-
-	// Check if special file enviroment variable exists
-	const fileEnvironmentVariable = process.env[`${currentSchema.env}_FILE`];
-	if (fileEnvironmentVariable === undefined) {
-		// Does not exist, so return value from config
-		return config.get(configKey);
-	}
-
-	let data;
-	try {
-		data = fsReadFileSync(fileEnvironmentVariable, 'utf8');
-	} catch (error) {
-		if (error.code === 'ENOENT') {
-			throw new Error(`The file "${fileEnvironmentVariable}" could not be found.`);
-		}
-
-		throw error;
-	}
-
-	return data;
-}
-
 /**
  * Generate a unique name for a workflow or credentials entity.
  *

packages/cli/src/Interfaces.ts

@@ -7,18 +7,19 @@ import {
 	ICredentialsEncrypted,
 	ICredentialType,
 	IDataObject,
+	IDeferredPromise,
+	IExecuteResponsePromiseData,
 	IRun,
 	IRunData,
 	IRunExecutionData,
 	ITaskData,
+	ITelemetrySettings,
 	IWorkflowBase as IWorkflowBaseWorkflow,
-	// eslint-disable-next-line @typescript-eslint/no-unused-vars
-	IWorkflowCredentials,
 	Workflow,
 	WorkflowExecuteMode,
 } from 'n8n-workflow';
 
-import { IDeferredPromise, WorkflowExecute } from 'n8n-core';
+import { WorkflowExecute } from 'n8n-core';
 
 // eslint-disable-next-line import/no-extraneous-dependencies
 import * as PCancelable from 'p-cancelable';
@@ -46,6 +47,11 @@ export interface IBullJobResponse {
 	success: boolean;
 }
 
+export interface IBullWebhookResponse {
+	executionId: string;
+	response: IExecuteResponsePromiseData;
+}
+
 export interface ICustomRequest extends Request {
 	parsedUrl: Url | undefined;
 }
@@ -236,6 +242,7 @@ export interface IExecutingWorkflowData {
 	process?: ChildProcess;
 	startedAt: Date;
 	postExecutePromises: Array<IDeferredPromise<IRun | undefined>>;
+	responsePromise?: IDeferredPromise<IExecuteResponsePromiseData>;
 	workflowExecution?: PCancelable<IRun>;
 }
 
@@ -281,6 +288,40 @@ export interface IExternalHooksClass {
 	run(hookName: string, hookParameters?: any[]): Promise<void>;
 }
 
+export interface IDiagnosticInfo {
+	versionCli: string;
+	databaseType: DatabaseType;
+	notificationsEnabled: boolean;
+	disableProductionWebhooksOnMainProcess: boolean;
+	basicAuthActive: boolean;
+	systemInfo: {
+		os: {
+			type?: string;
+			version?: string;
+		};
+		memory?: number;
+		cpus: {
+			count?: number;
+			model?: string;
+			speed?: number;
+		};
+	};
+	executionVariables: {
+		[key: string]: string | number | undefined;
+	};
+	deploymentType: string;
+}
+
+export interface IInternalHooksClass {
+	onN8nStop(): Promise<void>;
+	onServerStarted(diagnosticInfo: IDiagnosticInfo): Promise<unknown[]>;
+	onPersonalizationSurveySubmitted(answers: IPersonalizationSurveyAnswers): Promise<void>;
+	onWorkflowCreated(workflow: IWorkflowBase): Promise<void>;
+	onWorkflowDeleted(workflowId: string): Promise<void>;
+	onWorkflowSaved(workflow: IWorkflowBase): Promise<void>;
+	onWorkflowPostExecute(workflow: IWorkflowBase, runData?: IRun): Promise<void>;
+}
+
 export interface IN8nConfig {
 	database: IN8nConfigDatabase;
 	endpoints: IN8nConfigEndpoints;
@@ -357,6 +398,20 @@ export interface IN8nUISettings {
 	};
 	versionNotifications: IVersionNotificationSettings;
 	instanceId: string;
+	telemetry: ITelemetrySettings;
+	personalizationSurvey: IPersonalizationSurvey;
+}
+
+export interface IPersonalizationSurveyAnswers {
+	companySize: string | null;
+	codingSkill: string | null;
+	workArea: string | null;
+	otherWorkArea: string | null;
+}
+
+export interface IPersonalizationSurvey {
+	answers?: IPersonalizationSurveyAnswers;
+	shouldShow: boolean;
 }
 
 export interface IPackageVersions {
@@ -441,6 +496,7 @@ export interface IPushDataConsoleMessage {
 
 export interface IResponseCallbackData {
 	data?: IDataObject | IDataObject[];
+	headers?: object;
 	noWebhookResponse?: boolean;
 	responseCode?: number;
 }

packages/cli/src/InternalHooks.ts

@@ -0,0 +1,114 @@
+/* eslint-disable import/no-cycle */
+import { IDataObject, IRun, TelemetryHelpers } from 'n8n-workflow';
+import {
+	IDiagnosticInfo,
+	IInternalHooksClass,
+	IPersonalizationSurveyAnswers,
+	IWorkflowBase,
+} from '.';
+import { Telemetry } from './telemetry';
+
+export class InternalHooksClass implements IInternalHooksClass {
+	constructor(private telemetry: Telemetry) {}
+
+	async onServerStarted(diagnosticInfo: IDiagnosticInfo): Promise<unknown[]> {
+		const info = {
+			version_cli: diagnosticInfo.versionCli,
+			db_type: diagnosticInfo.databaseType,
+			n8n_version_notifications_enabled: diagnosticInfo.notificationsEnabled,
+			n8n_disable_production_main_process: diagnosticInfo.disableProductionWebhooksOnMainProcess,
+			n8n_basic_auth_active: diagnosticInfo.basicAuthActive,
+			system_info: diagnosticInfo.systemInfo,
+			execution_variables: diagnosticInfo.executionVariables,
+			n8n_deployment_type: diagnosticInfo.deploymentType,
+		};
+
+		return Promise.all([
+			this.telemetry.identify(info),
+			this.telemetry.track('Instance started', info),
+		]);
+	}
+
+	async onPersonalizationSurveySubmitted(answers: IPersonalizationSurveyAnswers): Promise<void> {
+		return this.telemetry.track('User responded to personalization questions', {
+			company_size: answers.companySize,
+			coding_skill: answers.codingSkill,
+			work_area: answers.workArea,
+			other_work_area: answers.otherWorkArea,
+		});
+	}
+
+	async onWorkflowCreated(workflow: IWorkflowBase): Promise<void> {
+		return this.telemetry.track('User created workflow', {
+			workflow_id: workflow.id,
+			node_graph: TelemetryHelpers.generateNodesGraph(workflow).nodeGraph,
+		});
+	}
+
+	async onWorkflowDeleted(workflowId: string): Promise<void> {
+		return this.telemetry.track('User deleted workflow', {
+			workflow_id: workflowId,
+		});
+	}
+
+	async onWorkflowSaved(workflow: IWorkflowBase): Promise<void> {
+		return this.telemetry.track('User saved workflow', {
+			workflow_id: workflow.id,
+			node_graph: TelemetryHelpers.generateNodesGraph(workflow).nodeGraph,
+		});
+	}
+
+	async onWorkflowPostExecute(workflow: IWorkflowBase, runData?: IRun): Promise<void> {
+		const properties: IDataObject = {
+			workflow_id: workflow.id,
+			is_manual: false,
+		};
+
+		if (runData !== undefined) {
+			properties.execution_mode = runData.mode;
+			if (runData.mode === 'manual') {
+				properties.is_manual = true;
+			}
+
+			properties.success = !!runData.finished;
+
+			if (!properties.success && runData?.data.resultData.error) {
+				properties.error_message = runData?.data.resultData.error.message;
+				let errorNodeName = runData?.data.resultData.error.node?.name;
+				properties.error_node_type = runData?.data.resultData.error.node?.type;
+
+				if (runData.data.resultData.lastNodeExecuted) {
+					const lastNode = TelemetryHelpers.getNodeTypeForName(
+						workflow,
+						runData.data.resultData.lastNodeExecuted,
+					);
+
+					if (lastNode !== undefined) {
+						properties.error_node_type = lastNode.type;
+						errorNodeName = lastNode.name;
+					}
+				}
+
+				if (properties.is_manual) {
+					const nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow);
+					properties.node_graph = nodeGraphResult.nodeGraph;
+					if (errorNodeName) {
+						properties.error_node_id = nodeGraphResult.nameIndices[errorNodeName];
+					}
+				}
+			}
+		}
+
+		return this.telemetry.trackWorkflowExecution(properties);
+	}
+
+	async onN8nStop(): Promise<void> {
+		const timeoutPromise = new Promise<void>((resolve) => {
+			setTimeout(() => {
+				resolve();
+			}, 3000);
+		});
+
+		return Promise.race([timeoutPromise, this.telemetry.trackN8nStop()]);
+	}
+}

packages/cli/src/InternalHooksManager.ts

@@ -0,0 +1,23 @@
+/* eslint-disable import/no-cycle */
+import { InternalHooksClass } from './InternalHooks';
+import { Telemetry } from './telemetry';
+
+export class InternalHooksManager {
+	private static internalHooksInstance: InternalHooksClass;
+
+	static getInstance(): InternalHooksClass {
+		if (this.internalHooksInstance) {
+			return this.internalHooksInstance;
+		}
+
+		throw new Error('InternalHooks not initialized');
+	}
+
+	static init(instanceId: string): InternalHooksClass {
+		if (!this.internalHooksInstance) {
+			this.internalHooksInstance = new InternalHooksClass(new Telemetry(instanceId));
+		}
+
+		return this.internalHooksInstance;
+	}
+}

packages/cli/src/NodeTypes.ts

@@ -40,6 +40,9 @@ class NodeTypesClass implements INodeTypes {
 	}
 
 	getByNameAndVersion(nodeType: string, version?: number): INodeType {
+		if (this.nodeTypes[nodeType] === undefined) {
+			throw new Error(`The node-type "${nodeType}" is not known!`);
+		}
 		return NodeHelpers.getVersionedTypeNode(this.nodeTypes[nodeType].type, version);
 	}
 }

packages/cli/src/PersonalizationSurvey.ts

@@ -0,0 +1,63 @@
+import { readFileSync, writeFile } from 'fs';
+import { promisify } from 'util';
+import { UserSettings } from 'n8n-core';
+
+import * as config from '../config';
+// eslint-disable-next-line import/no-cycle
+import { Db, IPersonalizationSurvey, IPersonalizationSurveyAnswers } from '.';
+
+const fsWriteFile = promisify(writeFile);
+
+const PERSONALIZATION_SURVEY_FILENAME = 'personalizationSurvey.json';
+
+function loadSurveyFromDisk(): IPersonalizationSurveyAnswers | undefined {
+	const userSettingsPath = UserSettings.getUserN8nFolderPath();
+	try {
+		const surveyFile = readFileSync(
+			`${userSettingsPath}/${PERSONALIZATION_SURVEY_FILENAME}`,
+			'utf-8',
+		);
+		return JSON.parse(surveyFile) as IPersonalizationSurveyAnswers;
+	} catch (error) {
+		return undefined;
+	}
+}
+
+export async function writeSurveyToDisk(
+	surveyAnswers: IPersonalizationSurveyAnswers,
+): Promise<void> {
+	const userSettingsPath = UserSettings.getUserN8nFolderPath();
+	await fsWriteFile(
+		`${userSettingsPath}/${PERSONALIZATION_SURVEY_FILENAME}`,
+		JSON.stringify(surveyAnswers, null, '\t'),
+	);
+}
+
+export async function preparePersonalizationSurvey(): Promise<IPersonalizationSurvey> {
+	const survey: IPersonalizationSurvey = {
+		shouldShow: false,
+	};
+
+	survey.answers = loadSurveyFromDisk();
+
+	if (survey.answers) {
+		return survey;
+	}
+
+	const enabled =
+		(config.get('personalization.enabled') as boolean) &&
+		(config.get('diagnostics.enabled') as boolean);
+
+	if (!enabled) {
+		return survey;
+	}
+
+	const workflowsExist = !!(await Db.collections.Workflow?.findOne());
+
+	if (workflowsExist) {
+		return survey;
+	}
+
+	survey.shouldShow = true;
+	return survey;
+}

packages/cli/src/Queue.ts

@@ -1,12 +1,21 @@
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
 import * as Bull from 'bull';
 import * as config from '../config';
 // eslint-disable-next-line import/no-cycle
-import { IBullJobData } from './Interfaces';
+import { IBullJobData, IBullWebhookResponse } from './Interfaces';
+// eslint-disable-next-line import/no-cycle
+import * as ActiveExecutions from './ActiveExecutions';
+// eslint-disable-next-line import/no-cycle
+import * as WebhookHelpers from './WebhookHelpers';
 
 export class Queue {
+	private activeExecutions: ActiveExecutions.ActiveExecutions;
+
 	private jobQueue: Bull.Queue;
 
 	constructor() {
+		this.activeExecutions = ActiveExecutions.getInstance();
+
 		const prefix = config.get('queue.bull.prefix') as string;
 		const redisOptions = config.get('queue.bull.redis') as object;
 		// Disabling ready check is necessary as it allows worker to
@@ -16,6 +25,14 @@ export class Queue {
 		// More here: https://github.com/OptimalBits/bull/issues/890
 		// @ts-ignore
 		this.jobQueue = new Bull('jobs', { prefix, redis: redisOptions, enableReadyCheck: false });
+
+		this.jobQueue.on('global:progress', (jobId, progress: IBullWebhookResponse) => {
+			this.activeExecutions.resolveResponsePromise(
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+				progress.executionId,
+				WebhookHelpers.decodeWebhookResponse(progress.response),
+			);
+		});
 	}
 
 	async add(jobData: IBullJobData, jobOptions: object): Promise<Bull.Job> {

packages/cli/src/ResponseHelper.ts

@@ -72,11 +72,16 @@ export function sendSuccessResponse(
 	data: any,
 	raw?: boolean,
 	responseCode?: number,
+	responseHeader?: object,
 ) {
 	if (responseCode !== undefined) {
 		res.status(responseCode);
 	}
 
+	if (responseHeader) {
+		res.header(responseHeader);
+	}
+
 	if (raw === true) {
 		if (typeof data === 'string') {
 			res.send(data);
@@ -90,13 +95,13 @@ export function sendSuccessResponse(
 	}
 }
 
-export function sendErrorResponse(res: Response, error: ResponseError) {
+export function sendErrorResponse(res: Response, error: ResponseError, shouldLog = true) {
 	let httpStatusCode = 500;
 	if (error.httpStatusCode) {
 		httpStatusCode = error.httpStatusCode;
 	}
 
-	if (process.env.NODE_ENV !== 'production') {
+	if (process.env.NODE_ENV !== 'production' && shouldLog) {
 		console.error('ERROR RESPONSE');
 		console.error(error);
 	}

packages/cli/src/Server.ts

@@ -27,18 +27,10 @@
 import * as express from 'express';
 import { readFileSync } from 'fs';
 import { dirname as pathDirname, join as pathJoin, resolve as pathResolve } from 'path';
-import {
+import { FindManyOptions, getConnectionManager, In, IsNull, LessThanOrEqual, Not } from 'typeorm';
-	getConnectionManager,
-	In,
-	Like,
-	FindManyOptions,
-	FindOneOptions,
-	IsNull,
-	LessThanOrEqual,
-	Not,
-} from 'typeorm';
 import * as bodyParser from 'body-parser';
 import * as history from 'connect-history-api-fallback';
+import * as os from 'os';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import * as _ from 'lodash';
 import * as clientOAuth2 from 'client-oauth2';
@@ -46,7 +38,7 @@ import * as clientOAuth1 from 'oauth-1.0a';
 import { RequestOptions } from 'oauth-1.0a';
 import * as csrf from 'csrf';
 import * as requestPromise from 'request-promise-native';
-import { createHash, createHmac } from 'crypto';
+import { createHmac } from 'crypto';
 // IMPORTANT! Do not switch to anther bcrypt library unless really necessary and
 // tested with all possible systems like Windows, Alpine on ARM, FreeBSD, ...
 import { compare } from 'bcryptjs';
@@ -62,24 +54,24 @@ import {
 
 import {
 	ICredentialsDecrypted,
-	ICredentialsEncrypted,
 	ICredentialType,
 	IDataObject,
 	INodeCredentials,
+	INodeCredentialsDetails,
 	INodeParameters,
 	INodePropertyOptions,
 	INodeType,
 	INodeTypeDescription,
 	INodeTypeNameVersion,
-	IRunData,
 	INodeVersionedType,
+	ITelemetrySettings,
 	IWorkflowBase,
-	IWorkflowCredentials,
 	LoggerProxy,
 	NodeCredentialTestRequest,
 	NodeCredentialTestResult,
 	NodeHelpers,
 	Workflow,
+	ICredentialsEncrypted,
 	WorkflowExecuteMode,
 } from 'n8n-workflow';
 
@@ -123,12 +115,13 @@ import {
 	IExecutionsStopData,
 	IExecutionsSummary,
 	IExternalHooksClass,
+	IDiagnosticInfo,
 	IN8nUISettings,
 	IPackageVersions,
 	ITagWithCountDb,
 	IWorkflowExecutionDataProcess,
 	IWorkflowResponse,
-	LoadNodesAndCredentials,
+	IPersonalizationSurveyAnswers,
 	NodeTypes,
 	Push,
 	ResponseHelper,
@@ -141,9 +134,13 @@ import {
 	WorkflowHelpers,
 	WorkflowRunner,
 } from '.';
+
 import * as config from '../config';
 
 import * as TagHelpers from './TagHelpers';
+import * as PersonalizationSurvey from './PersonalizationSurvey';
+
+import { InternalHooksManager } from './InternalHooksManager';
 import { TagEntity } from './databases/entities/TagEntity';
 import { WorkflowEntity } from './databases/entities/WorkflowEntity';
 import { NameRequest } from './WorkflowHelpers';
@@ -242,6 +239,22 @@ class App {
 
 		const urlBaseWebhook = WebhookHelpers.getWebhookBaseUrl();
 
+		const telemetrySettings: ITelemetrySettings = {
+			enabled: config.get('diagnostics.enabled') as boolean,
+		};
+
+		if (telemetrySettings.enabled) {
+			const conf = config.get('diagnostics.config.frontend') as string;
+			const [key, url] = conf.split(';');
+
+			if (!key || !url) {
+				LoggerProxy.warn('Diagnostics frontend config is invalid');
+				telemetrySettings.enabled = false;
+			}
+
+			telemetrySettings.config = { key, url };
+		}
+
 		this.frontendSettings = {
 			endpointWebhook: this.endpointWebhook,
 			endpointWebhookTest: this.endpointWebhookTest,
@@ -263,6 +276,10 @@ class App {
 				infoUrl: config.get('versionNotifications.infoUrl'),
 			},
 			instanceId: '',
+			telemetry: telemetrySettings,
+			personalizationSurvey: {
+				shouldShow: false,
+			},
 		};
 	}
 
@@ -289,7 +306,11 @@ class App {
 
 		this.versions = await GenericHelpers.getVersions();
 		this.frontendSettings.versionCli = this.versions.cli;
-		this.frontendSettings.instanceId = (await generateInstanceId()) as string;
+
+		this.frontendSettings.instanceId = await UserSettings.getInstanceId();
+
+		this.frontendSettings.personalizationSurvey =
+			await PersonalizationSurvey.preparePersonalizationSurvey();
 
 		await this.externalHooks.run('frontend.settings', [this.frontendSettings]);
 
@@ -457,10 +478,13 @@ class App {
 				};
 
 				jwt.verify(token, getKey, jwtVerifyOptions, (err: jwt.VerifyErrors, decoded: object) => {
-					if (err) ResponseHelper.jwtAuthAuthorizationError(res, 'Invalid token');
+					if (err) {
-					else if (!isTenantAllowed(decoded))
+						ResponseHelper.jwtAuthAuthorizationError(res, 'Invalid token');
+					} else if (!isTenantAllowed(decoded)) {
 						ResponseHelper.jwtAuthAuthorizationError(res, 'Tenant not allowed');
-					else next();
+					} else {
+						next();
+					}
 				});
 			});
 		}
@@ -642,6 +666,9 @@ class App {
 						});
 					}
 
+					// check credentials for old format
+					await WorkflowHelpers.replaceInvalidCredentials(newWorkflow);
+
 					await this.externalHooks.run('workflow.create', [newWorkflow]);
 
 					await WorkflowHelpers.validateWorkflow(newWorkflow);
@@ -652,6 +679,8 @@ class App {
 
 					// @ts-ignore
 					savedWorkflow.id = savedWorkflow.id.toString();
+					await this.externalHooks.run('workflow.afterCreate', [savedWorkflow]);
+					void InternalHooksManager.getInstance().onWorkflowCreated(newWorkflow as IWorkflowBase);
 					return savedWorkflow;
 				},
 			),
@@ -782,6 +811,9 @@ class App {
 					const { id } = req.params;
 					updateData.id = id;
 
+					// check credentials for old format
+					await WorkflowHelpers.replaceInvalidCredentials(updateData as WorkflowEntity);
+
 					await this.externalHooks.run('workflow.update', [updateData]);
 
 					const isActive = await this.activeWorkflowRunner.isActive(id);
@@ -851,12 +883,12 @@ class App {
 					}
 
 					await this.externalHooks.run('workflow.afterUpdate', [workflow]);
+					void InternalHooksManager.getInstance().onWorkflowSaved(workflow as IWorkflowBase);
 
 					if (workflow.active) {
 						// When the workflow is supposed to be active add it again
 						try {
 							await this.externalHooks.run('workflow.activate', [workflow]);
-
 							await this.activeWorkflowRunner.add(id, isActive ? 'update' : 'activate');
 						} catch (error) {
 							// If workflow could not be activated set it again to inactive
@@ -894,6 +926,7 @@ class App {
 				}
 
 				await Db.collections.Workflow!.delete(id);
+				void InternalHooksManager.getInstance().onWorkflowDeleted(id);
 				await this.externalHooks.run('workflow.afterDelete', [id]);
 
 				return true;
@@ -1293,26 +1326,9 @@ class App {
 						throw new Error('Credentials have to have a name set!');
 					}
 
-					// Check if credentials with the same name and type exist already
-					const findQuery = {
-						where: {
-							name: incomingData.name,
-							type: incomingData.type,
-						},
-					} as FindOneOptions;
-
-					const checkResult = await Db.collections.Credentials!.findOne(findQuery);
-					if (checkResult !== undefined) {
-						throw new ResponseHelper.ResponseError(
-							`Credentials with the same type and name exist already.`,
-							undefined,
-							400,
-						);
-					}
-
 					// Encrypt the data
 					const credentials = new Credentials(
-						incomingData.name,
+						{ id: null, name: incomingData.name },
 						incomingData.type,
 						incomingData.nodesAccess,
 					);
@@ -1321,10 +1337,6 @@ class App {
 
 					await this.externalHooks.run('credentials.create', [newCredentialsData]);
 
-					// Add special database related data
-
-					// TODO: also add user automatically depending on who is logged in, if anybody is logged in
-
 					// Save the credentials in DB
 					const result = await Db.collections.Credentials!.save(newCredentialsData);
 					result.data = incomingData.data;
@@ -1445,24 +1457,6 @@ class App {
 						}
 					}
 
-					// Check if credentials with the same name and type exist already
-					const findQuery = {
-						where: {
-							id: Not(id),
-							name: incomingData.name,
-							type: incomingData.type,
-						},
-					} as FindOneOptions;
-
-					const checkResult = await Db.collections.Credentials!.findOne(findQuery);
-					if (checkResult !== undefined) {
-						throw new ResponseHelper.ResponseError(
-							`Credentials with the same type and name exist already.`,
-							undefined,
-							400,
-						);
-					}
-
 					const encryptionKey = await UserSettings.getEncryptionKey();
 					if (encryptionKey === undefined) {
 						throw new Error('No encryption key got found to encrypt the credentials!');
@@ -1479,7 +1473,7 @@ class App {
 					}
 
 					const currentlySavedCredentials = new Credentials(
-						result.name,
+						result as INodeCredentialsDetails,
 						result.type,
 						result.nodesAccess,
 						result.data,
@@ -1494,7 +1488,7 @@ class App {
 
 					// Encrypt the data
 					const credentials = new Credentials(
-						incomingData.name,
+						{ id, name: incomingData.name },
 						incomingData.type,
 						incomingData.nodesAccess,
 					);
@@ -1563,7 +1557,7 @@ class App {
 						}
 
 						const credentials = new Credentials(
-							result.name,
+							result as INodeCredentialsDetails,
 							result.type,
 							result.nodesAccess,
 							result.data,
@@ -1586,11 +1580,11 @@ class App {
 					const findQuery = {} as FindManyOptions;
 					if (req.query.filter) {
 						findQuery.where = JSON.parse(req.query.filter as string);
-						if ((findQuery.where! as IDataObject).id !== undefined) {
+						if (findQuery.where.id !== undefined) {
 							// No idea if multiple where parameters make db search
 							// slower but to be sure that that is not the case we
 							// remove all unnecessary fields in case the id is defined.
-							findQuery.where = { id: (findQuery.where! as IDataObject).id };
+							findQuery.where = { id: findQuery.where.id };
 						}
 					}
 
@@ -1707,7 +1701,7 @@ class App {
 				const mode: WorkflowExecuteMode = 'internal';
 				const credentialsHelper = new CredentialsHelper(encryptionKey);
 				const decryptedDataOriginal = await credentialsHelper.getDecrypted(
-					result.name,
+					result as INodeCredentialsDetails,
 					result.type,
 					mode,
 					true,
@@ -1766,7 +1760,11 @@ class App {
 				}`;
 
 				// Encrypt the data
-				const credentials = new Credentials(result.name, result.type, result.nodesAccess);
+				const credentials = new Credentials(
+					result as INodeCredentialsDetails,
+					result.type,
+					result.nodesAccess,
+				);
 
 				credentials.setData(decryptedDataOriginal, encryptionKey);
 				const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb;
@@ -1820,16 +1818,10 @@ class App {
 						return ResponseHelper.sendErrorResponse(res, errorResponse);
 					}
 
-					// Decrypt the currently saved credentials
-					const workflowCredentials: IWorkflowCredentials = {
-						[result.type]: {
-							[result.name]: result as ICredentialsEncrypted,
-						},
-					};
 					const mode: WorkflowExecuteMode = 'internal';
 					const credentialsHelper = new CredentialsHelper(encryptionKey);
 					const decryptedDataOriginal = await credentialsHelper.getDecrypted(
-						result.name,
+						result as INodeCredentialsDetails,
 						result.type,
 						mode,
 						true,
@@ -1868,7 +1860,11 @@ class App {
 
 					decryptedDataOriginal.oauthTokenData = oauthTokenJson;
 
-					const credentials = new Credentials(result.name, result.type, result.nodesAccess);
+					const credentials = new Credentials(
+						result as INodeCredentialsDetails,
+						result.type,
+						result.nodesAccess,
+					);
 					credentials.setData(decryptedDataOriginal, encryptionKey);
 					const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb;
 					// Add special database related data
@@ -1913,7 +1909,7 @@ class App {
 				const mode: WorkflowExecuteMode = 'internal';
 				const credentialsHelper = new CredentialsHelper(encryptionKey);
 				const decryptedDataOriginal = await credentialsHelper.getDecrypted(
-					result.name,
+					result as INodeCredentialsDetails,
 					result.type,
 					mode,
 					true,
@@ -1950,7 +1946,11 @@ class App {
 				const oAuthObj = new clientOAuth2(oAuthOptions);
 
 				// Encrypt the data
-				const credentials = new Credentials(result.name, result.type, result.nodesAccess);
+				const credentials = new Credentials(
+					result as INodeCredentialsDetails,
+					result.type,
+					result.nodesAccess,
+				);
 				decryptedDataOriginal.csrfSecret = csrfSecret;
 
 				credentials.setData(decryptedDataOriginal, encryptionKey);
@@ -2036,17 +2036,10 @@ class App {
 						return ResponseHelper.sendErrorResponse(res, errorResponse);
 					}
 
-					// Decrypt the currently saved credentials
-					const workflowCredentials: IWorkflowCredentials = {
-						[result.type]: {
-							[result.name]: result as ICredentialsEncrypted,
-						},
-					};
-
 					const mode: WorkflowExecuteMode = 'internal';
 					const credentialsHelper = new CredentialsHelper(encryptionKey);
 					const decryptedDataOriginal = await credentialsHelper.getDecrypted(
-						result.name,
+						result as INodeCredentialsDetails,
 						result.type,
 						mode,
 						true,
@@ -2128,7 +2121,11 @@ class App {
 
 					_.unset(decryptedDataOriginal, 'csrfSecret');
 
-					const credentials = new Credentials(result.name, result.type, result.nodesAccess);
+					const credentials = new Credentials(
+						result as INodeCredentialsDetails,
+						result.type,
+						result.nodesAccess,
+					);
 					credentials.setData(decryptedDataOriginal, encryptionKey);
 					const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb;
 					// Add special database related data
@@ -2617,6 +2614,31 @@ class App {
 			),
 		);
 
+		// ----------------------------------------
+		// User Survey
+		// ----------------------------------------
+
+		// Process personalization survey responses
+		this.app.post(
+			`/${this.restEndpoint}/user-survey`,
+			async (req: express.Request, res: express.Response) => {
+				if (!this.frontendSettings.personalizationSurvey.shouldShow) {
+					ResponseHelper.sendErrorResponse(
+						res,
+						new ResponseHelper.ResponseError('User survey already submitted', undefined, 400),
+						false,
+					);
+				}
+
+				const answers = req.body as IPersonalizationSurveyAnswers;
+				await PersonalizationSurvey.writeSurveyToDisk(answers);
+				this.frontendSettings.personalizationSurvey.shouldShow = false;
+				this.frontendSettings.personalizationSurvey.answers = answers;
+				ResponseHelper.sendSuccessResponse(res, undefined, true, 200);
+				void InternalHooksManager.getInstance().onPersonalizationSurveySubmitted(answers);
+			},
+		);
+
 		// ----------------------------------------
 		// Webhooks
 		// ----------------------------------------
@@ -2647,7 +2669,13 @@ class App {
 					return;
 				}
 
-				ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
+				ResponseHelper.sendSuccessResponse(
+					res,
+					response.data,
+					true,
+					response.responseCode,
+					response.headers,
+				);
 			},
 		);
 
@@ -2698,7 +2726,13 @@ class App {
 					return;
 				}
 
-				ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
+				ResponseHelper.sendSuccessResponse(
+					res,
+					response.data,
+					true,
+					response.responseCode,
+					response.headers,
+				);
 			},
 		);
 
@@ -2724,7 +2758,13 @@ class App {
 					return;
 				}
 
-				ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
+				ResponseHelper.sendSuccessResponse(
+					res,
+					response.data,
+					true,
+					response.responseCode,
+					response.headers,
+				);
 			},
 		);
 
@@ -2746,16 +2786,10 @@ class App {
 							return;
 						}
 
-						const loadNodesAndCredentials = LoadNodesAndCredentials();
-
 						const credentialsOverwrites = CredentialsOverwrites();
 
 						await credentialsOverwrites.init(body);
 
-						const credentialTypes = CredentialTypes();
-
-						await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
-
 						this.presetCredentialsLoaded = true;
 
 						ResponseHelper.sendSuccessResponse(res, { success: true }, true, 200);
@@ -2826,6 +2860,43 @@ export async function start(): Promise<void> {
 		console.log(`Version: ${versions.cli}`);
 
 		await app.externalHooks.run('n8n.ready', [app]);
+		const cpus = os.cpus();
+		const diagnosticInfo: IDiagnosticInfo = {
+			basicAuthActive: config.get('security.basicAuth.active') as boolean,
+			databaseType: (await GenericHelpers.getConfigValue('database.type')) as DatabaseType,
+			disableProductionWebhooksOnMainProcess:
+				config.get('endpoints.disableProductionWebhooksOnMainProcess') === true,
+			notificationsEnabled: config.get('versionNotifications.enabled') === true,
+			versionCli: versions.cli,
+			systemInfo: {
+				os: {
+					type: os.type(),
+					version: os.version(),
+				},
+				memory: os.totalmem() / 1024,
+				cpus: {
+					count: cpus.length,
+					model: cpus[0].model,
+					speed: cpus[0].speed,
+				},
+			},
+			executionVariables: {
+				executions_process: config.get('executions.process'),
+				executions_mode: config.get('executions.mode'),
+				executions_timeout: config.get('executions.timeout'),
+				executions_timeout_max: config.get('executions.maxTimeout'),
+				executions_data_save_on_error: config.get('executions.saveDataOnError'),
+				executions_data_save_on_success: config.get('executions.saveDataOnSuccess'),
+				executions_data_save_on_progress: config.get('executions.saveExecutionProgress'),
+				executions_data_save_manual_executions: config.get('executions.saveDataManualExecutions'),
+				executions_data_prune: config.get('executions.pruneData'),
+				executions_data_max_age: config.get('executions.pruneDataMaxAge'),
+				executions_data_prune_timeout: config.get('executions.pruneDataTimeout'),
+			},
+			deploymentType: config.get('deployment.type'),
+		};
+
+		void InternalHooksManager.getInstance().onServerStarted(diagnosticInfo);
 	});
 }
 
@@ -2864,14 +2935,3 @@ async function getExecutionsCount(
 	const count = await Db.collections.Execution!.count(countFilter);
 	return { count, estimate: false };
 }
-
-async function generateInstanceId() {
-	const encryptionKey = await UserSettings.getEncryptionKey();
-	const hash = encryptionKey
-		? createHash('sha256')
-				.update(encryptionKey.slice(Math.round(encryptionKey.length / 2)))
-				.digest('hex')
-		: undefined;
-
-	return hash;
-}

packages/cli/src/WebhookHelpers.ts

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-unsafe-call */
 /* eslint-disable no-param-reassign */
 /* eslint-disable @typescript-eslint/prefer-optional-chain */
 /* eslint-disable @typescript-eslint/no-shadow */
@@ -18,9 +19,13 @@ import { get } from 'lodash';
 import { BINARY_ENCODING, NodeExecuteFunctions } from 'n8n-core';
 
 import {
+	createDeferredPromise,
 	IBinaryKeyData,
 	IDataObject,
+	IDeferredPromise,
 	IExecuteData,
+	IExecuteResponsePromiseData,
+	IN8nHttpFullResponse,
 	INode,
 	IRunExecutionData,
 	IWebhookData,
@@ -34,20 +39,20 @@ import {
 } from 'n8n-workflow';
 // eslint-disable-next-line import/no-cycle
 import {
-	ActiveExecutions,
 	GenericHelpers,
 	IExecutionDb,
 	IResponseCallbackData,
 	IWorkflowDb,
 	IWorkflowExecutionDataProcess,
 	ResponseHelper,
-	// eslint-disable-next-line @typescript-eslint/no-unused-vars
-	WorkflowCredentials,
 	WorkflowExecuteAdditionalData,
 	WorkflowHelpers,
 	WorkflowRunner,
 } from '.';
 
+// eslint-disable-next-line import/no-cycle
+import * as ActiveExecutions from './ActiveExecutions';
+
 const activeExecutions = ActiveExecutions.getInstance();
 
 /**
@@ -91,6 +96,35 @@ export function getWorkflowWebhooks(
 	return returnData;
 }
 
+export function decodeWebhookResponse(
+	response: IExecuteResponsePromiseData,
+): IExecuteResponsePromiseData {
+	if (
+		typeof response === 'object' &&
+		typeof response.body === 'object' &&
+		(response.body as IDataObject)['__@N8nEncodedBuffer@__']
+	) {
+		response.body = Buffer.from(
+			(response.body as IDataObject)['__@N8nEncodedBuffer@__'] as string,
+			BINARY_ENCODING,
+		);
+	}
+
+	return response;
+}
+
+export function encodeWebhookResponse(
+	response: IExecuteResponsePromiseData,
+): IExecuteResponsePromiseData {
+	if (typeof response === 'object' && Buffer.isBuffer(response.body)) {
+		response.body = {
+			'__@N8nEncodedBuffer@__': response.body.toString(BINARY_ENCODING),
+		};
+	}
+
+	return response;
+}
+
 /**
  * Returns all the webhooks which should be created for the give workflow
  *
@@ -169,7 +203,7 @@ export async function executeWebhook(
 		200,
 	) as number;
 
-	if (!['onReceived', 'lastNode'].includes(responseMode as string)) {
+	if (!['onReceived', 'lastNode', 'responseNode'].includes(responseMode as string)) {
 		// If the mode is not known we error. Is probably best like that instead of using
 		// the default that people know as early as possible (probably already testing phase)
 		// that something does not resolve properly.
@@ -356,9 +390,52 @@ export async function executeWebhook(
 			workflowData,
 		};
 
+		let responsePromise: IDeferredPromise<IN8nHttpFullResponse> | undefined;
+		if (responseMode === 'responseNode') {
+			responsePromise = await createDeferredPromise<IN8nHttpFullResponse>();
+			responsePromise
+				.promise()
+				.then((response: IN8nHttpFullResponse) => {
+					if (didSendResponse) {
+						return;
+					}
+
+					if (Buffer.isBuffer(response.body)) {
+						res.header(response.headers);
+						res.end(response.body);
+
+						responseCallback(null, {
+							noWebhookResponse: true,
+						});
+					} else {
+						// TODO: This probably needs some more changes depending on the options on the
+						//       Webhook Response node
+						responseCallback(null, {
+							data: response.body as IDataObject,
+							headers: response.headers,
+							responseCode: response.statusCode,
+						});
+					}
+
+					didSendResponse = true;
+				})
+				.catch(async (error) => {
+					Logger.error(
+						`Error with Webhook-Response for execution "${executionId}": "${error.message}"`,
+						{ executionId, workflowId: workflow.id },
+					);
+				});
+		}
+
 		// Start now to run the workflow
 		const workflowRunner = new WorkflowRunner();
-		executionId = await workflowRunner.run(runData, true, !didSendResponse, executionId);
+		executionId = await workflowRunner.run(
+			runData,
+			true,
+			!didSendResponse,
+			executionId,
+			responsePromise,
+		);
 
 		Logger.verbose(
 			`Started execution of workflow "${workflow.name}" from webhook with execution ID ${executionId}`,
@@ -398,6 +475,20 @@ export async function executeWebhook(
 					return data;
 				}
 
+				if (responseMode === 'responseNode') {
+					if (!didSendResponse) {
+						// Return an error if no Webhook-Response node did send any data
+						responseCallback(null, {
+							data: {
+								message: 'Workflow executed sucessfully.',
+							},
+							responseCode,
+						});
+						didSendResponse = true;
+					}
+					return undefined;
+				}
+
 				if (returnData === undefined) {
 					if (!didSendResponse) {
 						responseCallback(null, {

packages/cli/src/WebhookServer.ts

@@ -64,7 +64,13 @@ export function registerProductionWebhooks() {
 				return;
 			}
 
-			ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
+			ResponseHelper.sendSuccessResponse(
+				res,
+				response.data,
+				true,
+				response.responseCode,
+				response.headers,
+			);
 		},
 	);
 
@@ -115,7 +121,13 @@ export function registerProductionWebhooks() {
 				return;
 			}
 
-			ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
+			ResponseHelper.sendSuccessResponse(
+				res,
+				response.data,
+				true,
+				response.responseCode,
+				response.headers,
+			);
 		},
 	);
 
@@ -141,7 +153,13 @@ export function registerProductionWebhooks() {
 				return;
 			}
 
-			ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
+			ResponseHelper.sendSuccessResponse(
+				res,
+				response.data,
+				true,
+				response.responseCode,
+				response.headers,
+			);
 		},
 	);
 
@@ -173,7 +191,13 @@ export function registerProductionWebhooks() {
 				return;
 			}
 
-			ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
+			ResponseHelper.sendSuccessResponse(
+				res,
+				response.data,
+				true,
+				response.responseCode,
+				response.headers,
+			);
 		},
 	);
 
@@ -199,7 +223,13 @@ export function registerProductionWebhooks() {
 				return;
 			}
 
-			ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
+			ResponseHelper.sendSuccessResponse(
+				res,
+				response.data,
+				true,
+				response.responseCode,
+				response.headers,
+			);
 		},
 	);
 
@@ -225,7 +255,13 @@ export function registerProductionWebhooks() {
 				return;
 			}
 
-			ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
+			ResponseHelper.sendSuccessResponse(
+				res,
+				response.data,
+				true,
+				response.responseCode,
+				response.headers,
+			);
 		},
 	);
 }

packages/cli/src/WorkflowCredentials.ts

@@ -10,7 +10,7 @@ export async function WorkflowCredentials(nodes: INode[]): Promise<IWorkflowCred
 
 	let node;
 	let type;
-	let name;
+	let nodeCredentials;
 	let foundCredentials;
 	// eslint-disable-next-line no-restricted-syntax
 	for (node of nodes) {
@@ -21,19 +21,30 @@ export async function WorkflowCredentials(nodes: INode[]): Promise<IWorkflowCred
 
 		// eslint-disable-next-line no-restricted-syntax
 		for (type of Object.keys(node.credentials)) {
-			if (!returnCredentials.hasOwnProperty(type)) {
+			if (!returnCredentials[type]) {
 				returnCredentials[type] = {};
 			}
-			name = node.credentials[type];
+			nodeCredentials = node.credentials[type];
 
-			if (!returnCredentials[type].hasOwnProperty(name)) {
+			if (!nodeCredentials.id) {
+				throw new Error(
+					`Credentials with name "${nodeCredentials.name}" for type "${type}" miss an ID.`,
+				);
+			}
+
+			if (!returnCredentials[type][nodeCredentials.id]) {
 				// eslint-disable-next-line no-await-in-loop, @typescript-eslint/no-non-null-assertion
-				foundCredentials = await Db.collections.Credentials!.find({ name, type });
+				foundCredentials = await Db.collections.Credentials!.findOne({
-				if (!foundCredentials.length) {
+					id: nodeCredentials.id,
-					throw new Error(`Could not find credentials for type "${type}" with name "${name}".`);
+					type,
+				});
+				if (!foundCredentials) {
+					throw new Error(
+						`Could not find credentials for type "${type}" with ID "${nodeCredentials.id}".`,
+					);
 				}
 				// eslint-disable-next-line prefer-destructuring
-				returnCredentials[type][name] = foundCredentials[0];
+				returnCredentials[type][nodeCredentials.id] = foundCredentials;
 			}
 		}
 	}

packages/cli/src/WorkflowExecuteAdditionalData.ts

@@ -48,6 +48,7 @@ import {
 	IExecutionDb,
 	IExecutionFlattedDb,
 	IExecutionResponse,
+	InternalHooksManager,
 	IPushDataExecutionFinished,
 	IWorkflowBase,
 	IWorkflowExecuteProcess,
@@ -903,6 +904,7 @@ export async function executeWorkflow(
 	}
 
 	await externalHooks.run('workflow.postExecute', [data, workflowData]);
+	void InternalHooksManager.getInstance().onWorkflowPostExecute(workflowData, data);
 
 	if (data.finished === true) {
 		// Workflow did finish successfully

packages/cli/src/WorkflowHelpers.ts

@@ -12,6 +12,7 @@ import {
 	IDataObject,
 	IExecuteData,
 	INode,
+	INodeCredentialsDetails,
 	IRun,
 	IRunExecutionData,
 	ITaskData,
@@ -385,6 +386,113 @@ export async function getStaticDataById(workflowId: string | number) {
 	return workflowData.staticData || {};
 }
 
+// Checking if credentials of old format are in use and run a DB check if they might exist uniquely
+export async function replaceInvalidCredentials(workflow: WorkflowEntity): Promise<WorkflowEntity> {
+	const { nodes } = workflow;
+	if (!nodes) return workflow;
+
+	// caching
+	const credentialsByName: Record<string, Record<string, INodeCredentialsDetails>> = {};
+	const credentialsById: Record<string, Record<string, INodeCredentialsDetails>> = {};
+
+	// for loop to run DB fetches sequential and use cache to keep pressure off DB
+	// trade-off: longer response time for less DB queries
+	/* eslint-disable no-await-in-loop */
+	for (const node of nodes) {
+		if (!node.credentials || node.disabled) {
+			continue;
+		}
+		// extract credentials types
+		const allNodeCredentials = Object.entries(node.credentials);
+		for (const [nodeCredentialType, nodeCredentials] of allNodeCredentials) {
+			// Check if Node applies old credentials style
+			if (typeof nodeCredentials === 'string' || nodeCredentials.id === null) {
+				const name = typeof nodeCredentials === 'string' ? nodeCredentials : nodeCredentials.name;
+				// init cache for type
+				if (!credentialsByName[nodeCredentialType]) {
+					credentialsByName[nodeCredentialType] = {};
+				}
+				if (credentialsByName[nodeCredentialType][name] === undefined) {
+					const credentials = await Db.collections.Credentials?.find({
+						name,
+						type: nodeCredentialType,
+					});
+					// if credential name-type combination is unique, use it
+					if (credentials?.length === 1) {
+						credentialsByName[nodeCredentialType][name] = {
+							id: credentials[0].id.toString(),
+							name: credentials[0].name,
+						};
+						node.credentials[nodeCredentialType] = credentialsByName[nodeCredentialType][name];
+						continue;
+					}
+
+					// nothing found - add invalid credentials to cache to prevent further DB checks
+					credentialsByName[nodeCredentialType][name] = {
+						id: null,
+						name,
+					};
+				} else {
+					// get credentials from cache
+					node.credentials[nodeCredentialType] = credentialsByName[nodeCredentialType][name];
+				}
+				continue;
+			}
+
+			// Node has credentials with an ID
+
+			// init cache for type
+			if (!credentialsById[nodeCredentialType]) {
+				credentialsById[nodeCredentialType] = {};
+			}
+
+			// check if credentials for ID-type are not yet cached
+			if (credentialsById[nodeCredentialType][nodeCredentials.id] === undefined) {
+				// check first if ID-type combination exists
+				const credentials = await Db.collections.Credentials?.findOne({
+					id: nodeCredentials.id,
+					type: nodeCredentialType,
+				});
+				if (credentials) {
+					credentialsById[nodeCredentialType][nodeCredentials.id] = {
+						id: credentials.id.toString(),
+						name: credentials.name,
+					};
+					node.credentials[nodeCredentialType] =
+						credentialsById[nodeCredentialType][nodeCredentials.id];
+					continue;
+				}
+				// no credentials found for ID, check if some exist for name
+				const credsByName = await Db.collections.Credentials?.find({
+					name: nodeCredentials.name,
+					type: nodeCredentialType,
+				});
+				// if credential name-type combination is unique, take it
+				if (credsByName?.length === 1) {
+					// add found credential to cache
+					credentialsById[nodeCredentialType][credsByName[0].id] = {
+						id: credsByName[0].id.toString(),
+						name: credsByName[0].name,
+					};
+					node.credentials[nodeCredentialType] =
+						credentialsById[nodeCredentialType][credsByName[0].id];
+					continue;
+				}
+
+				// nothing found - add invalid credentials to cache to prevent further DB checks
+				credentialsById[nodeCredentialType][nodeCredentials.id] = nodeCredentials;
+				continue;
+			}
+
+			// get credentials from cache
+			node.credentials[nodeCredentialType] =
+				credentialsById[nodeCredentialType][nodeCredentials.id];
+		}
+	}
+	/* eslint-enable no-await-in-loop */
+	return workflow;
+}
+
 // TODO: Deduplicate `validateWorkflow` and `throwDuplicateEntryError` with TagHelpers?
 
 // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types

packages/cli/src/WorkflowRunner.ts

@@ -15,8 +15,9 @@ import { IProcessMessage, WorkflowExecute } from 'n8n-core';
 
 import {
 	ExecutionError,
+	IDeferredPromise,
+	IExecuteResponsePromiseData,
 	IRun,
-	IWorkflowBase,
 	LoggerProxy as Logger,
 	Workflow,
 	WorkflowExecuteMode,
@@ -42,9 +43,7 @@ import {
 	IBullJobResponse,
 	ICredentialsOverwrite,
 	ICredentialsTypeData,
-	IExecutionDb,
 	IExecutionFlattedDb,
-	IExecutionResponse,
 	IProcessMessageDataHook,
 	ITransferNodeTypes,
 	IWorkflowExecutionDataProcess,
@@ -52,10 +51,12 @@ import {
 	NodeTypes,
 	Push,
 	ResponseHelper,
+	WebhookHelpers,
 	WorkflowExecuteAdditionalData,
 	WorkflowHelpers,
 } from '.';
 import * as Queue from './Queue';
+import { InternalHooksManager } from './InternalHooksManager';
 
 export class WorkflowRunner {
 	activeExecutions: ActiveExecutions.ActiveExecutions;
@@ -146,6 +147,7 @@ export class WorkflowRunner {
 		loadStaticData?: boolean,
 		realtime?: boolean,
 		executionId?: string,
+		responsePromise?: IDeferredPromise<IExecuteResponsePromiseData>,
 	): Promise<string> {
 		const executionsProcess = config.get('executions.process') as string;
 		const executionsMode = config.get('executions.mode') as string;
@@ -153,17 +155,35 @@ export class WorkflowRunner {
 		if (executionsMode === 'queue' && data.executionMode !== 'manual') {
 			// Do not run "manual" executions in bull because sending events to the
 			// frontend would not be possible
-			executionId = await this.runBull(data, loadStaticData, realtime, executionId);
+			executionId = await this.runBull(
+				data,
+				loadStaticData,
+				realtime,
+				executionId,
+				responsePromise,
+			);
 		} else if (executionsProcess === 'main') {
-			executionId = await this.runMainProcess(data, loadStaticData, executionId);
+			executionId = await this.runMainProcess(data, loadStaticData, executionId, responsePromise);
 		} else {
-			executionId = await this.runSubprocess(data, loadStaticData, executionId);
+			executionId = await this.runSubprocess(data, loadStaticData, executionId, responsePromise);
 		}
 
+		const postExecutePromise = this.activeExecutions.getPostExecutePromise(executionId);
+
 		const externalHooks = ExternalHooks();
+		postExecutePromise
+			.then(async (executionData) => {
+				void InternalHooksManager.getInstance().onWorkflowPostExecute(
+					data.workflowData,
+					executionData,
+				);
+			})
+			.catch((error) => {
+				console.error('There was a problem running internal hook "onWorkflowPostExecute"', error);
+			});
+
 		if (externalHooks.exists('workflow.postExecute')) {
-			this.activeExecutions
+			postExecutePromise
-				.getPostExecutePromise(executionId)
 				.then(async (executionData) => {
 					await externalHooks.run('workflow.postExecute', [executionData, data.workflowData]);
 				})
@@ -188,6 +208,7 @@ export class WorkflowRunner {
 		data: IWorkflowExecutionDataProcess,
 		loadStaticData?: boolean,
 		restartExecutionId?: string,
+		responsePromise?: IDeferredPromise<IExecuteResponsePromiseData>,
 	): Promise<string> {
 		if (loadStaticData === true && data.workflowData.id) {
 			data.workflowData.staticData = await WorkflowHelpers.getStaticDataById(
@@ -244,6 +265,15 @@ export class WorkflowRunner {
 				executionId,
 				true,
 			);
+
+			additionalData.hooks.hookFunctions.sendResponse = [
+				async (response: IExecuteResponsePromiseData): Promise<void> => {
+					if (responsePromise) {
+						responsePromise.resolve(response);
+					}
+				},
+			];
+
 			additionalData.sendMessageToUI = WorkflowExecuteAdditionalData.sendMessageToUI.bind({
 				sessionId: data.sessionId,
 			});
@@ -329,11 +359,15 @@ export class WorkflowRunner {
 		loadStaticData?: boolean,
 		realtime?: boolean,
 		restartExecutionId?: string,
+		responsePromise?: IDeferredPromise<IExecuteResponsePromiseData>,
 	): Promise<string> {
 		// TODO: If "loadStaticData" is set to true it has to load data new on worker
 
 		// Register the active execution
 		const executionId = await this.activeExecutions.add(data, undefined, restartExecutionId);
+		if (responsePromise) {
+			this.activeExecutions.attachResponsePromise(executionId, responsePromise);
+		}
 
 		const jobData: IBullJobData = {
 			executionId,
@@ -533,6 +567,7 @@ export class WorkflowRunner {
 		data: IWorkflowExecutionDataProcess,
 		loadStaticData?: boolean,
 		restartExecutionId?: string,
+		responsePromise?: IDeferredPromise<IExecuteResponsePromiseData>,
 	): Promise<string> {
 		let startedAt = new Date();
 		const subprocess = fork(pathJoin(__dirname, 'WorkflowRunnerProcess.js'));
@@ -641,6 +676,10 @@ export class WorkflowRunner {
 			} else if (message.type === 'end') {
 				clearTimeout(executionTimeout);
 				this.activeExecutions.remove(executionId, message.data.runData);
+			} else if (message.type === 'sendResponse') {
+				if (responsePromise) {
+					responsePromise.resolve(WebhookHelpers.decodeWebhookResponse(message.data.response));
+				}
 			} else if (message.type === 'sendMessageToUI') {
 				// eslint-disable-next-line @typescript-eslint/no-unsafe-call
 				WorkflowExecuteAdditionalData.sendMessageToUI.bind({ sessionId: data.sessionId })(

packages/cli/src/WorkflowRunnerProcess.ts

@@ -5,11 +5,12 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 /* eslint-disable @typescript-eslint/no-use-before-define */
 /* eslint-disable @typescript-eslint/unbound-method */
-import { IProcessMessage, WorkflowExecute } from 'n8n-core';
+import { IProcessMessage, UserSettings, WorkflowExecute } from 'n8n-core';
 
 import {
 	ExecutionError,
 	IDataObject,
+	IExecuteResponsePromiseData,
 	IExecuteWorkflowInfo,
 	ILogger,
 	INodeExecutionData,
@@ -33,6 +34,7 @@ import {
 	IWorkflowExecuteProcess,
 	IWorkflowExecutionDataProcessWithExecution,
 	NodeTypes,
+	WebhookHelpers,
 	WorkflowExecuteAdditionalData,
 	WorkflowHelpers,
 } from '.';
@@ -40,6 +42,7 @@ import {
 import { getLogger } from './Logger';
 
 import * as config from '../config';
+import { InternalHooksManager } from './InternalHooksManager';
 
 export class WorkflowRunnerProcess {
 	data: IWorkflowExecutionDataProcessWithExecution | undefined;
@@ -133,6 +136,9 @@ export class WorkflowRunnerProcess {
 		const externalHooks = ExternalHooks();
 		await externalHooks.init();
 
+		const instanceId = (await UserSettings.prepareUserSettings()).instanceId ?? '';
+		InternalHooksManager.init(instanceId);
+
 		// Credentials should now be loaded from database.
 		// We check if any node uses credentials. If it does, then
 		// init database.
@@ -196,6 +202,15 @@ export class WorkflowRunnerProcess {
 			workflowTimeout <= 0 ? undefined : Date.now() + workflowTimeout * 1000,
 		);
 		additionalData.hooks = this.getProcessForwardHooks();
+
+		additionalData.hooks.hookFunctions.sendResponse = [
+			async (response: IExecuteResponsePromiseData): Promise<void> => {
+				await sendToParentProcess('sendResponse', {
+					response: WebhookHelpers.encodeWebhookResponse(response),
+				});
+			},
+		];
+
 		additionalData.executionId = inputData.executionId;
 
 		// eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -243,6 +258,7 @@ export class WorkflowRunnerProcess {
 				const { workflow } = executeWorkflowFunctionOutput;
 				result = await workflowExecute.processRunExecutionData(workflow);
 				await externalHooks.run('workflow.postExecute', [result, workflowData]);
+				void InternalHooksManager.getInstance().onWorkflowPostExecute(workflowData, result);
 				await sendToParentProcess('finishExecution', { executionId, result });
 				delete this.childExecutions[executionId];
 			} catch (e) {

packages/cli/src/databases/MigrationHelpers.ts

@@ -0,0 +1,39 @@
+import { QueryRunner } from 'typeorm';
+
+export class MigrationHelpers {
+	queryRunner: QueryRunner;
+
+	constructor(queryRunner: QueryRunner) {
+		this.queryRunner = queryRunner;
+	}
+
+	// runs an operation sequential on chunks of a query that returns a potentially large Array.
+	/* eslint-disable no-await-in-loop */
+	async runChunked(
+		query: string,
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		operation: (results: any[]) => Promise<void>,
+		limit = 100,
+	): Promise<void> {
+		let offset = 0;
+		let chunkedQuery: string;
+		let chunkedQueryResults: unknown[];
+
+		do {
+			chunkedQuery = this.chunkQuery(query, limit, offset);
+			chunkedQueryResults = (await this.queryRunner.query(chunkedQuery)) as unknown[];
+			// pass a copy to prevent errors from mutation
+			await operation([...chunkedQueryResults]);
+			offset += limit;
+		} while (chunkedQueryResults.length === limit);
+	}
+	/* eslint-enable no-await-in-loop */
+
+	private chunkQuery(query: string, limit: number, offset = 0): string {
+		return `
+			${query}
+			LIMIT ${limit}
+			OFFSET ${offset}
+		`;
+	}
+}

packages/cli/src/databases/entities/CredentialsEntity.ts

@@ -11,9 +11,40 @@ import {
 	PrimaryGeneratedColumn,
 	UpdateDateColumn,
 } from 'typeorm';
-import { getTimestampSyntax, resolveDataType } from '../utils';
 
-import { ICredentialsDb } from '../..';
+import config = require('../../../config');
+import { DatabaseType, ICredentialsDb } from '../..';
+
+function resolveDataType(dataType: string) {
+	const dbType = config.get('database.type') as DatabaseType;
+
+	const typeMap: { [key in DatabaseType]: { [key: string]: string } } = {
+		sqlite: {
+			json: 'simple-json',
+		},
+		postgresdb: {
+			datetime: 'timestamptz',
+		},
+		mysqldb: {},
+		mariadb: {},
+	};
+
+	return typeMap[dbType][dataType] ?? dataType;
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+function getTimestampSyntax() {
+	const dbType = config.get('database.type') as DatabaseType;
+
+	const map: { [key in DatabaseType]: string } = {
+		sqlite: "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')",
+		postgresdb: 'CURRENT_TIMESTAMP(3)',
+		mysqldb: 'CURRENT_TIMESTAMP(3)',
+		mariadb: 'CURRENT_TIMESTAMP(3)',
+	};
+
+	return map[dbType];
+}
 
 @Entity()
 export class CredentialsEntity implements ICredentialsDb {

packages/cli/src/databases/entities/ExecutionEntity.ts

@@ -2,9 +2,25 @@
 import { WorkflowExecuteMode } from 'n8n-workflow';
 
 import { Column, ColumnOptions, Entity, Index, PrimaryGeneratedColumn } from 'typeorm';
-import { IExecutionFlattedDb, IWorkflowDb } from '../..';
+import config = require('../../../config');
+import { DatabaseType, IExecutionFlattedDb, IWorkflowDb } from '../..';
 
-import { resolveDataType } from '../utils';
+function resolveDataType(dataType: string) {
+	const dbType = config.get('database.type') as DatabaseType;
+
+	const typeMap: { [key in DatabaseType]: { [key: string]: string } } = {
+		sqlite: {
+			json: 'simple-json',
+		},
+		postgresdb: {
+			datetime: 'timestamptz',
+		},
+		mysqldb: {},
+		mariadb: {},
+	};
+
+	return typeMap[dbType][dataType] ?? dataType;
+}
 
 @Entity()
 export class ExecutionEntity implements IExecutionFlattedDb {

packages/cli/src/databases/entities/TagEntity.ts

@@ -12,9 +12,24 @@ import {
 } from 'typeorm';
 import { IsDate, IsOptional, IsString, Length } from 'class-validator';
 
+import config = require('../../../config');
+import { DatabaseType } from '../../index';
 import { ITagDb } from '../../Interfaces';
 import { WorkflowEntity } from './WorkflowEntity';
-import { getTimestampSyntax } from '../utils';
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+function getTimestampSyntax() {
+	const dbType = config.get('database.type') as DatabaseType;
+
+	const map: { [key in DatabaseType]: string } = {
+		sqlite: "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')",
+		postgresdb: 'CURRENT_TIMESTAMP(3)',
+		mysqldb: 'CURRENT_TIMESTAMP(3)',
+		mariadb: 'CURRENT_TIMESTAMP(3)',
+	};
+
+	return map[dbType];
+}
 
 @Entity()
 export class TagEntity implements ITagDb {

packages/cli/src/databases/entities/WorkflowEntity.ts

@@ -17,12 +17,41 @@ import {
 	UpdateDateColumn,
 } from 'typeorm';
 
-import { IWorkflowDb } from '../..';
+import config = require('../../../config');
-
+import { DatabaseType, IWorkflowDb } from '../..';
-import { getTimestampSyntax, resolveDataType } from '../utils';
-
 import { TagEntity } from './TagEntity';
 
+function resolveDataType(dataType: string) {
+	const dbType = config.get('database.type') as DatabaseType;
+
+	const typeMap: { [key in DatabaseType]: { [key: string]: string } } = {
+		sqlite: {
+			json: 'simple-json',
+		},
+		postgresdb: {
+			datetime: 'timestamptz',
+		},
+		mysqldb: {},
+		mariadb: {},
+	};
+
+	return typeMap[dbType][dataType] ?? dataType;
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+function getTimestampSyntax() {
+	const dbType = config.get('database.type') as DatabaseType;
+
+	const map: { [key in DatabaseType]: string } = {
+		sqlite: "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')",
+		postgresdb: 'CURRENT_TIMESTAMP(3)',
+		mysqldb: 'CURRENT_TIMESTAMP(3)',
+		mariadb: 'CURRENT_TIMESTAMP(3)',
+	};
+
+	return map[dbType];
+}
+
 @Entity()
 export class WorkflowEntity implements IWorkflowDb {
 	@PrimaryGeneratedColumn()

packages/cli/src/databases/mysqldb/migrations/1630451444017-UpdateWorkflowCredentials.ts

@@ -0,0 +1,304 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+import config = require('../../../../config');
+import { MigrationHelpers } from '../../MigrationHelpers';
+
+// replacing the credentials in workflows and execution
+// `nodeType: name` changes to `nodeType: { id, name }`
+
+export class UpdateWorkflowCredentials1630451444017 implements MigrationInterface {
+	name = 'UpdateWorkflowCredentials1630451444017';
+
+	public async up(queryRunner: QueryRunner): Promise<void> {
+		console.log('Start migration', this.name);
+		console.time(this.name);
+		const tablePrefix = config.get('database.tablePrefix');
+		const helpers = new MigrationHelpers(queryRunner);
+
+		const credentialsEntities = await queryRunner.query(`
+			SELECT id, name, type
+			FROM ${tablePrefix}credentials_entity
+		`);
+
+		const workflowsQuery = `
+			SELECT id, nodes
+			FROM ${tablePrefix}workflow_entity
+		`;
+		// @ts-ignore
+		await helpers.runChunked(workflowsQuery, (workflows) => {
+			workflows.forEach(async (workflow) => {
+				const nodes = workflow.nodes;
+				let credentialsUpdated = false;
+				// @ts-ignore
+				nodes.forEach((node) => {
+					if (node.credentials) {
+						const allNodeCredentials = Object.entries(node.credentials);
+						for (const [type, name] of allNodeCredentials) {
+							if (typeof name === 'string') {
+								const matchingCredentials = credentialsEntities.find(
+									// @ts-ignore
+									(credentials) => credentials.name === name && credentials.type === type,
+								);
+								node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name };
+								credentialsUpdated = true;
+							}
+						}
+					}
+				});
+				if (credentialsUpdated) {
+					const [updateQuery, updateParams] =
+						queryRunner.connection.driver.escapeQueryWithParameters(
+							`
+								UPDATE ${tablePrefix}workflow_entity
+								SET nodes = :nodes
+								WHERE id = '${workflow.id}'
+							`,
+							{ nodes: JSON.stringify(nodes) },
+							{},
+						);
+
+					queryRunner.query(updateQuery, updateParams);
+				}
+			});
+		});
+
+		const waitingExecutionsQuery = `
+			SELECT id, workflowData
+			FROM ${tablePrefix}execution_entity
+			WHERE waitTill IS NOT NULL AND finished = 0
+		`;
+		// @ts-ignore
+		await helpers.runChunked(waitingExecutionsQuery, (waitingExecutions) => {
+			waitingExecutions.forEach(async (execution) => {
+				const data = execution.workflowData;
+				let credentialsUpdated = false;
+				// @ts-ignore
+				data.nodes.forEach((node) => {
+					if (node.credentials) {
+						const allNodeCredentials = Object.entries(node.credentials);
+						for (const [type, name] of allNodeCredentials) {
+							if (typeof name === 'string') {
+								const matchingCredentials = credentialsEntities.find(
+									// @ts-ignore
+									(credentials) => credentials.name === name && credentials.type === type,
+								);
+								node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name };
+								credentialsUpdated = true;
+							}
+						}
+					}
+				});
+				if (credentialsUpdated) {
+					const [updateQuery, updateParams] =
+						queryRunner.connection.driver.escapeQueryWithParameters(
+							`
+								UPDATE ${tablePrefix}execution_entity
+								SET workflowData = :data
+								WHERE id = '${execution.id}'
+							`,
+							{ data: JSON.stringify(data) },
+							{},
+						);
+
+					queryRunner.query(updateQuery, updateParams);
+				}
+			});
+		});
+
+		const retryableExecutions = await queryRunner.query(`
+			SELECT id, workflowData
+			FROM ${tablePrefix}execution_entity
+			WHERE waitTill IS NULL AND finished = 0 AND mode != 'retry'
+			ORDER BY startedAt DESC
+			LIMIT 200
+		`);
+		// @ts-ignore
+		retryableExecutions.forEach(async (execution) => {
+			const data = execution.workflowData;
+			let credentialsUpdated = false;
+			// @ts-ignore
+			data.nodes.forEach((node) => {
+				if (node.credentials) {
+					const allNodeCredentials = Object.entries(node.credentials);
+					for (const [type, name] of allNodeCredentials) {
+						if (typeof name === 'string') {
+							const matchingCredentials = credentialsEntities.find(
+								// @ts-ignore
+								(credentials) => credentials.name === name && credentials.type === type,
+							);
+							node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name };
+							credentialsUpdated = true;
+						}
+					}
+				}
+			});
+			if (credentialsUpdated) {
+				const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters(
+					`
+						UPDATE ${tablePrefix}execution_entity
+						SET workflowData = :data
+						WHERE id = '${execution.id}'
+					`,
+					{ data: JSON.stringify(data) },
+					{},
+				);
+
+				queryRunner.query(updateQuery, updateParams);
+			}
+		});
+		console.timeEnd(this.name);
+	}
+
+	public async down(queryRunner: QueryRunner): Promise<void> {
+		const tablePrefix = config.get('database.tablePrefix');
+		const helpers = new MigrationHelpers(queryRunner);
+
+		const credentialsEntities = await queryRunner.query(`
+			SELECT id, name, type
+			FROM ${tablePrefix}credentials_entity
+		`);
+
+		const workflowsQuery = `
+			SELECT id, nodes
+			FROM ${tablePrefix}workflow_entity
+		`;
+		// @ts-ignore
+		await helpers.runChunked(workflowsQuery, (workflows) => {
+			workflows.forEach(async (workflow) => {
+				const nodes = workflow.nodes;
+				let credentialsUpdated = false;
+				// @ts-ignore
+				nodes.forEach((node) => {
+					if (node.credentials) {
+						const allNodeCredentials = Object.entries(node.credentials);
+						for (const [type, creds] of allNodeCredentials) {
+							if (typeof creds === 'object') {
+								const matchingCredentials = credentialsEntities.find(
+									// @ts-ignore
+									(credentials) => credentials.id === creds.id && credentials.type === type,
+								);
+								if (matchingCredentials) {
+									node.credentials[type] = matchingCredentials.name;
+								} else {
+									// @ts-ignore
+									node.credentials[type] = creds.name;
+								}
+								credentialsUpdated = true;
+							}
+						}
+					}
+				});
+				if (credentialsUpdated) {
+					const [updateQuery, updateParams] =
+						queryRunner.connection.driver.escapeQueryWithParameters(
+							`
+								UPDATE ${tablePrefix}workflow_entity
+								SET nodes = :nodes
+								WHERE id = '${workflow.id}'
+							`,
+							{ nodes: JSON.stringify(nodes) },
+							{},
+						);
+
+					queryRunner.query(updateQuery, updateParams);
+				}
+			});
+		});
+
+		const waitingExecutionsQuery = `
+			SELECT id, workflowData
+			FROM ${tablePrefix}execution_entity
+			WHERE waitTill IS NOT NULL AND finished = 0
+		`;
+		// @ts-ignore
+		await helpers.runChunked(waitingExecutionsQuery, (waitingExecutions) => {
+			waitingExecutions.forEach(async (execution) => {
+				const data = execution.workflowData;
+				let credentialsUpdated = false;
+				// @ts-ignore
+				data.nodes.forEach((node) => {
+					if (node.credentials) {
+						const allNodeCredentials = Object.entries(node.credentials);
+						for (const [type, creds] of allNodeCredentials) {
+							if (typeof creds === 'object') {
+								// @ts-ignore
+								const matchingCredentials = credentialsEntities.find(
+									// @ts-ignore
+									(credentials) => credentials.id === creds.id && credentials.type === type,
+								);
+								if (matchingCredentials) {
+									node.credentials[type] = matchingCredentials.name;
+								} else {
+									// @ts-ignore
+									node.credentials[type] = creds.name;
+								}
+								credentialsUpdated = true;
+							}
+						}
+					}
+				});
+				if (credentialsUpdated) {
+					const [updateQuery, updateParams] =
+						queryRunner.connection.driver.escapeQueryWithParameters(
+							`
+								UPDATE ${tablePrefix}execution_entity
+								SET workflowData = :data
+								WHERE id = '${execution.id}'
+							`,
+							{ data: JSON.stringify(data) },
+							{},
+						);
+
+					queryRunner.query(updateQuery, updateParams);
+				}
+			});
+		});
+
+		const retryableExecutions = await queryRunner.query(`
+			SELECT id, workflowData
+			FROM ${tablePrefix}execution_entity
+			WHERE waitTill IS NULL AND finished = 0 AND mode != 'retry'
+			ORDER BY startedAt DESC
+			LIMIT 200
+		`);
+		// @ts-ignore
+		retryableExecutions.forEach(async (execution) => {
+			const data = execution.workflowData;
+			let credentialsUpdated = false;
+			// @ts-ignore
+			data.nodes.forEach((node) => {
+				if (node.credentials) {
+					const allNodeCredentials = Object.entries(node.credentials);
+					for (const [type, creds] of allNodeCredentials) {
+						if (typeof creds === 'object') {
+							// @ts-ignore
+							const matchingCredentials = credentialsEntities.find(
+								// @ts-ignore
+								(credentials) => credentials.id === creds.id && credentials.type === type,
+							);
+							if (matchingCredentials) {
+								node.credentials[type] = matchingCredentials.name;
+							} else {
+								// @ts-ignore
+								node.credentials[type] = creds.name;
+							}
+							credentialsUpdated = true;
+						}
+					}
+				}
+			});
+			if (credentialsUpdated) {
+				const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters(
+					`
+						UPDATE ${tablePrefix}execution_entity
+						SET workflowData = :data
+						WHERE id = '${execution.id}'
+					`,
+					{ data: JSON.stringify(data) },
+					{},
+				);
+
+				queryRunner.query(updateQuery, updateParams);
+			}
+		});
+	}
+}

packages/cli/src/databases/mysqldb/migrations/index.ts

@@ -9,6 +9,7 @@ import { CreateTagEntity1617268711084 } from './1617268711084-CreateTagEntity';
 import { UniqueWorkflowNames1620826335440 } from './1620826335440-UniqueWorkflowNames';
 import { CertifyCorrectCollation1623936588000 } from './1623936588000-CertifyCorrectCollation';
 import { AddWaitColumnId1626183952959 } from './1626183952959-AddWaitColumn';
+import { UpdateWorkflowCredentials1630451444017 } from './1630451444017-UpdateWorkflowCredentials';
 
 export const mysqlMigrations = [
 	InitialMigration1588157391238,
@@ -22,4 +23,5 @@ export const mysqlMigrations = [
 	UniqueWorkflowNames1620826335440,
 	CertifyCorrectCollation1623936588000,
 	AddWaitColumnId1626183952959,
+	UpdateWorkflowCredentials1630451444017,
 ];

packages/cli/src/databases/postgresdb/migrations/1630419189837-UpdateWorkflowCredentials.ts

@@ -0,0 +1,315 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+import config = require('../../../../config');
+import { MigrationHelpers } from '../../MigrationHelpers';
+
+// replacing the credentials in workflows and execution
+// `nodeType: name` changes to `nodeType: { id, name }`
+
+export class UpdateWorkflowCredentials1630419189837 implements MigrationInterface {
+	name = 'UpdateWorkflowCredentials1630419189837';
+
+	public async up(queryRunner: QueryRunner): Promise<void> {
+		console.log('Start migration', this.name);
+		console.time(this.name);
+		let tablePrefix = config.get('database.tablePrefix');
+		const schema = config.get('database.postgresdb.schema');
+		if (schema) {
+			tablePrefix = schema + '.' + tablePrefix;
+		}
+		const helpers = new MigrationHelpers(queryRunner);
+
+		const credentialsEntities = await queryRunner.query(`
+			SELECT id, name, type
+			FROM ${tablePrefix}credentials_entity
+		`);
+
+		const workflowsQuery = `
+			SELECT id, nodes
+			FROM ${tablePrefix}workflow_entity
+		`;
+		// @ts-ignore
+		await helpers.runChunked(workflowsQuery, (workflows) => {
+			workflows.forEach(async (workflow) => {
+				const nodes = workflow.nodes;
+				let credentialsUpdated = false;
+				// @ts-ignore
+				nodes.forEach((node) => {
+					if (node.credentials) {
+						const allNodeCredentials = Object.entries(node.credentials);
+						for (const [type, name] of allNodeCredentials) {
+							if (typeof name === 'string') {
+								const matchingCredentials = credentialsEntities.find(
+									// @ts-ignore
+									(credentials) => credentials.name === name && credentials.type === type,
+								);
+								node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name };
+								credentialsUpdated = true;
+							}
+						}
+					}
+				});
+				if (credentialsUpdated) {
+					const [updateQuery, updateParams] =
+						queryRunner.connection.driver.escapeQueryWithParameters(
+							`
+								UPDATE ${tablePrefix}workflow_entity
+								SET nodes = :nodes
+								WHERE id = '${workflow.id}'
+							`,
+							{ nodes: JSON.stringify(nodes) },
+							{},
+						);
+
+					queryRunner.query(updateQuery, updateParams);
+				}
+			});
+		});
+
+		const waitingExecutionsQuery = `
+			SELECT id, "workflowData"
+			FROM ${tablePrefix}execution_entity
+			WHERE "waitTill" IS NOT NULL AND finished = FALSE
+		`;
+		// @ts-ignore
+		await helpers.runChunked(waitingExecutionsQuery, (waitingExecutions) => {
+			waitingExecutions.forEach(async (execution) => {
+				const data = execution.workflowData;
+				let credentialsUpdated = false;
+				// @ts-ignore
+				data.nodes.forEach((node) => {
+					if (node.credentials) {
+						const allNodeCredentials = Object.entries(node.credentials);
+						for (const [type, name] of allNodeCredentials) {
+							if (typeof name === 'string') {
+								const matchingCredentials = credentialsEntities.find(
+									// @ts-ignore
+									(credentials) => credentials.name === name && credentials.type === type,
+								);
+								node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name };
+								credentialsUpdated = true;
+							}
+						}
+					}
+				});
+				if (credentialsUpdated) {
+					const [updateQuery, updateParams] =
+						queryRunner.connection.driver.escapeQueryWithParameters(
+							`
+							UPDATE ${tablePrefix}execution_entity
+							SET "workflowData" = :data
+							WHERE id = '${execution.id}'
+						`,
+							{ data: JSON.stringify(data) },
+							{},
+						);
+
+					queryRunner.query(updateQuery, updateParams);
+				}
+			});
+		});
+
+		const retryableExecutions = await queryRunner.query(`
+			SELECT id, "workflowData"
+			FROM ${tablePrefix}execution_entity
+			WHERE "waitTill" IS NULL AND finished = FALSE AND mode != 'retry'
+			ORDER BY "startedAt" DESC
+			LIMIT 200
+		`);
+
+		// @ts-ignore
+		retryableExecutions.forEach(async (execution) => {
+			const data = execution.workflowData;
+			let credentialsUpdated = false;
+			// @ts-ignore
+			data.nodes.forEach((node) => {
+				if (node.credentials) {
+					const allNodeCredentials = Object.entries(node.credentials);
+					for (const [type, name] of allNodeCredentials) {
+						if (typeof name === 'string') {
+							// @ts-ignore
+							const matchingCredentials = credentialsEntities.find(
+								// @ts-ignore
+								(credentials) => credentials.name === name && credentials.type === type,
+							);
+							node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name };
+							credentialsUpdated = true;
+						}
+					}
+				}
+			});
+			if (credentialsUpdated) {
+				const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters(
+					`
+				UPDATE ${tablePrefix}execution_entity
+				SET "workflowData" = :data
+				WHERE id = '${execution.id}'
+				`,
+					{ data: JSON.stringify(data) },
+					{},
+				);
+
+				queryRunner.query(updateQuery, updateParams);
+			}
+		});
+		console.timeEnd(this.name);
+	}
+
+	public async down(queryRunner: QueryRunner): Promise<void> {
+		let tablePrefix = config.get('database.tablePrefix');
+		const schema = config.get('database.postgresdb.schema');
+		if (schema) {
+			tablePrefix = schema + '.' + tablePrefix;
+		}
+		const helpers = new MigrationHelpers(queryRunner);
+
+		const credentialsEntities = await queryRunner.query(`
+			SELECT id, name, type
+			FROM ${tablePrefix}credentials_entity
+		`);
+
+		const workflowsQuery = `
+			SELECT id, nodes
+			FROM ${tablePrefix}workflow_entity
+		`;
+		// @ts-ignore
+		await helpers.runChunked(workflowsQuery, (workflows) => {
+			workflows.forEach(async (workflow) => {
+				const nodes = workflow.nodes;
+				let credentialsUpdated = false;
+				// @ts-ignore
+				nodes.forEach((node) => {
+					if (node.credentials) {
+						const allNodeCredentials = Object.entries(node.credentials);
+						for (const [type, creds] of allNodeCredentials) {
+							if (typeof creds === 'object') {
+								// @ts-ignore
+								const matchingCredentials = credentialsEntities.find(
+									// @ts-ignore
+									(credentials) => credentials.id === creds.id && credentials.type === type,
+								);
+								if (matchingCredentials) {
+									node.credentials[type] = matchingCredentials.name;
+								} else {
+									// @ts-ignore
+									node.credentials[type] = creds.name;
+								}
+								credentialsUpdated = true;
+							}
+						}
+					}
+				});
+				if (credentialsUpdated) {
+					const [updateQuery, updateParams] =
+						queryRunner.connection.driver.escapeQueryWithParameters(
+							`
+								UPDATE ${tablePrefix}workflow_entity
+								SET nodes = :nodes
+								WHERE id = '${workflow.id}'
+							`,
+							{ nodes: JSON.stringify(nodes) },
+							{},
+						);
+
+					queryRunner.query(updateQuery, updateParams);
+				}
+			});
+		});
+
+		const waitingExecutionsQuery = `
+			SELECT id, "workflowData"
+			FROM ${tablePrefix}execution_entity
+			WHERE "waitTill" IS NOT NULL AND finished = FALSE
+		`;
+		// @ts-ignore
+		await helpers.runChunked(waitingExecutionsQuery, (waitingExecutions) => {
+			waitingExecutions.forEach(async (execution) => {
+				const data = execution.workflowData;
+				let credentialsUpdated = false;
+				// @ts-ignore
+				data.nodes.forEach((node) => {
+					if (node.credentials) {
+						const allNodeCredentials = Object.entries(node.credentials);
+						for (const [type, creds] of allNodeCredentials) {
+							if (typeof creds === 'object') {
+								// @ts-ignore
+								const matchingCredentials = credentialsEntities.find(
+									// @ts-ignore
+									(credentials) => credentials.id === creds.id && credentials.type === type,
+								);
+								if (matchingCredentials) {
+									node.credentials[type] = matchingCredentials.name;
+								} else {
+									// @ts-ignore
+									node.credentials[type] = creds.name;
+								}
+								credentialsUpdated = true;
+							}
+						}
+					}
+				});
+				if (credentialsUpdated) {
+					const [updateQuery, updateParams] =
+						queryRunner.connection.driver.escapeQueryWithParameters(
+							`
+								UPDATE ${tablePrefix}execution_entity
+								SET "workflowData" = :data
+								WHERE id = '${execution.id}'
+							`,
+							{ data: JSON.stringify(data) },
+							{},
+						);
+
+					queryRunner.query(updateQuery, updateParams);
+				}
+			});
+		});
+
+		const retryableExecutions = await queryRunner.query(`
+			SELECT id, "workflowData"
+			FROM ${tablePrefix}execution_entity
+			WHERE "waitTill" IS NULL AND finished = FALSE AND mode != 'retry'
+			ORDER BY "startedAt" DESC
+			LIMIT 200
+		`);
+		// @ts-ignore
+		retryableExecutions.forEach(async (execution) => {
+			const data = execution.workflowData;
+			let credentialsUpdated = false;
+			// @ts-ignore
+			data.nodes.forEach((node) => {
+				if (node.credentials) {
+					const allNodeCredentials = Object.entries(node.credentials);
+					for (const [type, creds] of allNodeCredentials) {
+						if (typeof creds === 'object') {
+							// @ts-ignore
+							const matchingCredentials = credentialsEntities.find(
+								// @ts-ignore
+								(credentials) => credentials.id === creds.id && credentials.type === type,
+							);
+							if (matchingCredentials) {
+								node.credentials[type] = matchingCredentials.name;
+							} else {
+								// @ts-ignore
+								node.credentials[type] = creds.name;
+							}
+							credentialsUpdated = true;
+						}
+					}
+				}
+			});
+			if (credentialsUpdated) {
+				const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters(
+					`
+						UPDATE ${tablePrefix}execution_entity
+						SET "workflowData" = :data
+						WHERE id = '${execution.id}'
+					`,
+					{ data: JSON.stringify(data) },
+					{},
+				);
+
+				queryRunner.query(updateQuery, updateParams);
+			}
+		});
+	}
+}

packages/cli/src/databases/postgresdb/migrations/index.ts

@@ -6,6 +6,7 @@ import { MakeStoppedAtNullable1607431743768 } from './1607431743768-MakeStoppedA
 import { CreateTagEntity1617270242566 } from './1617270242566-CreateTagEntity';
 import { UniqueWorkflowNames1620824779533 } from './1620824779533-UniqueWorkflowNames';
 import { AddwaitTill1626176912946 } from './1626176912946-AddwaitTill';
+import { UpdateWorkflowCredentials1630419189837 } from './1630419189837-UpdateWorkflowCredentials';
 
 export const postgresMigrations = [
 	InitialMigration1587669153312,
@@ -16,4 +17,5 @@ export const postgresMigrations = [
 	CreateTagEntity1617270242566,
 	UniqueWorkflowNames1620824779533,
 	AddwaitTill1626176912946,
+	UpdateWorkflowCredentials1630419189837,
 ];

packages/cli/src/databases/sqlite/migrations/1630330987096-UpdateWorkflowCredentials.ts

@@ -0,0 +1,308 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+import config = require('../../../../config');
+import { MigrationHelpers } from '../../MigrationHelpers';
+
+// replacing the credentials in workflows and execution
+// `nodeType: name` changes to `nodeType: { id, name }`
+
+export class UpdateWorkflowCredentials1630330987096 implements MigrationInterface {
+	name = 'UpdateWorkflowCredentials1630330987096';
+
+	public async up(queryRunner: QueryRunner): Promise<void> {
+		console.log('Start migration', this.name);
+		console.time(this.name);
+		const tablePrefix = config.get('database.tablePrefix');
+		const helpers = new MigrationHelpers(queryRunner);
+
+		const credentialsEntities = await queryRunner.query(`
+			SELECT id, name, type
+			FROM "${tablePrefix}credentials_entity"
+		`);
+
+		const workflowsQuery = `
+			SELECT id, nodes
+			FROM "${tablePrefix}workflow_entity"
+		`;
+
+		// @ts-ignore
+		await helpers.runChunked(workflowsQuery, (workflows) => {
+			workflows.forEach(async (workflow) => {
+				const nodes = JSON.parse(workflow.nodes);
+				let credentialsUpdated = false;
+				// @ts-ignore
+				nodes.forEach((node) => {
+					if (node.credentials) {
+						const allNodeCredentials = Object.entries(node.credentials);
+						for (const [type, name] of allNodeCredentials) {
+							if (typeof name === 'string') {
+								const matchingCredentials = credentialsEntities.find(
+									// @ts-ignore
+									(credentials) => credentials.name === name && credentials.type === type,
+								);
+								node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name };
+								credentialsUpdated = true;
+							}
+						}
+					}
+				});
+				if (credentialsUpdated) {
+					const [updateQuery, updateParams] =
+						queryRunner.connection.driver.escapeQueryWithParameters(
+							`
+								UPDATE "${tablePrefix}workflow_entity"
+								SET nodes = :nodes
+								WHERE id = '${workflow.id}'
+							`,
+							{ nodes: JSON.stringify(nodes) },
+							{},
+						);
+
+					queryRunner.query(updateQuery, updateParams);
+				}
+			});
+		});
+
+		const waitingExecutionsQuery = `
+			SELECT id, "workflowData"
+			FROM "${tablePrefix}execution_entity"
+			WHERE "waitTill" IS NOT NULL AND finished = 0
+		`;
+		// @ts-ignore
+		await helpers.runChunked(waitingExecutionsQuery, (waitingExecutions) => {
+			waitingExecutions.forEach(async (execution) => {
+				const data = JSON.parse(execution.workflowData);
+				let credentialsUpdated = false;
+				// @ts-ignore
+				data.nodes.forEach((node) => {
+					if (node.credentials) {
+						const allNodeCredentials = Object.entries(node.credentials);
+						for (const [type, name] of allNodeCredentials) {
+							if (typeof name === 'string') {
+								const matchingCredentials = credentialsEntities.find(
+									// @ts-ignore
+									(credentials) => credentials.name === name && credentials.type === type,
+								);
+								node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name };
+								credentialsUpdated = true;
+							}
+						}
+					}
+				});
+				if (credentialsUpdated) {
+					const [updateQuery, updateParams] =
+						queryRunner.connection.driver.escapeQueryWithParameters(
+							`
+								UPDATE "${tablePrefix}execution_entity"
+								SET "workflowData" = :data
+								WHERE id = '${execution.id}'
+							`,
+							{ data: JSON.stringify(data) },
+							{},
+						);
+
+					queryRunner.query(updateQuery, updateParams);
+				}
+			});
+		});
+
+		const retryableExecutions = await queryRunner.query(`
+			SELECT id, "workflowData"
+			FROM "${tablePrefix}execution_entity"
+			WHERE "waitTill" IS NULL AND finished = 0 AND mode != 'retry'
+			ORDER BY "startedAt" DESC
+			LIMIT 200
+		`);
+		// @ts-ignore
+		retryableExecutions.forEach(async (execution) => {
+			const data = JSON.parse(execution.workflowData);
+			let credentialsUpdated = false;
+			// @ts-ignore
+			data.nodes.forEach((node) => {
+				if (node.credentials) {
+					const allNodeCredentials = Object.entries(node.credentials);
+					for (const [type, name] of allNodeCredentials) {
+						if (typeof name === 'string') {
+							const matchingCredentials = credentialsEntities.find(
+								// @ts-ignore
+								(credentials) => credentials.name === name && credentials.type === type,
+							);
+							node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name };
+							credentialsUpdated = true;
+						}
+					}
+				}
+			});
+			if (credentialsUpdated) {
+				const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters(
+					`
+						UPDATE "${tablePrefix}execution_entity"
+						SET "workflowData" = :data
+						WHERE id = '${execution.id}'
+					`,
+					{ data: JSON.stringify(data) },
+					{},
+				);
+
+				queryRunner.query(updateQuery, updateParams);
+			}
+		});
+		console.timeEnd(this.name);
+	}
+
+	public async down(queryRunner: QueryRunner): Promise<void> {
+		const tablePrefix = config.get('database.tablePrefix');
+		const helpers = new MigrationHelpers(queryRunner);
+
+		const credentialsEntities = await queryRunner.query(`
+			SELECT id, name, type
+			FROM "${tablePrefix}credentials_entity"
+		`);
+
+		const workflowsQuery = `
+			SELECT id, nodes
+			FROM "${tablePrefix}workflow_entity"
+		`;
+
+		// @ts-ignore
+		await helpers.runChunked(workflowsQuery, (workflows) => {
+			// @ts-ignore
+			workflows.forEach(async (workflow) => {
+				const nodes = JSON.parse(workflow.nodes);
+				let credentialsUpdated = false;
+				// @ts-ignore
+				nodes.forEach((node) => {
+					if (node.credentials) {
+						const allNodeCredentials = Object.entries(node.credentials);
+						for (const [type, creds] of allNodeCredentials) {
+							if (typeof creds === 'object') {
+								const matchingCredentials = credentialsEntities.find(
+									// @ts-ignore double-equals because creds.id can be string or number
+									(credentials) => credentials.id == creds.id && credentials.type === type,
+								);
+								if (matchingCredentials) {
+									node.credentials[type] = matchingCredentials.name;
+								} else {
+									// @ts-ignore
+									node.credentials[type] = creds.name;
+								}
+								credentialsUpdated = true;
+							}
+						}
+					}
+				});
+				if (credentialsUpdated) {
+					const [updateQuery, updateParams] =
+						queryRunner.connection.driver.escapeQueryWithParameters(
+							`
+								UPDATE "${tablePrefix}workflow_entity"
+								SET nodes = :nodes
+								WHERE id = '${workflow.id}'
+							`,
+							{ nodes: JSON.stringify(nodes) },
+							{},
+						);
+
+					queryRunner.query(updateQuery, updateParams);
+				}
+			});
+		});
+
+		const waitingExecutionsQuery = `
+			SELECT id, "workflowData"
+			FROM "${tablePrefix}execution_entity"
+			WHERE "waitTill" IS NOT NULL AND finished = 0
+		`;
+
+		// @ts-ignore
+		await helpers.runChunked(waitingExecutionsQuery, (waitingExecutions) => {
+			// @ts-ignore
+			waitingExecutions.forEach(async (execution) => {
+				const data = JSON.parse(execution.workflowData);
+				let credentialsUpdated = false;
+				// @ts-ignore
+				data.nodes.forEach((node) => {
+					if (node.credentials) {
+						const allNodeCredentials = Object.entries(node.credentials);
+						for (const [type, creds] of allNodeCredentials) {
+							if (typeof creds === 'object') {
+								const matchingCredentials = credentialsEntities.find(
+									// @ts-ignore double-equals because creds.id can be string or number
+									(credentials) => credentials.id == creds.id && credentials.type === type,
+								);
+								if (matchingCredentials) {
+									node.credentials[type] = matchingCredentials.name;
+								} else {
+									// @ts-ignore
+									node.credentials[type] = creds.name;
+								}
+								credentialsUpdated = true;
+							}
+						}
+					}
+				});
+				if (credentialsUpdated) {
+					const [updateQuery, updateParams] =
+						queryRunner.connection.driver.escapeQueryWithParameters(
+							`
+								UPDATE "${tablePrefix}execution_entity"
+								SET "workflowData" = :data
+								WHERE id = '${execution.id}'
+							`,
+							{ data: JSON.stringify(data) },
+							{},
+						);
+
+					await queryRunner.query(updateQuery, updateParams);
+				}
+			});
+		});
+
+		const retryableExecutions = await queryRunner.query(`
+			SELECT id, "workflowData"
+			FROM "${tablePrefix}execution_entity"
+			WHERE "waitTill" IS NULL AND finished = 0 AND mode != 'retry'
+			ORDER BY "startedAt" DESC
+			LIMIT 200
+		`);
+
+		// @ts-ignore
+		retryableExecutions.forEach(async (execution) => {
+			const data = JSON.parse(execution.workflowData);
+			let credentialsUpdated = false;
+			// @ts-ignore
+			data.nodes.forEach((node) => {
+				if (node.credentials) {
+					const allNodeCredentials = Object.entries(node.credentials);
+					for (const [type, creds] of allNodeCredentials) {
+						if (typeof creds === 'object') {
+							const matchingCredentials = credentialsEntities.find(
+								// @ts-ignore double-equals because creds.id can be string or number
+								(credentials) => credentials.id == creds.id && credentials.type === type,
+							);
+							if (matchingCredentials) {
+								node.credentials[type] = matchingCredentials.name;
+							} else {
+								// @ts-ignore
+								node.credentials[type] = creds.name;
+							}
+							credentialsUpdated = true;
+						}
+					}
+				}
+			});
+			if (credentialsUpdated) {
+				const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters(
+					`
+						UPDATE "${tablePrefix}execution_entity"
+						SET "workflowData" = :data
+						WHERE id = '${execution.id}'
+					`,
+					{ data: JSON.stringify(data) },
+					{},
+				);
+
+				queryRunner.query(updateQuery, updateParams);
+			}
+		});
+	}
+}

packages/cli/src/databases/sqlite/migrations/index.ts

@@ -6,6 +6,7 @@ import { MakeStoppedAtNullable1607431743769 } from './1607431743769-MakeStoppedA
 import { CreateTagEntity1617213344594 } from './1617213344594-CreateTagEntity';
 import { UniqueWorkflowNames1620821879465 } from './1620821879465-UniqueWorkflowNames';
 import { AddWaitColumn1621707690587 } from './1621707690587-AddWaitColumn';
+import { UpdateWorkflowCredentials1630330987096 } from './1630330987096-UpdateWorkflowCredentials';
 
 export const sqliteMigrations = [
 	InitialMigration1588102412422,
@@ -16,4 +17,5 @@ export const sqliteMigrations = [
 	CreateTagEntity1617213344594,
 	UniqueWorkflowNames1620821879465,
 	AddWaitColumn1621707690587,
+	UpdateWorkflowCredentials1630330987096,
 ];

packages/cli/src/databases/utils.ts

@@ -1,42 +0,0 @@
-/* eslint-disable import/no-cycle */
-import { DatabaseType } from '../index';
-import { getConfigValueSync } from '../GenericHelpers';
-
-/**
- * Resolves the data type for the used database type
- *
- * @export
- * @param {string} dataType
- * @returns {string}
- */
-// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
-export function resolveDataType(dataType: string) {
-	const dbType = getConfigValueSync('database.type') as DatabaseType;
-
-	const typeMap: { [key in DatabaseType]: { [key: string]: string } } = {
-		sqlite: {
-			json: 'simple-json',
-		},
-		postgresdb: {
-			datetime: 'timestamptz',
-		},
-		mysqldb: {},
-		mariadb: {},
-	};
-
-	return typeMap[dbType][dataType] ?? dataType;
-}
-
-// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
-export function getTimestampSyntax() {
-	const dbType = getConfigValueSync('database.type') as DatabaseType;
-
-	const map: { [key in DatabaseType]: string } = {
-		sqlite: "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')",
-		postgresdb: 'CURRENT_TIMESTAMP(3)',
-		mysqldb: 'CURRENT_TIMESTAMP(3)',
-		mariadb: 'CURRENT_TIMESTAMP(3)',
-	};
-
-	return map[dbType];
-}

packages/cli/src/index.ts

@@ -5,6 +5,7 @@ export * from './CredentialTypes';
 export * from './CredentialsOverwrites';
 export * from './ExternalHooks';
 export * from './Interfaces';
+export * from './InternalHooksManager';
 export * from './LoadNodesAndCredentials';
 export * from './NodeTypes';
 export * from './WaitTracker';

packages/cli/src/telemetry/index.ts

@@ -0,0 +1,153 @@
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+import TelemetryClient = require('@rudderstack/rudder-sdk-node');
+import { IDataObject, LoggerProxy } from 'n8n-workflow';
+import config = require('../../config');
+import { getLogger } from '../Logger';
+
+interface IExecutionCountsBufferItem {
+	manual_success_count: number;
+	manual_error_count: number;
+	prod_success_count: number;
+	prod_error_count: number;
+}
+
+interface IExecutionCountsBuffer {
+	[workflowId: string]: IExecutionCountsBufferItem;
+}
+
+export class Telemetry {
+	private client?: TelemetryClient;
+
+	private instanceId: string;
+
+	private pulseIntervalReference: NodeJS.Timeout;
+
+	private executionCountsBuffer: IExecutionCountsBuffer = {};
+
+	constructor(instanceId: string) {
+		this.instanceId = instanceId;
+
+		const enabled = config.get('diagnostics.enabled') as boolean;
+		if (enabled) {
+			const conf = config.get('diagnostics.config.backend') as string;
+			const [key, url] = conf.split(';');
+
+			if (!key || !url) {
+				const logger = getLogger();
+				LoggerProxy.init(logger);
+				logger.warn('Diagnostics backend config is invalid');
+				return;
+			}
+
+			this.client = new TelemetryClient(key, url);
+
+			this.pulseIntervalReference = setInterval(async () => {
+				void this.pulse();
+			}, 6 * 60 * 60 * 1000); // every 6 hours
+		}
+	}
+
+	private async pulse(): Promise<unknown> {
+		if (!this.client) {
+			return Promise.resolve();
+		}
+
+		const allPromises = Object.keys(this.executionCountsBuffer).map(async (workflowId) => {
+			const promise = this.track('Workflow execution count', {
+				workflow_id: workflowId,
+				...this.executionCountsBuffer[workflowId],
+			});
+			this.executionCountsBuffer[workflowId].manual_error_count = 0;
+			this.executionCountsBuffer[workflowId].manual_success_count = 0;
+			this.executionCountsBuffer[workflowId].prod_error_count = 0;
+			this.executionCountsBuffer[workflowId].prod_success_count = 0;
+
+			return promise;
+		});
+
+		allPromises.push(this.track('pulse'));
+		return Promise.all(allPromises);
+	}
+
+	async trackWorkflowExecution(properties: IDataObject): Promise<void> {
+		if (this.client) {
+			const workflowId = properties.workflow_id as string;
+			this.executionCountsBuffer[workflowId] = this.executionCountsBuffer[workflowId] ?? {
+				manual_error_count: 0,
+				manual_success_count: 0,
+				prod_error_count: 0,
+				prod_success_count: 0,
+			};
+
+			if (
+				properties.success === false &&
+				properties.error_node_type &&
+				(properties.error_node_type as string).startsWith('n8n-nodes-base')
+			) {
+				// errored exec
+				void this.track('Workflow execution errored', properties);
+
+				if (properties.is_manual) {
+					this.executionCountsBuffer[workflowId].manual_error_count++;
+				} else {
+					this.executionCountsBuffer[workflowId].prod_error_count++;
+				}
+			} else if (properties.is_manual) {
+				this.executionCountsBuffer[workflowId].manual_success_count++;
+			} else {
+				this.executionCountsBuffer[workflowId].prod_success_count++;
+			}
+		}
+	}
+
+	async trackN8nStop(): Promise<void> {
+		clearInterval(this.pulseIntervalReference);
+		void this.track('User instance stopped');
+		return new Promise<void>((resolve) => {
+			if (this.client) {
+				this.client.flush(resolve);
+			} else {
+				resolve();
+			}
+		});
+	}
+
+	async identify(traits?: IDataObject): Promise<void> {
+		return new Promise<void>((resolve) => {
+			if (this.client) {
+				this.client.identify(
+					{
+						userId: this.instanceId,
+						anonymousId: '000000000000',
+						traits: {
+							...traits,
+							instanceId: this.instanceId,
+						},
+					},
+					resolve,
+				);
+			} else {
+				resolve();
+			}
+		});
+	}
+
+	async track(eventName: string, properties?: IDataObject): Promise<void> {
+		return new Promise<void>((resolve) => {
+			if (this.client) {
+				this.client.track(
+					{
+						userId: this.instanceId,
+						anonymousId: '000000000000',
+						event: eventName,
+						properties,
+					},
+					resolve,
+				);
+			} else {
+				resolve();
+			}
+		});
+	}
+}

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "n8n-core",
-  "version": "0.84.0",
+  "version": "0.95.0",
   "description": "Core functionality of n8n",
   "license": "SEE LICENSE IN LICENSE.md",
   "homepage": "https://n8n.io",
@@ -27,13 +27,13 @@
     "dist"
   ],
   "devDependencies": {
-    "@types/cron": "^1.7.1",
+    "@types/cron": "~1.7.1",
     "@types/crypto-js": "^4.0.1",
     "@types/express": "^4.17.6",
     "@types/jest": "^26.0.13",
     "@types/lodash.get": "^4.4.6",
     "@types/mime-types": "^2.1.0",
-    "@types/node": "^14.14.40",
+    "@types/node": "14.17.27",
     "@types/request-promise-native": "~1.0.15",
     "jest": "^26.4.2",
     "source-map-support": "^0.5.9",
@@ -44,13 +44,13 @@
   "dependencies": {
     "axios": "^0.21.1",
     "client-oauth2": "^4.2.5",
-    "cron": "^1.7.2",
+    "cron": "~1.7.2",
     "crypto-js": "~4.1.1",
     "file-type": "^14.6.2",
     "form-data": "^4.0.0",
     "lodash.get": "^4.4.2",
     "mime-types": "^2.1.27",
-    "n8n-workflow": "~0.70.0",
+    "n8n-workflow": "~0.78.0",
     "oauth-1.0a": "^2.2.6",
     "p-cancelable": "^2.0.0",
     "qs": "^6.10.1",

packages/core/src/ActiveWorkflows.ts

@@ -194,7 +194,7 @@ export class ActiveWorkflows {
 		// The trigger function to execute when the cron-time got reached
 		const executeTrigger = async () => {
 			// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-			Logger.info(`Polling trigger initiated for workflow "${workflow.name}"`, {
+			Logger.debug(`Polling trigger initiated for workflow "${workflow.name}"`, {
 				workflowName: workflow.name,
 				workflowId: workflow.id,
 			});

packages/core/src/Credentials.ts

@@ -98,6 +98,7 @@ export class Credentials extends ICredentials {
 		}
 
 		return {
+			id: this.id,
 			name: this.name,
 			type: this.type,
 			data: this.data,

packages/core/src/Interfaces.ts

@@ -145,6 +145,7 @@ export interface ITriggerTime {
 export interface IUserSettings {
 	encryptionKey?: string;
 	tunnelSubdomain?: string;
+	instanceId?: string;
 }
 
 export interface ILoadOptionsFunctions extends ILoadOptionsFunctionsBase {

packages/core/src/NodeExecuteFunctions.ts

@@ -22,6 +22,7 @@ import {
 	ICredentialsExpressionResolveValues,
 	IDataObject,
 	IExecuteFunctions,
+	IExecuteResponsePromiseData,
 	IExecuteSingleFunctions,
 	IExecuteWorkflowInfo,
 	IHttpRequestOptions,
@@ -71,7 +72,7 @@ import { fromBuffer } from 'file-type';
 import { lookup } from 'mime-types';
 
 import axios, { AxiosProxyConfig, AxiosRequestConfig, Method } from 'axios';
-import { URLSearchParams } from 'url';
+import { URL, URLSearchParams } from 'url';
 // eslint-disable-next-line import/no-cycle
 import {
 	BINARY_ENCODING,
@@ -86,11 +87,78 @@ import {
 axios.defaults.timeout = 300000;
 // Prevent axios from adding x-form-www-urlencoded headers by default
 axios.defaults.headers.post = {};
+axios.defaults.headers.put = {};
+axios.defaults.headers.patch = {};
+axios.defaults.paramsSerializer = (params) => {
+	if (params instanceof URLSearchParams) {
+		return params.toString();
+	}
+	return stringify(params, { arrayFormat: 'indices' });
+};
 
 const requestPromiseWithDefaults = requestPromise.defaults({
 	timeout: 300000, // 5 minutes
 });
 
+const pushFormDataValue = (form: FormData, key: string, value: any) => {
+	if (value?.hasOwnProperty('value') && value.hasOwnProperty('options')) {
+		// @ts-ignore
+		form.append(key, value.value, value.options);
+	} else {
+		form.append(key, value);
+	}
+};
+
+const createFormDataObject = (data: object) => {
+	const formData = new FormData();
+	const keys = Object.keys(data);
+	keys.forEach((key) => {
+		// @ts-ignore
+		const formField = data[key];
+
+		if (formField instanceof Array) {
+			formField.forEach((item) => {
+				pushFormDataValue(formData, key, item);
+			});
+		} else {
+			pushFormDataValue(formData, key, formField);
+		}
+	});
+	return formData;
+};
+
+function searchForHeader(headers: IDataObject, headerName: string) {
+	if (headers === undefined) {
+		return undefined;
+	}
+
+	const headerNames = Object.keys(headers);
+	headerName = headerName.toLowerCase();
+	return headerNames.find((thisHeader) => thisHeader.toLowerCase() === headerName);
+}
+
+async function generateContentLengthHeader(formData: FormData, headers: IDataObject) {
+	if (!formData || !formData.getLength) {
+		return;
+	}
+	try {
+		const length = await new Promise((res, rej) => {
+			formData.getLength((error: Error | null, length: number) => {
+				if (error) {
+					rej(error);
+					return;
+				}
+				res(length);
+			});
+		});
+		headers = Object.assign(headers, {
+			'content-length': length,
+		});
+	} catch (error) {
+		Logger.error('Unable to calculate form data length', { error });
+	}
+}
+
 async function parseRequestObject(requestObject: IDataObject) {
 	// This function is a temporary implementation
 	// That translates all http requests done via
@@ -125,42 +193,29 @@ async function parseRequestObject(requestObject: IDataObject) {
 		// and also using formData. Request lib takes precedence for the formData.
 		// We will do the same.
 		// Merge body and form properties.
-		// @ts-ignore
+		if (typeof requestObject.body === 'string') {
-		axiosConfig.data =
+			axiosConfig.data = requestObject.body;
-			typeof requestObject.body === 'string'
+		} else {
-				? requestObject.body
+			const allData = Object.assign(requestObject.body || {}, requestObject.form || {}) as Record<
-				: new URLSearchParams(
+				string,
-						Object.assign(requestObject.body || {}, requestObject.form || {}) as Record<
+				string
-							string,
+			>;
-							string
+			if (requestObject.useQuerystring === true) {
-						>,
+				axiosConfig.data = stringify(allData, { arrayFormat: 'repeat' });
-				  );
+			} else {
+				axiosConfig.data = stringify(allData);
+			}
+		}
 	} else if (contentType && contentType.includes('multipart/form-data') !== false) {
 		if (requestObject.formData !== undefined && requestObject.formData instanceof FormData) {
 			axiosConfig.data = requestObject.formData;
 		} else {
-			const allData = Object.assign(requestObject.body || {}, requestObject.formData || {});
+			const allData = {
+				...(requestObject.body as object | undefined),
+				...(requestObject.formData as object | undefined),
+			};
 
-			const objectKeys = Object.keys(allData);
+			axiosConfig.data = createFormDataObject(allData);
-			if (objectKeys.length > 0) {
-				// Should be a standard object. We must convert to formdata
-				const form = new FormData();
-
-				objectKeys.forEach((key) => {
-					const formField = (allData as IDataObject)[key] as IDataObject;
-					if (formField.hasOwnProperty('value') && formField.value instanceof Buffer) {
-						let filename;
-						// @ts-ignore
-						if (!!formField.options && formField.options.filename !== undefined) {
-							filename = (formField.options as IDataObject).filename as string;
-						}
-						form.append(key, formField.value, filename);
-					} else {
-						form.append(key, formField);
-					}
-				});
-				axiosConfig.data = form;
-			}
 		}
 		// replace the existing header with a new one that
 		// contains the boundary property.
@@ -168,17 +223,20 @@ async function parseRequestObject(requestObject: IDataObject) {
 		delete axiosConfig.headers[contentTypeHeaderKeyName];
 		const headers = axiosConfig.data.getHeaders();
 		axiosConfig.headers = Object.assign(axiosConfig.headers || {}, headers);
+		await generateContentLengthHeader(axiosConfig.data, axiosConfig.headers);
 	} else {
 		// When using the `form` property it means the content should be x-www-form-urlencoded.
 		if (requestObject.form !== undefined && requestObject.body === undefined) {
 			// If we have only form
-			axiosConfig.data = new URLSearchParams(requestObject.form as Record<string, string>);
+			axiosConfig.data =
+				typeof requestObject.form === 'string'
+					? stringify(requestObject.form, { format: 'RFC3986' })
+					: stringify(requestObject.form).toString();
 			if (axiosConfig.headers !== undefined) {
-				// remove possibly existing content-type headers
+				const headerName = searchForHeader(axiosConfig.headers, 'content-type');
-				const headers = Object.keys(axiosConfig.headers);
+				if (headerName) {
-				headers.forEach((header) =>
+					delete axiosConfig.headers[headerName];
-					header.toLowerCase() === 'content-type' ? delete axiosConfig.headers[header] : null,
+				}
-				);
 				axiosConfig.headers['Content-Type'] = 'application/x-www-form-urlencoded';
 			} else {
 				axiosConfig.headers = {
@@ -197,30 +255,12 @@ async function parseRequestObject(requestObject: IDataObject) {
 			if (requestObject.formData instanceof FormData) {
 				axiosConfig.data = requestObject.formData;
 			} else {
-				const objectKeys = Object.keys(requestObject.formData as object);
+				axiosConfig.data = createFormDataObject(requestObject.formData as object);
-				if (objectKeys.length > 0) {
-					// Should be a standard object. We must convert to formdata
-					const form = new FormData();
-
-					objectKeys.forEach((key) => {
-						const formField = (requestObject.formData as IDataObject)[key] as IDataObject;
-						if (formField.hasOwnProperty('value') && formField.value instanceof Buffer) {
-							let filename;
-							// @ts-ignore
-							if (!!formField.options && formField.options.filename !== undefined) {
-								filename = (formField.options as IDataObject).filename as string;
-							}
-							form.append(key, formField.value, filename);
-						} else {
-							form.append(key, formField);
-						}
-					});
-					axiosConfig.data = form;
-				}
 			}
 			// Mix in headers as FormData creates the boundary.
 			const headers = axiosConfig.data.getHeaders();
 			axiosConfig.headers = Object.assign(axiosConfig.headers || {}, headers);
+			await generateContentLengthHeader(axiosConfig.data, axiosConfig.headers);
 		} else if (requestObject.body !== undefined) {
 			// If we have body and possibly form
 			if (requestObject.form !== undefined) {
@@ -232,11 +272,11 @@ async function parseRequestObject(requestObject: IDataObject) {
 	}
 
 	if (requestObject.uri !== undefined) {
-		axiosConfig.url = requestObject.uri as string;
+		axiosConfig.url = requestObject.uri?.toString() as string;
 	}
 
 	if (requestObject.url !== undefined) {
-		axiosConfig.url = requestObject.url as string;
+		axiosConfig.url = requestObject.url?.toString() as string;
 	}
 
 	if (requestObject.method !== undefined) {
@@ -247,10 +287,25 @@ async function parseRequestObject(requestObject: IDataObject) {
 		axiosConfig.params = requestObject.qs as IDataObject;
 	}
 
-	if (requestObject.useQuerystring === true) {
+	if (
+		requestObject.useQuerystring === true ||
+		// @ts-ignore
+		requestObject.qsStringifyOptions?.arrayFormat === 'repeat'
+	) {
 		axiosConfig.paramsSerializer = (params) => {
 			return stringify(params, { arrayFormat: 'repeat' });
 		};
+	} else if (requestObject.useQuerystring === false) {
+		axiosConfig.paramsSerializer = (params) => {
+			return stringify(params, { arrayFormat: 'indices' });
+		};
+	}
+
+	// @ts-ignore
+	if (requestObject.qsStringifyOptions?.arrayFormat === 'brackets') {
+		axiosConfig.paramsSerializer = (params) => {
+			return stringify(params, { arrayFormat: 'brackets' });
+		};
 	}
 
 	if (requestObject.auth !== undefined) {
@@ -286,7 +341,7 @@ async function parseRequestObject(requestObject: IDataObject) {
 			});
 		}
 	}
-	if (requestObject.json === false) {
+	if (requestObject.json === false || requestObject.json === undefined) {
 		// Prevent json parsing
 		axiosConfig.transformResponse = (res) => res;
 	}
@@ -299,7 +354,7 @@ async function parseRequestObject(requestObject: IDataObject) {
 		axiosConfig.maxRedirects = 0;
 	}
 	if (
-		requestObject.followAllRedirect === false &&
+		requestObject.followAllRedirects === false &&
 		((requestObject.method as string | undefined) || 'get').toLowerCase() !== 'get'
 	) {
 		axiosConfig.maxRedirects = 0;
@@ -316,7 +371,63 @@ async function parseRequestObject(requestObject: IDataObject) {
 	}
 
 	if (requestObject.proxy !== undefined) {
-		axiosConfig.proxy = requestObject.proxy as AxiosProxyConfig;
+		// try our best to parse the url provided.
+		if (typeof requestObject.proxy === 'string') {
+			try {
+				const url = new URL(requestObject.proxy);
+				axiosConfig.proxy = {
+					host: url.hostname,
+					port: parseInt(url.port, 10),
+					protocol: url.protocol,
+				};
+				if (!url.port) {
+					// Sets port to a default if not informed
+					if (url.protocol === 'http') {
+						axiosConfig.proxy.port = 80;
+					} else if (url.protocol === 'https') {
+						axiosConfig.proxy.port = 443;
+					}
+				}
+				if (url.username || url.password) {
+					axiosConfig.proxy.auth = {
+						username: url.username,
+						password: url.password,
+					};
+				}
+			} catch (error) {
+				// Not a valid URL. We will try to simply parse stuff
+				// such as user:pass@host:port without protocol (we'll assume http)
+				if (requestObject.proxy.includes('@')) {
+					const [userpass, hostport] = requestObject.proxy.split('@');
+					const [username, password] = userpass.split(':');
+					const [hostname, port] = hostport.split(':');
+					axiosConfig.proxy = {
+						host: hostname,
+						port: parseInt(port, 10),
+						protocol: 'http',
+						auth: {
+							username,
+							password,
+						},
+					};
+				} else if (requestObject.proxy.includes(':')) {
+					const [hostname, port] = requestObject.proxy.split(':');
+					axiosConfig.proxy = {
+						host: hostname,
+						port: parseInt(port, 10),
+						protocol: 'http',
+					};
+				} else {
+					axiosConfig.proxy = {
+						host: requestObject.proxy,
+						port: 80,
+						protocol: 'http',
+					};
+				}
+			}
+		} else {
+			axiosConfig.proxy = requestObject.proxy as AxiosProxyConfig;
+		}
 	}
 
 	if (requestObject.encoding === null) {
@@ -333,7 +444,9 @@ async function parseRequestObject(requestObject: IDataObject) {
 		axiosConfig.headers = Object.assign(axiosConfig.headers || {}, { accept: '*/*' });
 	}
 	if (
+		requestObject.json !== false &&
 		axiosConfig.data !== undefined &&
+		axiosConfig.data !== '' &&
 		!(axiosConfig.data instanceof Buffer) &&
 		!allHeaders.some((headerKey) => headerKey.toLowerCase() === 'content-type')
 	) {
@@ -383,37 +496,82 @@ async function proxyRequestToAxios(
 
 	axiosConfig = Object.assign(axiosConfig, await parseRequestObject(configObject));
 
+	Logger.debug('Proxying request to axios', {
+		originalConfig: configObject,
+		parsedConfig: axiosConfig,
+	});
+
 	return new Promise((resolve, reject) => {
 		axios(axiosConfig)
 			.then((response) => {
 				if (configObject.resolveWithFullResponse === true) {
+					let body = response.data;
+					if (response.data === '') {
+						if (axiosConfig.responseType === 'arraybuffer') {
+							body = Buffer.alloc(0);
+						} else {
+							body = undefined;
+						}
+					}
 					resolve({
-						body: response.data,
+						body,
 						headers: response.headers,
 						statusCode: response.status,
 						statusMessage: response.statusText,
 						request: response.request,
 					});
 				} else {
-					resolve(response.data);
+					let body = response.data;
+					if (response.data === '') {
+						if (axiosConfig.responseType === 'arraybuffer') {
+							body = Buffer.alloc(0);
+						} else {
+							body = undefined;
+						}
+					}
+					resolve(body);
 				}
 			})
 			.catch((error) => {
+				if (configObject.simple === false && error.response) {
+					if (configObject.resolveWithFullResponse) {
+						resolve({
+							body: error.response.data,
+							headers: error.response.headers,
+							statusCode: error.response.status,
+							statusMessage: error.response.statusText,
+						});
+					} else {
+						resolve(error.response.data);
+					}
+					return;
+				}
+
+				Logger.debug('Request proxied to Axios failed', { error });
+				// Axios hydrates the original error with more data. We extract them.
+				// https://github.com/axios/axios/blob/master/lib/core/enhanceError.js
+				// Note: `code` is ignored as it's an expected part of the errorData.
+				const { request, response, isAxiosError, toJSON, config, ...errorData } = error;
+				error.cause = errorData;
+				error.error = error.response?.data || errorData;
+				error.statusCode = error.response?.status;
+				error.options = config || {};
+
+				// Remove not needed data and so also remove circular references
+				error.request = undefined;
+				error.config = undefined;
+				error.options.adapter = undefined;
+				error.options.httpsAgent = undefined;
+				error.options.paramsSerializer = undefined;
+				error.options.transformRequest = undefined;
+				error.options.transformResponse = undefined;
+				error.options.validateStatus = undefined;
+
 				reject(error);
 			});
 	});
 }
 
-function searchForHeader(headers: IDataObject, headerName: string) {
-	if (headers === undefined) {
-		return undefined;
-	}
-
-	const headerNames = Object.keys(headers);
-	headerName = headerName.toLowerCase();
-	return headerNames.find((thisHeader) => thisHeader.toLowerCase() === headerName);
-}
-
 function convertN8nRequestToAxios(n8nRequest: IHttpRequestOptions): AxiosRequestConfig {
 	// Destructure properties with the same name first.
 	const { headers, method, timeout, auth, proxy, url } = n8nRequest;
@@ -686,16 +844,20 @@ export async function requestOAuth2(
 
 			credentials.oauthTokenData = newToken.data;
 
-			// Find the name of the credentials
+			// Find the credentials
 			if (!node.credentials || !node.credentials[credentialsType]) {
 				throw new Error(
 					`The node "${node.name}" does not have credentials of type "${credentialsType}"!`,
 				);
 			}
-			const name = node.credentials[credentialsType];
+			const nodeCredentials = node.credentials[credentialsType];
 
 			// Save the refreshed token
-			await additionalData.credentialsHelper.updateCredentials(name, credentialsType, credentials);
+			await additionalData.credentialsHelper.updateCredentials(
+				nodeCredentials,
+				credentialsType,
+				credentials,
+			);
 
 			Logger.debug(
 				`OAuth2 token for "${credentialsType}" used by node "${node.name}" has been saved to database successfully.`,
@@ -903,25 +1065,26 @@ export async function getCredentials(
 		} as ICredentialsExpressionResolveValues;
 	}
 
-	let name = node.credentials[type];
+	const nodeCredentials = node.credentials[type];
 
-	if (name.charAt(0) === '=') {
+	// TODO: solve using credentials via expression
-		// If the credential name is an expression resolve it
+	// if (name.charAt(0) === '=') {
-		const additionalKeys = getAdditionalKeys(additionalData);
+	// 	// If the credential name is an expression resolve it
-		name = workflow.expression.getParameterValue(
+	// 	const additionalKeys = getAdditionalKeys(additionalData);
-			name,
+	// 	name = workflow.expression.getParameterValue(
-			runExecutionData || null,
+	// 		name,
-			runIndex || 0,
+	// 		runExecutionData || null,
-			itemIndex || 0,
+	// 		runIndex || 0,
-			node.name,
+	// 		itemIndex || 0,
-			connectionInputData || [],
+	// 		node.name,
-			mode,
+	// 		connectionInputData || [],
-			additionalKeys,
+	// 		mode,
-		) as string;
+	// 		additionalKeys,
-	}
+	// 	) as string;
+	// }
 
 	const decryptedDataObject = await additionalData.credentialsHelper.getDecrypted(
-		name,
+		nodeCredentials,
 		type,
 		mode,
 		false,
@@ -1499,6 +1662,9 @@ export function getExecuteFunctions(
 					Logger.warn(`There was a problem sending messsage to UI: ${error.message}`);
 				}
 			},
+			async sendResponse(response: IExecuteResponsePromiseData): Promise<void> {
+				await additionalData.hooks?.executeHookFunctions('sendResponse', [response]);
+			},
 			helpers: {
 				httpRequest,
 				prepareBinaryData,

packages/core/src/UserSettings.ts

@@ -4,7 +4,7 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import * as fs from 'fs';
 import * as path from 'path';
-import { randomBytes } from 'crypto';
+import { createHash, randomBytes } from 'crypto';
 // eslint-disable-next-line import/no-cycle
 import {
 	ENCRYPTION_KEY_ENV_OVERWRITE,
@@ -37,7 +37,12 @@ export async function prepareUserSettings(): Promise<IUserSettings> {
 	if (userSettings !== undefined) {
 		// Settings already exist, check if they contain the encryptionKey
 		if (userSettings.encryptionKey !== undefined) {
-			// Key already exists so return
+			// Key already exists
+			if (userSettings.instanceId === undefined) {
+				userSettings.instanceId = await generateInstanceId(userSettings.encryptionKey);
+				settingsCache = userSettings;
+			}
+
 			return userSettings;
 		}
 	} else {
@@ -52,6 +57,8 @@ export async function prepareUserSettings(): Promise<IUserSettings> {
 		userSettings.encryptionKey = randomBytes(24).toString('base64');
 	}
 
+	userSettings.instanceId = await generateInstanceId(userSettings.encryptionKey);
+
 	// eslint-disable-next-line no-console
 	console.log(`UserSettings were generated and saved to: ${settingsPath}`);
 
@@ -65,8 +72,8 @@ export async function prepareUserSettings(): Promise<IUserSettings> {
  * @export
  * @returns
  */
-// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+
-export async function getEncryptionKey() {
+export async function getEncryptionKey(): Promise<string | undefined> {
 	if (process.env[ENCRYPTION_KEY_ENV_OVERWRITE] !== undefined) {
 		return process.env[ENCRYPTION_KEY_ENV_OVERWRITE];
 	}
@@ -84,6 +91,36 @@ export async function getEncryptionKey() {
 	return userSettings.encryptionKey;
 }
 
+/**
+ * Returns the instance ID
+ *
+ * @export
+ * @returns
+ */
+export async function getInstanceId(): Promise<string> {
+	const userSettings = await getUserSettings();
+
+	if (userSettings === undefined) {
+		return '';
+	}
+
+	if (userSettings.instanceId === undefined) {
+		return '';
+	}
+
+	return userSettings.instanceId;
+}
+
+async function generateInstanceId(key?: string) {
+	const hash = key
+		? createHash('sha256')
+				.update(key.slice(Math.round(key.length / 2)))
+				.digest('hex')
+		: undefined;
+
+	return hash;
+}
+
 /**
  * Adds/Overwrite the given settings in the currently
  * saved user settings
@@ -141,7 +178,12 @@ export async function writeUserSettings(
 		await fsMkdir(path.dirname(settingsPath));
 	}
 
-	await fsWriteFile(settingsPath, JSON.stringify(userSettings, null, '\t'));
+	const settingsToWrite = { ...userSettings };
+	if (settingsToWrite.instanceId !== undefined) {
+		delete settingsToWrite.instanceId;
+	}
+
+	await fsWriteFile(settingsPath, JSON.stringify(settingsToWrite, null, '\t'));
 	settingsCache = JSON.parse(JSON.stringify(userSettings));
 
 	return userSettings;

packages/core/src/WorkflowExecute.ts

@@ -74,8 +74,11 @@ export class WorkflowExecute {
 	 * @returns {(Promise<string>)}
 	 * @memberof WorkflowExecute
 	 */
-	// @ts-ignore
+	// IMPORTANT: Do not add "async" to this function, it will then convert the
-	async run(workflow: Workflow, startNode?: INode, destinationNode?: string): PCancelable<IRun> {
+	//            PCancelable to a regular Promise and does so not allow canceling
+	//            active executions anymore
+	// eslint-disable-next-line @typescript-eslint/promise-function-async
+	run(workflow: Workflow, startNode?: INode, destinationNode?: string): PCancelable<IRun> {
 		// Get the nodes to start workflow execution from
 		startNode = startNode || workflow.getStartNode(destinationNode);
 
@@ -134,8 +137,11 @@ export class WorkflowExecute {
 	 * @returns {(Promise<string>)}
 	 * @memberof WorkflowExecute
 	 */
-	// @ts-ignore
+	// IMPORTANT: Do not add "async" to this function, it will then convert the
-	async runPartialWorkflow(
+	//            PCancelable to a regular Promise and does so not allow canceling
+	//            active executions anymore
+	// eslint-disable-next-line @typescript-eslint/promise-function-async
+	runPartialWorkflow(
 		workflow: Workflow,
 		runData: IRunData,
 		startNodes: string[],
@@ -576,13 +582,23 @@ export class WorkflowExecute {
 	 * @returns {Promise<string>}
 	 * @memberof WorkflowExecute
 	 */
-	// @ts-ignore
+	// IMPORTANT: Do not add "async" to this function, it will then convert the
-	async processRunExecutionData(workflow: Workflow): PCancelable<IRun> {
+	//            PCancelable to a regular Promise and does so not allow canceling
+	//            active executions anymore
+	// eslint-disable-next-line @typescript-eslint/promise-function-async
+	processRunExecutionData(workflow: Workflow): PCancelable<IRun> {
 		Logger.verbose('Workflow execution started', { workflowId: workflow.id });
 
 		const startedAt = new Date();
 
-		const workflowIssues = workflow.checkReadyForExecution();
+		const startNode = this.runExecutionData.executionData!.nodeExecutionStack[0].node.name;
+
+		let destinationNode: string | undefined;
+		if (this.runExecutionData.startData && this.runExecutionData.startData.destinationNode) {
+			destinationNode = this.runExecutionData.startData.destinationNode;
+		}
+
+		const workflowIssues = workflow.checkReadyForExecution({ startNode, destinationNode });
 		if (workflowIssues !== null) {
 			throw new Error(
 				'The workflow has issues and can for that reason not be executed. Please fix them first.',
@@ -896,7 +912,11 @@ export class WorkflowExecute {
 					// the `error` property.
 					for (const execution of nodeSuccessData!) {
 						for (const lineResult of execution) {
-							if (lineResult.json.$error !== undefined && lineResult.json.$json !== undefined) {
+							if (
+								lineResult.json !== undefined &&
+								lineResult.json.$error !== undefined &&
+								lineResult.json.$json !== undefined
+							) {
 								lineResult.error = lineResult.json.$error as NodeApiError | NodeOperationError;
 								lineResult.json = {
 									error: (lineResult.json.$error as NodeApiError | NodeOperationError).message,
@@ -914,6 +934,19 @@ export class WorkflowExecute {
 
 					this.runExecutionData.resultData.runData[executionNode.name].push(taskData);
 
+					if (this.runExecutionData.waitTill!) {
+						await this.executeHook('nodeExecuteAfter', [
+							executionNode.name,
+							taskData,
+							this.runExecutionData,
+						]);
+
+						// Add the node back to the stack that the workflow can start to execute again from that node
+						this.runExecutionData.executionData!.nodeExecutionStack.unshift(executionData);
+
+						break;
+					}
+
 					if (
 						this.runExecutionData.startData &&
 						this.runExecutionData.startData.destinationNode &&
@@ -931,19 +964,6 @@ export class WorkflowExecute {
 						continue;
 					}
 
-					if (this.runExecutionData.waitTill!) {
-						await this.executeHook('nodeExecuteAfter', [
-							executionNode.name,
-							taskData,
-							this.runExecutionData,
-						]);
-
-						// Add the node back to the stack that the workflow can start to execute again from that node
-						this.runExecutionData.executionData!.nodeExecutionStack.unshift(executionData);
-
-						break;
-					}
-
 					// Add the nodes to which the current node has an output connection to that they can
 					// be executed next
 					if (workflow.connectionsBySourceNode.hasOwnProperty(executionNode.name)) {
@@ -1052,8 +1072,7 @@ export class WorkflowExecute {
 		startedAt: Date,
 		workflow: Workflow,
 		executionError?: ExecutionError,
-		// @ts-ignore
+	): Promise<IRun> {
-	): PCancelable<IRun> {
 		const fullRunData = this.getFullRunData(startedAt);
 
 		if (executionError !== undefined) {

packages/core/src/index.ts

@@ -12,7 +12,6 @@ export * from './ActiveWorkflows';
 export * from './ActiveWebhooks';
 export * from './Constants';
 export * from './Credentials';
-export * from './DeferredPromise';
 export * from './Interfaces';
 export * from './LoadNodeParameterOptions';
 export * from './NodeExecuteFunctions';

packages/core/test/Credentials.test.ts

@@ -3,7 +3,7 @@ import { Credentials } from '../src';
 describe('Credentials', () => {
 	describe('without nodeType set', () => {
 		test('should be able to set and read key data without initial data set', () => {
-			const credentials = new Credentials('testName', 'testType', []);
+			const credentials = new Credentials({ id: null, name: 'testName' }, 'testType', []);
 
 			const key = 'key1';
 			const password = 'password';
@@ -23,7 +23,12 @@ describe('Credentials', () => {
 			const initialData = 4321;
 			const initialDataEncoded = 'U2FsdGVkX1+0baznXt+Ag/ub8A2kHLyoLxn/rR9h4XQ=';
 
-			const credentials = new Credentials('testName', 'testType', [], initialDataEncoded);
+			const credentials = new Credentials(
+				{ id: null, name: 'testName' },
+				'testType',
+				[],
+				initialDataEncoded,
+			);
 
 			const newData = 1234;
 
@@ -46,7 +51,7 @@ describe('Credentials', () => {
 				},
 			];
 
-			const credentials = new Credentials('testName', 'testType', nodeAccess);
+			const credentials = new Credentials({ id: null, name: 'testName' }, 'testType', nodeAccess);
 
 			const key = 'key1';
 			const password = 'password';

packages/core/test/Helpers.ts

@@ -4,7 +4,9 @@ import {
 	ICredentialDataDecryptedObject,
 	ICredentialsHelper,
 	IDataObject,
+	IDeferredPromise,
 	IExecuteWorkflowInfo,
+	INodeCredentialsDetails,
 	INodeExecutionData,
 	INodeParameters,
 	INodeType,
@@ -19,21 +21,24 @@ import {
 	WorkflowHooks,
 } from 'n8n-workflow';
 
-import { Credentials, IDeferredPromise, IExecuteFunctions } from '../src';
+import { Credentials, IExecuteFunctions } from '../src';
 
 export class CredentialsHelper extends ICredentialsHelper {
-	getDecrypted(name: string, type: string): Promise<ICredentialDataDecryptedObject> {
+	getDecrypted(
+		nodeCredentials: INodeCredentialsDetails,
+		type: string,
+	): Promise<ICredentialDataDecryptedObject> {
 		return new Promise((res) => res({}));
 	}
 
-	getCredentials(name: string, type: string): Promise<Credentials> {
+	getCredentials(nodeCredentials: INodeCredentialsDetails, type: string): Promise<Credentials> {
 		return new Promise((res) => {
-			res(new Credentials('', '', [], ''));
+			res(new Credentials({ id: null, name: '' }, '', [], ''));
 		});
 	}
 
 	async updateCredentials(
-		name: string,
+		nodeCredentials: INodeCredentialsDetails,
 		type: string,
 		data: ICredentialDataDecryptedObject,
 	): Promise<void> {}
@@ -611,10 +616,7 @@ class NodeTypesClass implements INodeTypes {
 									name: 'dotNotation',
 									type: 'boolean',
 									default: true,
-									description: `By default does dot-notation get used in property names..<br />
+									description: `<p>By default, dot-notation is used in property names. This means that "a.b" will set the property "b" underneath "a" so { "a": { "b": value} }.</p><p>If that is not intended this can be deactivated, it will then set { "a.b": value } instead.</p>`,
-						This means that "a.b" will set the property "b" underneath "a" so { "a": { "b": value} }.<br />
-						If that is not intended this can be deactivated, it will then set { "a.b": value } instead.
-						`,
 								},
 							],
 						},

packages/core/test/WorkflowExecute.test.ts

@@ -1,6 +1,14 @@
-import { IConnections, ILogger, INode, IRun, LoggerProxy, Workflow } from 'n8n-workflow';
+import {
+	createDeferredPromise,
+	IConnections,
+	ILogger,
+	INode,
+	IRun,
+	LoggerProxy,
+	Workflow,
+} from 'n8n-workflow';
 
-import { createDeferredPromise, WorkflowExecute } from '../src';
+import { WorkflowExecute } from '../src';
 
 import * as Helpers from './Helpers';
 

packages/design-system/package.json

@@ -1,6 +1,6 @@
 {
   "name": "n8n-design-system",
-  "version": "0.3.0",
+  "version": "0.8.0",
   "license": "SEE LICENSE IN LICENSE.md",
   "homepage": "https://n8n.io",
   "author": {

packages/design-system/src/components/N8nButton/Button.stories.js

@@ -18,7 +18,7 @@ export default {
 		size: {
 			control: {
 				type: 'select',
-				options: ['small', 'medium', 'large'],
+				options: ['mini', 'small', 'medium', 'large', 'xlarge'],
 			},
 		},
 		loading: {
@@ -31,12 +31,6 @@ export default {
 				type: 'text',
 			},
 		},
-		iconSize: {
-			control: {
-				type: 'select',
-				options: ['small', 'medium', 'large'],
-			},
-		},
 		circle: {
 			control: {
 				type: 'boolean',

packages/design-system/src/components/N8nButton/Button.vue

@@ -16,13 +16,13 @@
 			<component
 				:is="$options.components.N8nSpinner"
 				v-if="props.loading"
-				:size="props.iconSize"
+				:size="props.size"
 			/>
 			<component
 				:is="$options.components.N8nIcon"
 				v-else-if="props.icon"
 				:icon="props.icon"
-				:size="props.iconSize"
+				:size="props.size"
 			/>
 		</span>
 		<span v-if="props.label">{{ props.label }}</span>
@@ -58,7 +58,7 @@ export default {
 			type: String,
 			default: 'medium',
 			validator: (value: string): boolean =>
-				['small', 'medium', 'large'].indexOf(value) !== -1,
+				['mini', 'small', 'medium', 'large', 'xlarge'].indexOf(value) !== -1,
 		},
 		loading: {
 			type: Boolean,
@@ -71,9 +71,6 @@ export default {
 		icon: {
 			type: String,
 		},
-		iconSize: {
-			type: String,
-		},
 		round: {
 			type: Boolean,
 			default: true,

packages/design-system/src/components/N8nButton/index.d.ts

@@ -35,9 +35,6 @@ export declare class N8nButton extends N8nComponent {
 	/** Button icon, accepts an icon name of font awesome icon component */
 	icon: string;
 
-	/** Size of icon */
-	iconSize: N8nComponentSize;
-
 	/** Full width */
 	fullWidth: boolean;
 

packages/design-system/src/components/N8nHeading/Heading.stories.js

@@ -0,0 +1,30 @@
+import N8nHeading from './Heading.vue';
+
+export default {
+	title: 'Atoms/Heading',
+	component: N8nHeading,
+	argTypes: {
+		size: {
+			control: {
+				type: 'select',
+				options: ['2xlarge', 'xlarge', 'large', 'medium', 'small'],
+			},
+		},
+		color: {
+			control: {
+				type: 'select',
+				options: ['primary', 'text-dark', 'text-base', 'text-light'],
+			},
+		},
+	},
+};
+
+const Template = (args, { argTypes }) => ({
+	props: Object.keys(argTypes),
+	components: {
+		N8nHeading,
+	},
+	template: '<n8n-heading v-bind="$props">hello world</n8n-heading>',
+});
+
+export const Heading = Template.bind({});

packages/design-system/src/components/N8nHeading/Heading.vue

@@ -0,0 +1,128 @@
+<template functional>
+	<component :is="props.tag" :class="$style[$options.methods.getClass(props)]" :style="$options.methods.getStyles(props)">
+		<slot></slot>
+	</component>
+</template>
+
+<script lang="ts">
+export default {
+	name: 'n8n-heading',
+	props: {
+		tag: {
+			type: String,
+			default: 'span',
+		},
+		bold: {
+			type: Boolean,
+			default: false,
+		},
+		size: {
+			type: String,
+			default: 'medium',
+			validator: (value: string): boolean => ['2xlarge', 'xlarge', 'large', 'medium', 'small'].includes(value),
+		},
+		color: {
+			type: String,
+			validator: (value: string): boolean => ['primary', 'text-dark', 'text-base', 'text-light'].includes(value),
+		},
+	},
+	methods: {
+		getClass(props: {size: string, bold: boolean}) {
+			return `heading-${props.size}${props.bold ? '-bold' : '-regular'}`;
+		},
+		getStyles(props: {color: string}) {
+			const styles = {} as any;
+			if (props.color) {
+				styles.color = `var(--color-${props.color})`;
+			}
+			return styles;
+		},
+	},
+};
+</script>
+
+<style lang="scss" module>
+.bold {
+	font-weight: var(--font-weight-bold);
+}
+
+.regular {
+	font-weight: var(--font-weight-regular);
+}
+
+.heading-2xlarge {
+	font-size: var(--font-size-2xl);
+	line-height: var(--font-line-height-compact);
+}
+
+.heading-2xlarge-regular {
+	composes: regular;
+	composes: heading-2xlarge;
+}
+
+.heading-2xlarge-bold {
+	composes: bold;
+	composes: heading-2xlarge;
+}
+
+.heading-xlarge {
+	font-size: var(--font-size-xl);
+	line-height: var(--font-line-height-compact);
+}
+
+.heading-xlarge-regular {
+	composes: regular;
+	composes: heading-xlarge;
+}
+
+.heading-xlarge-bold {
+	composes: bold;
+	composes: heading-xlarge;
+}
+
+.heading-large {
+	font-size: var(--font-size-l);
+	line-height: var(--font-line-height-loose);
+}
+
+.heading-large-regular {
+	composes: regular;
+	composes: heading-large;
+}
+
+.heading-large-bold {
+	composes: bold;
+	composes: heading-large;
+}
+
+.heading-medium {
+	font-size: var(--font-size-m);
+	line-height: var(--font-line-height-loose);
+}
+
+.heading-medium-regular {
+	composes: regular;
+	composes: heading-medium;
+}
+
+.heading-medium-bold {
+	composes: bold;
+	composes: heading-medium;
+}
+
+.heading-small {
+	font-size: var(--font-size-s);
+	line-height: var(--font-line-height-regular);
+}
+
+.heading-small-regular {
+	composes: regular;
+	composes: heading-small;
+}
+
+.heading-small-bold {
+	composes: bold;
+	composes: heading-small;
+}
+
+</style>

packages/design-system/src/components/N8nHeading/index.js

@@ -0,0 +1,3 @@
+import N8nHeading from './Heading.vue';
+
+export default N8nHeading;

packages/design-system/src/components/N8nIcon/Icon.vue

@@ -1,19 +1,27 @@
 <template functional>
 	<component
-		:is="$options.components.FontAwesomeIcon"
+		:is="$options.components.N8nText"
-		:class="$style[`_${props.size}`]"
+		:size="props.size"
-		:icon="props.icon"
+		:compact="true"
-		:spin="props.spin"
+	>
-	/>
+		<component
+			:is="$options.components.FontAwesomeIcon"
+			:icon="props.icon"
+			:spin="props.spin"
+			:class="$style[props.size]"
+		/>
+	</component>
 </template>
 
 <script lang="ts">
 import { FontAwesomeIcon } from '@fortawesome/vue-fontawesome';
+import N8nText from '../N8nText';
 
 export default {
 	name: 'n8n-icon',
 	components: {
 		FontAwesomeIcon,
+		N8nText,
 	},
 	props: {
 		icon: {
@@ -23,9 +31,6 @@ export default {
 		size: {
 			type: String,
 			default: 'medium',
-			validator: function (value: string): boolean {
-				return ['small', 'medium', 'large'].indexOf(value) !== -1;
-			},
 		},
 		spin: {
 			type: Boolean,
@@ -35,22 +40,21 @@ export default {
 };
 </script>
 
+
 <style lang="scss" module>
-._small {
+.xlarge {
-	font-size: 0.85em;
+	width: var(--font-size-xl) !important;
-	height: 0.85em;
-	width: 0.85em !important;
 }
 
-._medium {
+.large {
-	font-size: 0.95em;
+	width: var(--font-size-m) !important;
-	height: 0.95em;
-	width: 0.95em !important;
 }
 
-._large {
+.medium {
-	font-size: 1.22em;
+	width: var(--font-size-s) !important;
-	height: 1.22em;
+}
-	width: 1.22em !important;
+
+.small {
+	width: var(--font-size-2xs) !important;
 }
 </style>

packages/design-system/src/components/N8nIconButton/IconButton.vue

@@ -1,12 +1,11 @@
 <template functional>
-	<n8n-button
+	<component :is="$options.components.N8nButton"
 		:type="props.type"
 		:disabled="props.disabled"
-		:size="props.size === 'xlarge' ? 'large' : props.size"
+		:size="props.size"
 		:loading="props.loading"
 		:title="props.title"
 		:icon="props.icon"
-		:iconSize="$options.iconSizeMap[props.size] || props.size"
 		:theme="props.theme"
 		@click="(e) => listeners.click && listeners.click(e)"
 		circle
@@ -14,18 +13,13 @@
 </template>
 
 <script lang="ts">
-import Vue from 'vue';
 import N8nButton from '../N8nButton';
 
-const iconSizeMap = {
-	large: 'medium',
-	xlarge: 'large',
-};
-
-Vue.component('N8nButton', N8nButton);
-
 export default {
 	name: 'n8n-icon-button',
+	components: {
+		N8nButton,
+	},
 	props: {
 		type: {
 			type: String,
@@ -36,8 +30,6 @@ export default {
 		size: {
 			type: String,
 			default: 'medium',
-			validator: (value: string): boolean =>
-				['small', 'medium', 'large', 'xlarge'].indexOf(value) !== -1,
 		},
 		loading: {
 			type: Boolean,
@@ -55,6 +47,5 @@ export default {
 			type: String,
 		},
 	},
-	iconSizeMap,
 };
 </script>

packages/design-system/src/components/N8nIconButton/index.d.ts

@@ -12,9 +12,6 @@ export declare class N8nIconButton extends N8nComponent {
 	/** Button size */
 	size: N8nComponentSize | 'xlarge';
 
-	/** icon size */
-	iconSize: N8nComponentSize;
-
 	/** Determine whether it's loading */
 	loading: boolean;
 

packages/design-system/src/components/N8nInfoTip/InfoTip.vue

@@ -1,19 +1,16 @@
 <template functional>
 	<div :class="$style.infotip">
-		<n8n-icon icon="info-circle" /> <span><slot></slot></span>
+		<component :is="$options.components.N8nIcon" icon="info-circle" /> <span><slot></slot></span>
 	</div>
 </template>
 
 <script lang="ts">
-import Vue from 'vue';
-
 import N8nIcon from '../N8nIcon';
 
-Vue.component('N8nIcon', N8nIcon);
-
 export default {
 	name: 'n8n-info-tip',
-	props: {
+	components: {
+		N8nIcon,
 	},
 };
 </script>

packages/design-system/src/components/N8nInputLabel/InputLabel.vue

@@ -1,15 +1,15 @@
 <template functional>
-	<div :class="$style.inputLabel">
+	<div :class="{[$style.inputLabelContainer]: !props.labelHoverableOnly}">
-		<div :class="$style.label">
+		<div :class="{[$style.inputLabel]: props.labelHoverableOnly, [$options.methods.getLabelClass(props, $style)]: true}">
-			<span>
+			<component v-if="props.label" :is="$options.components.N8nText" :bold="props.bold" :size="props.size" :compact="!props.underline">
-				{{ $options.methods.addTargetBlank(props.label) }}
+				{{ props.label }}
-				<span v-if="props.required" :class="$style.required">*</span>
+				<component :is="$options.components.N8nText" color="primary" :bold="props.bold" :size="props.size" v-if="props.required">*</component>
-			</span>
+			</component>
-			<span :class="$style.infoIcon" v-if="props.tooltipText">
+			<span :class="[$style.infoIcon, props.showTooltip ? $style.showIcon: $style.hiddenIcon]" v-if="props.tooltipText">
-				<n8n-tooltip placement="top" :popper-class="$style.tooltipPopper">
+				<component :is="$options.components.N8nTooltip" placement="top" :popper-class="$style.tooltipPopper">
-					<n8n-icon icon="question-circle" />
+					<component :is="$options.components.N8nIcon" icon="question-circle" size="small" />
-					<div slot="content" v-html="props.tooltipText"></div>
+					<div slot="content" v-html="$options.methods.addTargetBlank(props.tooltipText)"></div>
-				</n8n-tooltip>
+				</component>
 			</span>
 		</div>
 		<slot></slot>
@@ -17,22 +17,22 @@
 </template>
 
 <script lang="ts">
-import Vue from 'vue';
+import N8nText from '../N8nText';
-
 import N8nTooltip from '../N8nTooltip';
 import N8nIcon from '../N8nIcon';
 
 import { addTargetBlank } from '../utils/helpers';
 
-Vue.component('N8nIcon', N8nIcon);
-Vue.component('N8nTooltip', N8nTooltip);
-
 export default {
 	name: 'n8n-input-label',
+	components: {
+		N8nText,
+		N8nIcon,
+		N8nTooltip,
+	},
 	props: {
 		label: {
 			type: String,
-			required: true,
 		},
 		tooltipText: {
 			type: String,
@@ -40,40 +40,104 @@ export default {
 		required: {
 			type: Boolean,
 		},
+		bold: {
+			type: Boolean,
+			default: true,
+		},
+		size: {
+			type: String,
+			default: 'medium',
+			validator: (value: string): boolean =>
+				['small', 'medium'].includes(value),
+		},
+		underline: {
+			type: Boolean,
+		},
+		showTooltip: {
+			type: Boolean,
+		},
+		labelHoverableOnly: {
+			type: Boolean,
+			default: false,
+		},
 	},
 	methods: {
 		addTargetBlank,
+		getLabelClass(props: {label: string, size: string, underline: boolean}, $style: any) {
+			if (!props.label) {
+				return '';
+			}
+
+			if (props.underline) {
+				return $style[`label-${props.size}-underline`];
+			}
+
+			return $style[`label-${props.size}`];
+		},
 	},
 };
 </script>
 
 <style lang="scss" module>
-.inputLabel {
+.inputLabelContainer:hover {
-	&:hover {
+	> div > .infoIcon {
-		--info-icon-display: inline-block;
+		display: inline-block;
 	}
 }
 
-.label {
+.inputLabel:hover {
-	font-weight: var(--font-weight-bold);
+	> .infoIcon {
-	font-size: var(--font-size-s);
+		display: inline-block;
-	margin-bottom: var(--spacing-2xs);
-
-	* {
-		margin-right: var(--spacing-4xs);
 	}
 }
 
 .infoIcon {
 	color: var(--color-text-light);
-	display: var(--info-icon-display, none);
 }
 
-.required {
+.showIcon {
-	color: var(--color-primary);
+	display: inline-block;
+}
+
+.hiddenIcon {
+	display: none;
+}
+
+.label {
+	* {
+		margin-right: var(--spacing-5xs);
+	}
+}
+
+.label-small {
+	composes: label;
+	margin-bottom: var(--spacing-4xs);
+}
+
+.label-medium {
+	composes: label;
+	margin-bottom: var(--spacing-2xs);
+}
+
+.underline {
+	border-bottom: var(--border-base);
+}
+
+.label-small-underline {
+	composes: label-small;
+	composes: underline;
+}
+
+.label-medium-underline {
+	composes: label-medium;
+	composes: underline;
 }
 
 .tooltipPopper {
 	max-width: 400px;
+
+	li {
+		margin-left: var(--spacing-s);
+	}
 }
 </style>

packages/design-system/src/components/N8nSelect/Select.stories.js

@@ -1,5 +1,6 @@
 import N8nSelect from './Select.vue';
 import N8nOption from '../N8nOption';
+import N8nIcon from '../N8nIcon';
 import { action } from '@storybook/addon-actions';
 
 export default {
@@ -48,6 +49,7 @@ const Template = (args, { argTypes }) => ({
 	components: {
 		N8nSelect,
 		N8nOption,
+		N8nIcon,
 	},
 	template: '<n8n-select v-bind="$props" v-model="val" @input="onInput" @change="onChange"><n8n-option value="1">op1</n8n-option><n8n-option value="2">op2</n8n-option></n8n-select>',
 	data() {
@@ -73,6 +75,7 @@ const ManyTemplate = (args, { argTypes }) => ({
 	components: {
 		N8nSelect,
 		N8nOption,
+		N8nIcon,
 	},
 	template: `<div class="multi-container">${selects}</div>`,
 	methods,
@@ -97,6 +100,7 @@ const ManyTemplateWithIcon = (args, { argTypes }) => ({
 	components: {
 		N8nSelect,
 		N8nOption,
+		N8nIcon,
 	},
 	template: `<div class="multi-container">${selectsWithIcon}</div>`,
 	methods,
@@ -120,6 +124,7 @@ const LimitedWidthTemplate = (args, { argTypes }) => ({
 	components: {
 		N8nSelect,
 		N8nOption,
+		N8nIcon,
 	},
 	template: '<div style="width:100px;"><n8n-select v-bind="$props" v-model="val" @input="onInput" @change="onChange"><n8n-option value="1" label="opt1 11 1111" /><n8n-option value="2" label="opt2 test very long ipsum"/></n8n-select></div>',
 	data() {

packages/design-system/src/components/N8nText/Text.stories.js

@@ -0,0 +1,30 @@
+import N8nText from './Text.vue';
+
+export default {
+	title: 'Atoms/Text',
+	component: N8nText,
+	argTypes: {
+		size: {
+			control: {
+				type: 'select',
+				options: ['small', 'medium', 'large'],
+			},
+		},
+		color: {
+			control: {
+				type: 'select',
+				options: ['primary', 'text-dark', 'text-base', 'text-light'],
+			},
+		},
+	},
+};
+
+const Template = (args, { argTypes }) => ({
+	props: Object.keys(argTypes),
+	components: {
+		N8nText,
+	},
+	template: '<n8n-text v-bind="$props">hello world</n8n-text>',
+});
+
+export const Text = Template.bind({});

packages/design-system/src/components/N8nText/Text.vue

@@ -0,0 +1,125 @@
+<template functional>
+	<span :class="$style[$options.methods.getClass(props)]" :style="$options.methods.getStyles(props)">
+		<slot></slot>
+	</span>
+</template>
+
+<script lang="ts">
+import Vue from 'vue';
+export default Vue.extend({
+	name: 'n8n-text',
+	props: {
+		bold: {
+			type: Boolean,
+			default: false,
+		},
+		size: {
+			type: String,
+			default: 'medium',
+			validator: (value: string): boolean => ['mini', 'small', 'medium', 'large', 'xlarge'].includes(value),
+		},
+		color: {
+			type: String,
+			validator: (value: string): boolean => ['primary', 'text-dark', 'text-base', 'text-light'].includes(value),
+		},
+		align: {
+			type: String,
+			validator: (value: string): boolean => ['right', 'left', 'center'].includes(value),
+		},
+		compact: {
+			type: Boolean,
+			default: false,
+		},
+	},
+	methods: {
+		getClass(props: {size: string, bold: boolean}) {
+			return `body-${props.size}${props.bold ? '-bold' : '-regular'}`;
+		},
+		getStyles(props: {color: string, align: string, compact: false}) {
+			const styles = {} as any;
+			if (props.color) {
+				styles.color = `var(--color-${props.color})`;
+			}
+			if (props.compact) {
+				styles['line-height'] = 1;
+			}
+			if (props.align) {
+				styles['text-align'] = props.align;
+			}
+			return styles;
+		},
+	},
+});
+</script>
+
+<style lang="scss" module>
+.bold {
+	font-weight: var(--font-weight-bold);
+}
+
+.regular {
+	font-weight: var(--font-weight-regular);
+}
+
+.body-xlarge {
+	font-size: var(--font-size-xl);
+	line-height: var(--font-line-height-xloose);
+}
+
+.body-xlarge-regular {
+	composes: regular;
+	composes: body-xlarge;
+}
+
+.body-xlarge-bold {
+	composes: bold;
+	composes: body-xlarge;
+}
+
+
+.body-large {
+	font-size: var(--font-size-m);
+	line-height: var(--font-line-height-xloose);
+}
+
+.body-large-regular {
+	composes: regular;
+	composes: body-large;
+}
+
+.body-large-bold {
+	composes: bold;
+	composes: body-large;
+}
+
+.body-medium {
+	font-size: var(--font-size-s);
+	line-height: var(--font-line-height-loose);
+}
+
+.body-medium-regular {
+	composes: regular;
+	composes: body-medium;
+}
+
+.body-medium-bold {
+	composes: bold;
+	composes: body-medium;
+}
+
+.body-small {
+	font-size: var(--font-size-2xs);
+	line-height: var(--font-line-height-loose);
+}
+
+.body-small-regular {
+	composes: regular;
+	composes: body-small;
+}
+
+.body-small-bold {
+	composes: bold;
+	composes: body-small;
+}
+
+</style>

packages/design-system/src/components/N8nText/index.js

@@ -0,0 +1,3 @@
+import N8nText from './Text.vue';
+
+export default N8nText;

packages/design-system/src/components/component.d.ts

@@ -7,4 +7,4 @@ export declare class N8nComponent extends Vue {
 }
 
 /** Component size definition for button, input, etc */
-export type N8nComponentSize = 'large' | 'medium' | 'small';
+export type N8nComponentSize = 'xlarge' | 'large' | 'medium' | 'small' | 'mini';

packages/design-system/src/components/index.js

@@ -5,10 +5,12 @@ import N8nInput from './N8nInput';
 import N8nInfoTip from './N8nInfoTip';
 import N8nInputNumber from './N8nInputNumber';
 import N8nInputLabel from './N8nInputLabel';
+import N8nHeading from './N8nHeading';
 import N8nMenu from './N8nMenu';
 import N8nMenuItem from './N8nMenuItem';
 import N8nSelect from './N8nSelect';
 import N8nSpinner from './N8nSpinner';
+import N8nText from './N8nText';
 import N8nTooltip from './N8nTooltip';
 import N8nOption from './N8nOption';
 
@@ -20,10 +22,12 @@ export {
 	N8nInput,
 	N8nInputLabel,
 	N8nInputNumber,
+	N8nHeading,
 	N8nMenu,
 	N8nMenuItem,
 	N8nSelect,
 	N8nSpinner,
+	N8nText,
 	N8nTooltip,
 	N8nOption,
 };

packages/design-system/src/styleguide/TextClasses.vue

@@ -1,49 +0,0 @@
-<template>
-	<table :class="$style.table">
-		<tr v-for="c in classes" :key="c">
-			<td>.{{ c }}{{ postfix ? postfix : '' }}</td>
-			<td :class="$style[`${c}${postfix ? postfix : ''}`]">
-				Lorem ipsum dolor sit amet, consectetur adipiscing elit. Suspendisse in
-				luctus sapien, a suscipit neque.
-			</td>
-		</tr>
-	</table>
-</template>
-
-<script lang="ts">
-export default {
-	name: 'text-classes',
-	data(): { observer: null | MutationObserver; classes: string[] } {
-		return {
-			observer: null as null | MutationObserver,
-			classes: [
-				'heading1',
-				'heading2',
-				'heading3',
-				'heading4',
-				'body-large',
-				'body-medium',
-				'body-small',
-			],
-		};
-	},
-	props: {
-		postfix: {
-			type: String,
-		},
-	},
-};
-</script>
-
-<style lang="scss" module>
-@use "~/theme/src/common/typography.scss";
-
-.table {
-	text-align: center;
-	color: var(--color-text-dark);
-
-	* {
-		padding-bottom: 1em;
-	}
-}
-</style>

packages/design-system/src/styleguide/border.stories.mdx

@@ -16,7 +16,7 @@ import VariableTable from './VariableTable.vue';
 <Canvas>
 	<Story name="border-radius">
 		{{
-			template: `<variable-table :variables="['--border-radius-small','--border-radius-base']" />`,
+			template: `<variable-table :variables="['--border-radius-small','--border-radius-base', '--border-radius-large', '--border-radius-xlarge']" />`,
 			components: {
 				VariableTable,
 			},

packages/design-system/src/styleguide/colors.stories.mdx

@@ -44,7 +44,7 @@ import ColorCircles from './ColorCircles.vue';
 <Canvas>
 	<Story name="success">
 		{{
-			template: `<color-circles :colors="['--color-success', '--color-success-tint-1', '--color-success-tint-2']" />`,
+			template: `<color-circles :colors="['--color-success', '--color-success-tint-1', '--color-success-tint-2', '--color-success-light']" />`,
 			components: {
 				ColorCircles,
 			},
@@ -109,7 +109,7 @@ import ColorCircles from './ColorCircles.vue';
 <Canvas>
 	<Story name="foreground">
 		{{
-			template: `<color-circles :colors="['--color-foreground-base', '--color-foreground-light', '--color-foreground-xlight']" />`,
+			template: `<color-circles :colors="['--color-foreground-xdark', '--color-foreground-dark', '--color-foreground-base', '--color-foreground-light', '--color-foreground-xlight']" />`,
 			components: {
 				ColorCircles,
 			},
@@ -129,3 +129,16 @@ import ColorCircles from './ColorCircles.vue';
 		}}
 	</Story>
 </Canvas>
+
+## Canvas
+
+<Canvas>
+	<Story name="canvas">
+		{{
+			template: `<color-circles :colors="['--color-canvas-background', '--color-canvas-dot']" />`,
+			components: {
+				ColorCircles,
+			},
+		}}
+	</Story>
+</Canvas>

packages/design-system/src/styleguide/spacing.stories.mdx

@@ -1,6 +1,5 @@
 import { Meta, Story, Canvas } from '@storybook/addon-docs';
 import Sizes from './Sizes.vue';
-import TextClasses from './TextClasses.vue';
 
 <Meta
 	title="Styleguide/Spacing"

packages/design-system/src/styleguide/text.stories.mdx

@@ -1,38 +0,0 @@
-import { Meta, Story, Canvas } from '@storybook/addon-docs';
-import TextClasses from './TextClasses.vue';
-
-<Meta
-	title="Styleguide/Text"
-	parameters={{
-		design: {
-			type: 'figma',
-			url: 'https://www.figma.com/file/DxLbnIyMK8X0uLkUguFV4n/n8n-design-system_v1?node-id=79%3A6898',
-		},
-	}}
-/>
-
-# Regular
-
-<Canvas>
-	<Story name="regular">
-		{{
-			template: `<text-classes />`,
-			components: {
-				TextClasses,
-			},
-		}}
-	</Story>
-</Canvas>
-
-# Bold
-
-<Canvas>
-	<Story name="bold">
-		{{
-			template: `<text-classes postfix="-bold" />`,
-			components: {
-				TextClasses,
-			},
-		}}
-	</Story>
-</Canvas>

packages/design-system/theme/src/_tokens.scss

@@ -75,6 +75,15 @@
 		var(--color-success-tint-2-l)
 	);
 
+	--color-success-light-h: 150;
+	--color-success-light-s: 54%;
+	--color-success-light-l: 70%;
+	--color-success-light: hsl(
+		var(--color-success-light-h),
+		var(--color-success-light-s),
+		var(--color-success-light-l)
+	);
+
 	--color-warning-h: 36;
 	--color-warning-s: 77%;
 	--color-warning-l: 57%;
@@ -187,6 +196,24 @@
 		var(--color-text-xlight-l)
 	);
 
+	--color-foreground-xdark-h: 220;
+	--color-foreground-xdark-s: 7.4%;
+	--color-foreground-xdark-l: 52.5%;
+	--color-foreground-xdark: hsl(
+		var(--color-foreground-xdark-h),
+		var(--color-foreground-xdark-s),
+		var(--color-foreground-xdark-l)
+	);
+
+	--color-foreground-dark-h: 228;
+	--color-foreground-dark-s: 9.6%;
+	--color-foreground-dark-l: 79.6%;
+	--color-foreground-dark: hsl(
+		var(--color-foreground-dark-h),
+		var(--color-foreground-dark-s),
+		var(--color-foreground-dark-l)
+	);
+
 	--color-foreground-base-h: 220;
 	--color-foreground-base-s: 20%;
 	--color-foreground-base-l: 88.2%;
@@ -259,6 +286,26 @@
 		var(--color-background-xlight-l)
 	);
 
+	--color-canvas-dot-h: 204;
+	--color-canvas-dot-s: 15.6%;
+	--color-canvas-dot-l: 87.5%;
+	--color-canvas-dot: hsl(
+		var(--color-canvas-dot-h),
+		var(--color-canvas-dot-s),
+		var(--color-canvas-dot-l)
+	);
+
+	--color-canvas-background-h: 260;
+	--color-canvas-background-s: 100%;
+	--color-canvas-background-l: 99.4%;
+	--color-canvas-background: hsl(
+		var(--color-canvas-background-h),
+		var(--color-canvas-background-s),
+		var(--color-canvas-background-l)
+	);
+
+	--border-radius-xlarge: 12px;
+	--border-radius-large: 8px;
 	--border-radius-base: 4px;
 	--border-radius-small: 2px;
 	--border-color-base: var(--color-foreground-base);

packages/design-system/theme/src/button.scss

@@ -83,6 +83,17 @@ $loading-overlay-background-color: rgba(255, 255, 255, 0.35);
 		--button-border-radius: 50%;
 	}
 
+	@include mixins.m(mini) {
+		--button-padding-vertical: var(--spacing-4xs);
+		--button-padding-horizontal: var(--spacing-2xs);
+		--button-font-size: var(--font-size-2xs);
+
+		@include mixins.when(circle) {
+			--button-padding-vertical: var(--spacing-4xs);
+			--button-padding-horizontal: var(--spacing-4xs);
+		}
+	}
+
 	@include mixins.m(small) {
 		--button-padding-vertical: var(--spacing-3xs);
 		--button-padding-horizontal: var(--spacing-xs);
@@ -104,4 +115,15 @@ $loading-overlay-background-color: rgba(255, 255, 255, 0.35);
 			--button-padding-horizontal: var(--spacing-2xs);
 		}
 	}
+
+	@include mixins.m(xlarge) {
+		--button-padding-vertical: var(--spacing-xs);
+		--button-padding-horizontal: var(--spacing-s);
+		--button-font-size: var(--font-size-m);
+
+		@include mixins.when(circle) {
+			--button-padding-vertical: var(--spacing-xs);
+			--button-padding-horizontal: var(--spacing-xs);
+		}
+	}
 }

packages/design-system/theme/src/collapse.scss

@@ -29,7 +29,7 @@
 		@include mixins.e(arrow) {
 			margin: 0 8px 0 auto;
 			transition: transform 0.3s;
-			font-weight: 300;
+			font-weight: 400;
 			@include mixins.when(active) {
 				transform: rotate(90deg);
 			}

packages/design-system/theme/src/common/popup.scss

@@ -12,20 +12,16 @@
 @keyframes v-modal-in {
 	0% {
 		opacity: 0;
-		backdrop-filter: blur(4px) opacity(0);
 	}
 	100% {
-		backdrop-filter: blur(4px) opacity(1);
 	}
 }
 
 @keyframes v-modal-out {
 	0% {
-		backdrop-filter: blur(4px) opacity(1);
 	}
 	100% {
 		opacity: 0;
-		backdrop-filter: blur(4px) opacity(0);
 	}
 }
 
@@ -36,7 +32,6 @@
 	width: 100%;
 	height: 100%;
 	background-color: var.$popup-modal-background-color;
-	backdrop-filter: blur(4px) opacity(1);
 }
 
 @include mixins.b(popup-parent) {

packages/design-system/theme/src/common/typography.scss

@@ -1,66 +0,0 @@
-%bold {
-	font-weight: var(--font-weight-bold);
-}
-
-.heading1 {
-	font-size: var(--font-size-2xl);
-	line-height: var(--font-line-height-compact);
-}
-
-.heading1-bold {
-	@extend %bold, .heading1;
-}
-
-.heading2 {
-	font-size: var(--font-size-xl);
-	line-height: var(--font-line-height-loose);
-}
-
-.heading2-bold {
-	@extend %bold, .heading2;
-}
-
-.heading3 {
-	font-size: var(--font-size-m);
-	line-height: var(--font-line-height-loose);
-}
-
-.heading3-bold {
-	@extend %bold, .heading3;
-}
-
-.heading4 {
-	font-size: var(--font-size-s);
-	line-height: var(--font-line-height-regular);
-}
-
-.heading4-bold {
-	@extend %bold, .heading4;
-}
-
-.body-large {
-	font-size: var(--font-size-m);
-	line-height: var(--font-line-height-xloose);
-}
-
-.body-large-bold {
-	@extend %bold, .body-large;
-}
-
-.body-medium {
-	font-size: var(--font-size-s);
-	line-height: var(--font-line-height-loose);
-}
-
-.body-medium-bold {
-	@extend %bold, .body-medium;
-}
-
-.body-small {
-	font-size: var(--font-size-2xs);
-	line-height: var(--font-line-height-loose);
-}
-
-.body-small-bold {
-	@extend %bold, .body-small;
-}

packages/design-system/theme/src/common/var.scss

@@ -753,11 +753,7 @@ $switch-button-size: 16px;
 $dialog-background-color: $color-white;
 $dialog-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
 /// fontSize||Font|1
-$dialog-title-font-size: $font-size-large;
-/// fontSize||Font|1
 $dialog-content-font-size: 14px;
-/// fontLineHeight||LineHeight|2
-$dialog-font-line-height: $font-line-height-primary;
 /// padding||Spacing|3
 $dialog-padding-primary: var(--spacing-l);
 $dialog-close-top: var(--dialog-close-top, var(--spacing-l));

packages/design-system/theme/src/dialog.scss

@@ -59,16 +59,15 @@
 	}
 
 	@include mixins.e(title) {
-		line-height: var.$dialog-font-line-height;
+		line-height: var(--font-line-height-compact);
-		font-size: var.$dialog-title-font-size;
+		font-size: var(--font-size-xl);
 		color: var(--color-text-dark);
+		font-weight: var(--font-weight-regular);
 	}
 
 	@include mixins.e(body) {
 		padding: var.$dialog-padding-primary;
 		color: var(--color-text-base);
-		font-size: var.$dialog-content-font-size;
-		word-break: break-all;
 	}
 
 	@include mixins.e(footer) {

packages/design-system/theme/src/message-box.scss

@@ -48,6 +48,7 @@
 		font-size: var.$messagebox-font-size;
 		line-height: 1;
 		color: var.$messagebox-title-color;
+		font-weight: var(--font-weight-regular);
 	}
 
 	@include mixins.e(headerbtn) {
@@ -129,6 +130,7 @@
 
 	@include mixins.e(message) {
 		margin: 0;
+		font-weight: var(--font-weight-regular);
 
 		& p {
 			margin: 0;

packages/design-system/theme/src/reset.scss

@@ -11,7 +11,7 @@ body {
 	overscroll-behavior-x: none;
 	line-height: 1;
 	font-size: var(--font-size-m);
-	font-weight: 300;
+	font-weight: var(--font-weight-regular);
 	color: var(--color-text-dark);
 	-webkit-font-smoothing: antialiased;
 	-moz-osx-font-smoothing: grayscale;

packages/design-system/theme/src/tabs.scss

@@ -5,7 +5,7 @@
 	@include mixins.e(header) {
 		padding: 0;
 		position: relative;
-		margin: 0 0 15px;
+		margin: 0;
 	}
 	@include mixins.e(active-bar) {
 		position: absolute;