✨ Add possibility to execute workflows in same process

docs/configuration.md

@@ -77,6 +77,20 @@ These settings can also be overwritten on a per workflow basis in the workflow
 settings in the Editor UI.
 
 
+## Execute In Same Process
+
+All workflows get executed in their own separate process. This ensures that all CPU cores
+get used and that they do not block each other on CPU intensive tasks. Additionally does
+the crash of one execution not take down the whole application. The disadvantage is, however,
+that it slows down the start-time considerably and uses much more memory. So in case, the
+workflows are not CPU intensive and they have to start very fast it is possible to run them
+all directly in the main-process with this setting.
+
+```bash
+export EXECUTIONS_SAME_PROCESS=true
+```
+
+
 ## Exclude Nodes
 
 It is possible to not allow users to use nodes of a specific node type. If you, for example,

packages/cli/config/index.ts

@@ -84,6 +84,12 @@ const config = convict({
 			default: false,
 			env: 'EXECUTIONS_DATA_SAVE_MANUAL_EXECUTIONS'
 		},
+
+		sameProcessExecution: {
+			doc: 'Executes the workflows in the same process instead of in a separate one',
+			default: false,
+			env: 'EXECUTIONS_SAME_PROCESS'
+		},
 	},
 
 	generic: {

packages/cli/package.json

@@ -66,6 +66,7 @@
         "@types/request-promise-native": "^1.0.15",
         "jest": "^24.9.0",
         "nodemon": "^2.0.2",
+        "p-cancelable": "^2.0.0",
         "run-script-os": "^1.0.7",
         "ts-jest": "^24.0.2",
         "tslint": "^5.17.0",

packages/cli/src/ActiveExecutions.ts

@@ -13,6 +13,7 @@ import {
 } from '.';
 
 import { ChildProcess } from 'child_process';
+import * as PCancelable from 'p-cancelable';
 
 
 export class ActiveExecutions {
@@ -30,7 +31,7 @@ export class ActiveExecutions {
 	 * @returns {string}
 	 * @memberof ActiveExecutions
 	 */
-	add(process: ChildProcess, executionData: IWorkflowExecutionDataProcess): string {
+	add(executionData: IWorkflowExecutionDataProcess, process?: ChildProcess): string {
 		const executionId = this.nextId++;
 
 		this.activeExecutions[executionId] = {
@@ -44,6 +45,22 @@ export class ActiveExecutions {
 	}
 
 
+	/**
+	 * Attaches an execution
+	 *
+	 * @param {string} executionId
+	 * @param {PCancelable<IRun>} workflowExecution
+	 * @memberof ActiveExecutions
+	 */
+	attachWorkflowExecution(executionId: string, workflowExecution: PCancelable<IRun>) {
+		if (this.activeExecutions[executionId] === undefined) {
+			throw new Error(`No active execution with id "${executionId}" got found to attach to workflowExecution to!`);
+		}
+
+		this.activeExecutions[executionId].workflowExecution = workflowExecution;
+	}
+
+
 	/**
 	 * Remove an active execution
 	 *
@@ -82,13 +99,20 @@ export class ActiveExecutions {
 
 		// In case something goes wrong make sure that promise gets first
 		// returned that it gets then also resolved correctly.
+		if (this.activeExecutions[executionId].process !== undefined) {
+			// Workflow is running in subprocess
 			setTimeout(() => {
-			if (this.activeExecutions[executionId].process.connected) {
+				if (this.activeExecutions[executionId].process!.connected) {
-				this.activeExecutions[executionId].process.send({
+					this.activeExecutions[executionId].process!.send({
 						type: 'stopExecution'
 					});
 				}
+
 			}, 1);
+		} else {
+			// Workflow is running in current process
+			this.activeExecutions[executionId].workflowExecution!.cancel('Canceled by user');
+		}
 
 		return this.getPostExecutePromise(executionId);
 	}

packages/cli/src/Interfaces.ts

@@ -17,6 +17,7 @@ import {
 } from 'n8n-core';
 
 
+import * as PCancelable from 'p-cancelable';
 import { ObjectID, Repository } from 'typeorm';
 
 import { ChildProcess } from 'child_process';
@@ -181,9 +182,10 @@ export interface IExecutionDeleteFilter {
 
 export interface IExecutingWorkflowData {
 	executionData: IWorkflowExecutionDataProcess;
-	process: ChildProcess;
+	process?: ChildProcess;
 	startedAt: Date;
 	postExecutePromises: Array<IDeferredPromise<IRun | undefined>>;
+	workflowExecution?: PCancelable<IRun>;
 }
 
 export interface IN8nConfig {

packages/cli/src/NodeTypes.ts

@@ -15,7 +15,7 @@ class NodeTypesClass implements INodeTypes {
 		// Some nodeTypes need to get special parameters applied like the
 		// polling nodes the polling times
 		for (const nodeTypeData of Object.values(nodeTypes)) {
-			const applyParameters = NodeHelpers.getSpecialNodeParameters(nodeTypeData.type)
+			const applyParameters = NodeHelpers.getSpecialNodeParameters(nodeTypeData.type);
 
 			if (applyParameters.length) {
 				nodeTypeData.type.description.properties.unshift.apply(nodeTypeData.type.description.properties, applyParameters);

packages/cli/src/WorkflowRunner.ts

@@ -4,6 +4,7 @@ import {
 	ITransferNodeTypes,
 	IWorkflowExecutionDataProcess,
 	IWorkflowExecutionDataProcessWithExecution,
+	NodeTypes,
 	Push,
 	WorkflowExecuteAdditionalData,
 	WorkflowHelpers,
@@ -11,15 +12,19 @@ import {
 
 import {
 	IProcessMessage,
+	WorkflowExecute,
 } from 'n8n-core';
 
 import {
 	IExecutionError,
 	IRun,
+	Workflow,
 	WorkflowHooks,
 	WorkflowExecuteMode,
 } from 'n8n-workflow';
 
+import * as config from '../config';
+import * as PCancelable from 'p-cancelable';
 import { join as pathJoin } from 'path';
 import { fork } from 'child_process';
 
@@ -80,7 +85,7 @@ export class WorkflowRunner {
 
 
 	/**
-	 * Run the workflow in subprocess
+	 * Run the workflow
 	 *
 	 * @param {IWorkflowExecutionDataProcess} data
 	 * @param {boolean} [loadStaticData] If set will the static data be loaded from
@@ -89,6 +94,70 @@ export class WorkflowRunner {
 	 * @memberof WorkflowRunner
 	 */
 	async run(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean): Promise<string> {
+		const sameProcessExecution = config.get('executions.sameProcessExecution') as boolean;
+		if (sameProcessExecution === true) {
+			return this.runSameProcess(data, loadStaticData);
+		}
+
+		return this.runSubprocess(data, loadStaticData);
+	}
+
+
+	/**
+	 * Run the workflow in current process
+	 *
+	 * @param {IWorkflowExecutionDataProcess} data
+	 * @param {boolean} [loadStaticData] If set will the static data be loaded from
+	 *                                   the workflow and added to input data
+	 * @returns {Promise<string>}
+	 * @memberof WorkflowRunner
+	 */
+	async runSameProcess(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean): Promise<string> {
+		if (loadStaticData === true && data.workflowData.id) {
+			data.workflowData.staticData = await WorkflowHelpers.getStaticDataById(data.workflowData.id as string);
+		}
+
+		const nodeTypes = NodeTypes();
+
+		const workflow = new Workflow(data.workflowData.id as string | undefined, data.workflowData!.nodes, data.workflowData!.connections, data.workflowData!.active, nodeTypes, data.workflowData!.staticData);
+		const additionalData = await WorkflowExecuteAdditionalData.getBase(data.credentials);
+
+		// Register the active execution
+		const executionId = this.activeExecutions.add(data, undefined);
+
+		additionalData.hooks = WorkflowExecuteAdditionalData.getWorkflowHooksMain(data, executionId);
+
+		let workflowExecution: PCancelable<IRun>;
+		if (data.executionData !== undefined) {
+			const workflowExecute = new WorkflowExecute(additionalData, data.executionMode, data.executionData);
+			workflowExecution = workflowExecute.processRunExecutionData(workflow);
+		} else if (data.runData === undefined || data.startNodes === undefined || data.startNodes.length === 0 || data.destinationNode === undefined) {
+			// Execute all nodes
+
+			// Can execute without webhook so go on
+			const workflowExecute = new WorkflowExecute(additionalData, data.executionMode);
+			workflowExecution = workflowExecute.run(workflow, undefined, data.destinationNode);
+		} else {
+			// Execute only the nodes between start and destination nodes
+			const workflowExecute = new WorkflowExecute(additionalData, data.executionMode);
+			workflowExecution = workflowExecute.runPartialWorkflow(workflow, data.runData, data.startNodes, data.destinationNode);
+		}
+
+		this.activeExecutions.attachWorkflowExecution(executionId, workflowExecution);
+
+		return executionId;
+	}
+
+	/**
+	 * Run the workflow
+	 *
+	 * @param {IWorkflowExecutionDataProcess} data
+	 * @param {boolean} [loadStaticData] If set will the static data be loaded from
+	 *                                   the workflow and added to input data
+	 * @returns {Promise<string>}
+	 * @memberof WorkflowRunner
+	 */
+	async runSubprocess(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean): Promise<string> {
 		const startedAt = new Date();
 		const subprocess = fork(pathJoin(__dirname, 'WorkflowRunnerProcess.js'));
 
@@ -97,7 +166,7 @@ export class WorkflowRunner {
 		}
 
 		// Register the active execution
-		const executionId = this.activeExecutions.add(subprocess, data);
+		const executionId = this.activeExecutions.add(data, subprocess);
 
 		// Check if workflow contains a "executeWorkflow" Node as in this
 		// case we can not know which nodeTypes will be needed and so have

packages/core/package.json

@@ -44,6 +44,7 @@
         "lodash.get": "^4.4.2",
         "mmmagic": "^0.5.2",
         "n8n-workflow": "~0.20.0",
+        "p-cancelable": "^2.0.0",
         "request-promise-native": "^1.0.7"
     },
     "jest": {

packages/core/src/WorkflowExecute.ts

@@ -1,3 +1,5 @@
+import * as PCancelable from 'p-cancelable';
+
 import {
 	IConnection,
 	IDataObject,
@@ -54,7 +56,7 @@ export class WorkflowExecute {
 	 * @returns {(Promise<string>)}
 	 * @memberof WorkflowExecute
 	 */
-	async run(workflow: Workflow, startNode?: INode, destinationNode?: string): Promise<IRun> {
+	run(workflow: Workflow, startNode?: INode, destinationNode?: string): PCancelable<IRun> {
 		// Get the nodes to start workflow execution from
 		startNode = startNode || workflow.getStartNode(destinationNode);
 
@@ -115,7 +117,8 @@ export class WorkflowExecute {
 	 * @returns {(Promise<string>)}
 	 * @memberof WorkflowExecute
 	 */
-	async runPartialWorkflow(workflow: Workflow, runData: IRunData, startNodes: string[], destinationNode: string): Promise<IRun> {
+	// @ts-ignore
+	async runPartialWorkflow(workflow: Workflow, runData: IRunData, startNodes: string[], destinationNode: string): PCancelable<IRun> {
 		let incomingNodeConnections: INodeConnections | undefined;
 		let connection: IConnection;
 
@@ -209,7 +212,7 @@ export class WorkflowExecute {
 			},
 		};
 
-		return await this.processRunExecutionData(workflow);
+		return this.processRunExecutionData(workflow);
 	}
 
 
@@ -444,7 +447,7 @@ export class WorkflowExecute {
 	 * @returns {Promise<string>}
 	 * @memberof WorkflowExecute
 	 */
-	async processRunExecutionData(workflow: Workflow): Promise<IRun> {
+	processRunExecutionData(workflow: Workflow): PCancelable<IRun> {
 		const startedAt = new Date();
 
 		const workflowIssues = workflow.checkReadyForExecution();
@@ -470,9 +473,26 @@ export class WorkflowExecute {
 		let currentExecutionTry = '';
 		let lastExecutionTry = '';
 
-		return (async () => {
+		return new PCancelable((resolve, reject, onCancel) => {
+			let gotCancel = false;
+
+			onCancel.shouldReject = false;
+			onCancel(() => {
+				console.log('got cancellled');
+
+				gotCancel = true;
+			});
+
+			const returnPromise = (async () => {
+
 				executionLoop:
 				while (this.runExecutionData.executionData!.nodeExecutionStack.length !== 0) {
+
+					// @ts-ignore
+					if (gotCancel === true) {
+						return Promise.resolve();
+					}
+
 					nodeSuccessData = null;
 					executionError = undefined;
 					executionData = this.runExecutionData.executionData!.nodeExecutionStack.shift() as IExecuteData;
@@ -555,6 +575,10 @@ export class WorkflowExecute {
 					}
 
 					for (let tryIndex = 0; tryIndex < maxTries; tryIndex++) {
+						// @ts-ignore
+						if (gotCancel === true) {
+							return Promise.resolve();
+						}
 						try {
 
 							if (tryIndex !== 0) {
@@ -691,10 +715,13 @@ export class WorkflowExecute {
 				return fullRunData;
 			});
 
+			return returnPromise.then(resolve);
+		});
 	}
 
 
-	async processSuccessExecution(startedAt: Date, workflow: Workflow, executionError?: IExecutionError): Promise<IRun> {
+	// @ts-ignore
+	async processSuccessExecution(startedAt: Date, workflow: Workflow, executionError?: IExecutionError): PCancelable<IRun> {
 		const fullRunData = this.getFullRunData(startedAt);
 
 		if (executionError !== undefined) {