Add possibility to execute workflows in same process

This commit is contained in:
Jan Oberhauser 2020-01-17 19:34:31 -06:00
parent 3fe47ab89e
commit 95cb1b2788
9 changed files with 353 additions and 209 deletions

View file

@ -77,6 +77,20 @@ These settings can also be overwritten on a per workflow basis in the workflow
settings in the Editor UI.
## Execute In Same Process
All workflows get executed in their own separate process. This ensures that all CPU cores
get used and that they do not block each other on CPU intensive tasks. Additionally does
the crash of one execution not take down the whole application. The disadvantage is, however,
that it slows down the start-time considerably and uses much more memory. So in case, the
workflows are not CPU intensive and they have to start very fast it is possible to run them
all directly in the main-process with this setting.
```bash
export EXECUTIONS_SAME_PROCESS=true
```
## Exclude Nodes
It is possible to not allow users to use nodes of a specific node type. If you, for example,

View file

@ -84,6 +84,12 @@ const config = convict({
default: false,
env: 'EXECUTIONS_DATA_SAVE_MANUAL_EXECUTIONS'
},
sameProcessExecution: {
doc: 'Executes the workflows in the same process instead of in a separate one',
default: false,
env: 'EXECUTIONS_SAME_PROCESS'
},
},
generic: {

View file

@ -66,6 +66,7 @@
"@types/request-promise-native": "^1.0.15",
"jest": "^24.9.0",
"nodemon": "^2.0.2",
"p-cancelable": "^2.0.0",
"run-script-os": "^1.0.7",
"ts-jest": "^24.0.2",
"tslint": "^5.17.0",

View file

@ -13,6 +13,7 @@ import {
} from '.';
import { ChildProcess } from 'child_process';
import * as PCancelable from 'p-cancelable';
export class ActiveExecutions {
@ -30,7 +31,7 @@ export class ActiveExecutions {
* @returns {string}
* @memberof ActiveExecutions
*/
add(process: ChildProcess, executionData: IWorkflowExecutionDataProcess): string {
add(executionData: IWorkflowExecutionDataProcess, process?: ChildProcess): string {
const executionId = this.nextId++;
this.activeExecutions[executionId] = {
@ -44,6 +45,22 @@ export class ActiveExecutions {
}
/**
* Attaches an execution
*
* @param {string} executionId
* @param {PCancelable<IRun>} workflowExecution
* @memberof ActiveExecutions
*/
attachWorkflowExecution(executionId: string, workflowExecution: PCancelable<IRun>) {
if (this.activeExecutions[executionId] === undefined) {
throw new Error(`No active execution with id "${executionId}" got found to attach to workflowExecution to!`);
}
this.activeExecutions[executionId].workflowExecution = workflowExecution;
}
/**
* Remove an active execution
*
@ -82,13 +99,20 @@ export class ActiveExecutions {
// In case something goes wrong make sure that promise gets first
// returned that it gets then also resolved correctly.
if (this.activeExecutions[executionId].process !== undefined) {
// Workflow is running in subprocess
setTimeout(() => {
if (this.activeExecutions[executionId].process.connected) {
this.activeExecutions[executionId].process.send({
if (this.activeExecutions[executionId].process!.connected) {
this.activeExecutions[executionId].process!.send({
type: 'stopExecution'
});
}
}, 1);
} else {
// Workflow is running in current process
this.activeExecutions[executionId].workflowExecution!.cancel('Canceled by user');
}
return this.getPostExecutePromise(executionId);
}

View file

@ -17,6 +17,7 @@ import {
} from 'n8n-core';
import * as PCancelable from 'p-cancelable';
import { ObjectID, Repository } from 'typeorm';
import { ChildProcess } from 'child_process';
@ -181,9 +182,10 @@ export interface IExecutionDeleteFilter {
export interface IExecutingWorkflowData {
executionData: IWorkflowExecutionDataProcess;
process: ChildProcess;
process?: ChildProcess;
startedAt: Date;
postExecutePromises: Array<IDeferredPromise<IRun | undefined>>;
workflowExecution?: PCancelable<IRun>;
}
export interface IN8nConfig {

View file

@ -15,7 +15,7 @@ class NodeTypesClass implements INodeTypes {
// Some nodeTypes need to get special parameters applied like the
// polling nodes the polling times
for (const nodeTypeData of Object.values(nodeTypes)) {
const applyParameters = NodeHelpers.getSpecialNodeParameters(nodeTypeData.type)
const applyParameters = NodeHelpers.getSpecialNodeParameters(nodeTypeData.type);
if (applyParameters.length) {
nodeTypeData.type.description.properties.unshift.apply(nodeTypeData.type.description.properties, applyParameters);

View file

@ -4,6 +4,7 @@ import {
ITransferNodeTypes,
IWorkflowExecutionDataProcess,
IWorkflowExecutionDataProcessWithExecution,
NodeTypes,
Push,
WorkflowExecuteAdditionalData,
WorkflowHelpers,
@ -11,15 +12,19 @@ import {
import {
IProcessMessage,
WorkflowExecute,
} from 'n8n-core';
import {
IExecutionError,
IRun,
Workflow,
WorkflowHooks,
WorkflowExecuteMode,
} from 'n8n-workflow';
import * as config from '../config';
import * as PCancelable from 'p-cancelable';
import { join as pathJoin } from 'path';
import { fork } from 'child_process';
@ -80,7 +85,7 @@ export class WorkflowRunner {
/**
* Run the workflow in subprocess
* Run the workflow
*
* @param {IWorkflowExecutionDataProcess} data
* @param {boolean} [loadStaticData] If set will the static data be loaded from
@ -89,6 +94,70 @@ export class WorkflowRunner {
* @memberof WorkflowRunner
*/
async run(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean): Promise<string> {
const sameProcessExecution = config.get('executions.sameProcessExecution') as boolean;
if (sameProcessExecution === true) {
return this.runSameProcess(data, loadStaticData);
}
return this.runSubprocess(data, loadStaticData);
}
/**
* Run the workflow in current process
*
* @param {IWorkflowExecutionDataProcess} data
* @param {boolean} [loadStaticData] If set will the static data be loaded from
* the workflow and added to input data
* @returns {Promise<string>}
* @memberof WorkflowRunner
*/
async runSameProcess(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean): Promise<string> {
if (loadStaticData === true && data.workflowData.id) {
data.workflowData.staticData = await WorkflowHelpers.getStaticDataById(data.workflowData.id as string);
}
const nodeTypes = NodeTypes();
const workflow = new Workflow(data.workflowData.id as string | undefined, data.workflowData!.nodes, data.workflowData!.connections, data.workflowData!.active, nodeTypes, data.workflowData!.staticData);
const additionalData = await WorkflowExecuteAdditionalData.getBase(data.credentials);
// Register the active execution
const executionId = this.activeExecutions.add(data, undefined);
additionalData.hooks = WorkflowExecuteAdditionalData.getWorkflowHooksMain(data, executionId);
let workflowExecution: PCancelable<IRun>;
if (data.executionData !== undefined) {
const workflowExecute = new WorkflowExecute(additionalData, data.executionMode, data.executionData);
workflowExecution = workflowExecute.processRunExecutionData(workflow);
} else if (data.runData === undefined || data.startNodes === undefined || data.startNodes.length === 0 || data.destinationNode === undefined) {
// Execute all nodes
// Can execute without webhook so go on
const workflowExecute = new WorkflowExecute(additionalData, data.executionMode);
workflowExecution = workflowExecute.run(workflow, undefined, data.destinationNode);
} else {
// Execute only the nodes between start and destination nodes
const workflowExecute = new WorkflowExecute(additionalData, data.executionMode);
workflowExecution = workflowExecute.runPartialWorkflow(workflow, data.runData, data.startNodes, data.destinationNode);
}
this.activeExecutions.attachWorkflowExecution(executionId, workflowExecution);
return executionId;
}
/**
* Run the workflow
*
* @param {IWorkflowExecutionDataProcess} data
* @param {boolean} [loadStaticData] If set will the static data be loaded from
* the workflow and added to input data
* @returns {Promise<string>}
* @memberof WorkflowRunner
*/
async runSubprocess(data: IWorkflowExecutionDataProcess, loadStaticData?: boolean): Promise<string> {
const startedAt = new Date();
const subprocess = fork(pathJoin(__dirname, 'WorkflowRunnerProcess.js'));
@ -97,7 +166,7 @@ export class WorkflowRunner {
}
// Register the active execution
const executionId = this.activeExecutions.add(subprocess, data);
const executionId = this.activeExecutions.add(data, subprocess);
// Check if workflow contains a "executeWorkflow" Node as in this
// case we can not know which nodeTypes will be needed and so have

View file

@ -44,6 +44,7 @@
"lodash.get": "^4.4.2",
"mmmagic": "^0.5.2",
"n8n-workflow": "~0.20.0",
"p-cancelable": "^2.0.0",
"request-promise-native": "^1.0.7"
},
"jest": {

View file

@ -1,3 +1,5 @@
import * as PCancelable from 'p-cancelable';
import {
IConnection,
IDataObject,
@ -54,7 +56,7 @@ export class WorkflowExecute {
* @returns {(Promise<string>)}
* @memberof WorkflowExecute
*/
async run(workflow: Workflow, startNode?: INode, destinationNode?: string): Promise<IRun> {
run(workflow: Workflow, startNode?: INode, destinationNode?: string): PCancelable<IRun> {
// Get the nodes to start workflow execution from
startNode = startNode || workflow.getStartNode(destinationNode);
@ -115,7 +117,8 @@ export class WorkflowExecute {
* @returns {(Promise<string>)}
* @memberof WorkflowExecute
*/
async runPartialWorkflow(workflow: Workflow, runData: IRunData, startNodes: string[], destinationNode: string): Promise<IRun> {
// @ts-ignore
async runPartialWorkflow(workflow: Workflow, runData: IRunData, startNodes: string[], destinationNode: string): PCancelable<IRun> {
let incomingNodeConnections: INodeConnections | undefined;
let connection: IConnection;
@ -209,7 +212,7 @@ export class WorkflowExecute {
},
};
return await this.processRunExecutionData(workflow);
return this.processRunExecutionData(workflow);
}
@ -444,7 +447,7 @@ export class WorkflowExecute {
* @returns {Promise<string>}
* @memberof WorkflowExecute
*/
async processRunExecutionData(workflow: Workflow): Promise<IRun> {
processRunExecutionData(workflow: Workflow): PCancelable<IRun> {
const startedAt = new Date();
const workflowIssues = workflow.checkReadyForExecution();
@ -470,9 +473,26 @@ export class WorkflowExecute {
let currentExecutionTry = '';
let lastExecutionTry = '';
return (async () => {
return new PCancelable((resolve, reject, onCancel) => {
let gotCancel = false;
onCancel.shouldReject = false;
onCancel(() => {
console.log('got cancellled');
gotCancel = true;
});
const returnPromise = (async () => {
executionLoop:
while (this.runExecutionData.executionData!.nodeExecutionStack.length !== 0) {
// @ts-ignore
if (gotCancel === true) {
return Promise.resolve();
}
nodeSuccessData = null;
executionError = undefined;
executionData = this.runExecutionData.executionData!.nodeExecutionStack.shift() as IExecuteData;
@ -555,6 +575,10 @@ export class WorkflowExecute {
}
for (let tryIndex = 0; tryIndex < maxTries; tryIndex++) {
// @ts-ignore
if (gotCancel === true) {
return Promise.resolve();
}
try {
if (tryIndex !== 0) {
@ -691,10 +715,13 @@ export class WorkflowExecute {
return fullRunData;
});
return returnPromise.then(resolve);
});
}
async processSuccessExecution(startedAt: Date, workflow: Workflow, executionError?: IExecutionError): Promise<IRun> {
// @ts-ignore
async processSuccessExecution(startedAt: Date, workflow: Workflow, executionError?: IExecutionError): PCancelable<IRun> {
const fullRunData = this.getFullRunData(startedAt);
if (executionError !== undefined) {