From 051598d30ecf8fff35864af6374ff49d09d43925 Mon Sep 17 00:00:00 2001 From: Ben Hesseldieck <1849459+BHesseldieck@users.noreply.github.com> Date: Wed, 29 Jul 2020 14:12:54 +0200 Subject: [PATCH] :sparkles: Add max execution time for Workflows (#755) * :tada: basic setup and execution stopping * :construction: soft timeout for own process executions * :construction: add hard timeout for subprocesses * :construction: add soft timeout to main thread * :wrench: set default timeout to 5 mins --> 500s * :bulb: adding documentation to configs * :construction: deactivate timeout by default * :construction: add logic of max execution timeout * :zap: adding timeout to settings in frontend and server * :art: improve naming * :bulb: fix change in config docs * :heavy_check_mark: fixing compilation issue * :art: add format for new config variables * :ok_hand: type cast before checking equality * :zap: Improve error message if NodeType is not known * :whale: Tag also rpi latest image * :bug: Fix Postgres issue with Node.js 14 #776 * :construction: add toggle to activate workflow timeout * :lipstick: improving UX of setting a timeout and its duration Co-authored-by: Jan Oberhauser --- .github/workflows/docker-images-rpi.yml | 4 + packages/cli/config/index.ts | 24 +++++ packages/cli/src/ActiveExecutions.ts | 17 ++-- packages/cli/src/Interfaces.ts | 5 +- packages/cli/src/NodeTypes.ts | 3 + packages/cli/src/Server.ts | 11 ++- packages/cli/src/WorkflowRunner.ts | 60 ++++++++++-- packages/cli/src/WorkflowRunnerProcess.ts | 13 +-- packages/core/src/WorkflowExecute.ts | 3 + packages/editor-ui/src/Interface.ts | 9 ++ .../src/components/WorkflowSettings.vue | 92 ++++++++++++++++++- packages/editor-ui/src/store.ts | 14 +++ packages/editor-ui/src/views/NodeView.vue | 2 + 13 files changed, 232 insertions(+), 25 deletions(-) diff --git a/.github/workflows/docker-images-rpi.yml b/.github/workflows/docker-images-rpi.yml index c6db9ed95b..34c8761ecd 100644 --- a/.github/workflows/docker-images-rpi.yml +++ b/.github/workflows/docker-images-rpi.yml @@ -26,3 +26,7 @@ jobs: if: success() run: | docker buildx build --platform linux/arm/v7 --build-arg N8N_VERSION=${{steps.vars.outputs.tag}} -t n8nio/n8n:${{steps.vars.outputs.tag}}-rpi --output type=image,push=true docker/images/n8n-rpi + - name: Tag Docker image with latest + run: docker tag n8nio/n8n:${{steps.vars.outputs.tag}}-rpi n8nio/n8n:latest-rpi + - name: Push docker images of latest + run: docker push n8nio/n8n:latest-rpi diff --git a/packages/cli/config/index.ts b/packages/cli/config/index.ts index 2759c6d794..8f34f6d4c0 100644 --- a/packages/cli/config/index.ts +++ b/packages/cli/config/index.ts @@ -159,6 +159,30 @@ const config = convict({ env: 'EXECUTIONS_PROCESS' }, + // A Workflow times out and gets canceled after this time (seconds). + // If the workflow is executed in the main process a soft timeout + // is executed (takes effect after the current node finishes). + // If a workflow is running in its own process is a soft timeout + // tried first, before killing the process after waiting for an + // additional fifth of the given timeout duration. + // + // To deactivate timeout set it to -1 + // + // Timeout is currently not activated by default which will change + // in a future version. + timeout: { + doc: 'Max run time (seconds) before stopping the workflow execution', + format: Number, + default: -1, + env: 'EXECUTIONS_TIMEOUT' + }, + maxTimeout: { + doc: 'Max execution time (seconds) that can be set for a workflow individually', + format: Number, + default: 3600, + env: 'EXECUTIONS_TIMEOUT_MAX' + }, + // If a workflow executes all the data gets saved by default. This // could be a problem when a workflow gets executed a lot and processes // a lot of data. To not exceed the database's capacity it is possible to diff --git a/packages/cli/src/ActiveExecutions.ts b/packages/cli/src/ActiveExecutions.ts index 10323946f4..d62d09578b 100644 --- a/packages/cli/src/ActiveExecutions.ts +++ b/packages/cli/src/ActiveExecutions.ts @@ -88,10 +88,11 @@ export class ActiveExecutions { * Forces an execution to stop * * @param {string} executionId The id of the execution to stop + * @param {string} timeout String 'timeout' given if stop due to timeout * @returns {(Promise)} * @memberof ActiveExecutions */ - async stopExecution(executionId: string): Promise { + async stopExecution(executionId: string, timeout?: string): Promise { if (this.activeExecutions[executionId] === undefined) { // There is no execution running with that id return; @@ -101,17 +102,17 @@ export class ActiveExecutions { // returned that it gets then also resolved correctly. if (this.activeExecutions[executionId].process !== undefined) { // Workflow is running in subprocess - setTimeout(() => { - if (this.activeExecutions[executionId].process!.connected) { + if (this.activeExecutions[executionId].process!.connected) { + setTimeout(() => { + // execute on next event loop tick; this.activeExecutions[executionId].process!.send({ - type: 'stopExecution' + type: timeout ? timeout : 'stopExecution', }); - } - - }, 1); + }, 1) + } } else { // Workflow is running in current process - this.activeExecutions[executionId].workflowExecution!.cancel('Canceled by user'); + this.activeExecutions[executionId].workflowExecution!.cancel(); } return this.getPostExecutePromise(executionId); diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index 2aecd27466..bef98af6f9 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -286,17 +286,17 @@ export interface IN8nUISettings { saveDataErrorExecution: string; saveDataSuccessExecution: string; saveManualExecutions: boolean; + executionTimeout: number; + maxExecutionTimeout: number; timezone: string; urlBaseWebhook: string; versionCli: string; } - export interface IPackageVersions { cli: string; } - export interface IPushData { data: IPushDataExecutionFinished | IPushDataNodeExecuteAfter | IPushDataNodeExecuteBefore | IPushDataTestWebhook; type: IPushDataType; @@ -304,7 +304,6 @@ export interface IPushData { export type IPushDataType = 'executionFinished' | 'executionStarted' | 'nodeExecuteAfter' | 'nodeExecuteBefore' | 'testWebhookDeleted' | 'testWebhookReceived'; - export interface IPushDataExecutionFinished { data: IRun; executionIdActive: string; diff --git a/packages/cli/src/NodeTypes.ts b/packages/cli/src/NodeTypes.ts index f600e8b734..5f37a33763 100644 --- a/packages/cli/src/NodeTypes.ts +++ b/packages/cli/src/NodeTypes.ts @@ -29,6 +29,9 @@ class NodeTypesClass implements INodeTypes { } getByName(nodeType: string): INodeType | undefined { + if (this.nodeTypes[nodeType] === undefined) { + throw new Error(`The node-type "${nodeType}" is not known!`); + } return this.nodeTypes[nodeType].type; } } diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index 1791682f99..2b68e409be 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -113,6 +113,8 @@ class App { saveDataErrorExecution: string; saveDataSuccessExecution: string; saveManualExecutions: boolean; + executionTimeout: number; + maxExecutionTimeout: number; timezone: string; activeExecutionsInstance: ActiveExecutions.ActiveExecutions; push: Push.Push; @@ -133,6 +135,8 @@ class App { this.saveDataErrorExecution = config.get('executions.saveDataOnError') as string; this.saveDataSuccessExecution = config.get('executions.saveDataOnSuccess') as string; this.saveManualExecutions = config.get('executions.saveDataManualExecutions') as boolean; + this.executionTimeout = config.get('executions.timeout') as number; + this.maxExecutionTimeout = config.get('executions.maxTimeout') as number; this.timezone = config.get('generic.timezone') as string; this.restEndpoint = config.get('endpoints.rest') as string; @@ -482,9 +486,12 @@ class App { // Do not save when default got set delete newWorkflowData.settings.saveManualExecutions; } + if (parseInt(newWorkflowData.settings.executionTimeout as string) === this.executionTimeout) { + // Do not save when default got set + delete newWorkflowData.settings.executionTimeout + } } - newWorkflowData.updatedAt = this.getCurrentDate(); await Db.collections.Workflow!.update(id, newWorkflowData); @@ -1534,6 +1541,8 @@ class App { saveDataErrorExecution: this.saveDataErrorExecution, saveDataSuccessExecution: this.saveDataSuccessExecution, saveManualExecutions: this.saveManualExecutions, + executionTimeout: this.executionTimeout, + maxExecutionTimeout: this.maxExecutionTimeout, timezone: this.timezone, urlBaseWebhook: WebhookHelpers.getWebhookBaseUrl(), versionCli: this.versions!.cli, diff --git a/packages/cli/src/WorkflowRunner.ts b/packages/cli/src/WorkflowRunner.ts index c0d08f446c..5bb0120dfd 100644 --- a/packages/cli/src/WorkflowRunner.ts +++ b/packages/cli/src/WorkflowRunner.ts @@ -90,7 +90,6 @@ export class WorkflowRunner { WorkflowExecuteAdditionalData.pushExecutionFinished(executionMode, fullRunData, executionId); } - /** * Run the workflow * @@ -155,9 +154,27 @@ export class WorkflowRunner { this.activeExecutions.attachWorkflowExecution(executionId, workflowExecution); + // Soft timeout to stop workflow execution after current running node + let executionTimeout: NodeJS.Timeout; + let workflowTimeout = config.get('executions.timeout') as number > 0 && config.get('executions.timeout') as number; // initialize with default + if (data.workflowData.settings && data.workflowData.settings.executionTimeout) { + workflowTimeout = data.workflowData.settings!.executionTimeout as number > 0 && data.workflowData.settings!.executionTimeout as number // preference on workflow setting + } + + if (workflowTimeout) { + const timeout = Math.min(workflowTimeout, config.get('executions.maxTimeout') as number) * 1000; // as seconds + executionTimeout = setTimeout(() => { + this.activeExecutions.stopExecution(executionId, 'timeout') + }, timeout) + } + workflowExecution.then((fullRunData) => { + clearTimeout(executionTimeout); + if (workflowExecution.isCanceled) { + fullRunData.finished = false; + } this.activeExecutions.remove(executionId, fullRunData); - }); + }) return executionId; } @@ -218,24 +235,54 @@ export class WorkflowRunner { // Send all data to subprocess it needs to run the workflow subprocess.send({ type: 'startWorkflow', data } as IProcessMessage); + // Start timeout for the execution + let executionTimeout: NodeJS.Timeout; + let workflowTimeout = config.get('executions.timeout') as number > 0 && config.get('executions.timeout') as number; // initialize with default + if (data.workflowData.settings && data.workflowData.settings.executionTimeout) { + workflowTimeout = data.workflowData.settings!.executionTimeout as number > 0 && data.workflowData.settings!.executionTimeout as number // preference on workflow setting + } + + if (workflowTimeout) { + const timeout = Math.min(workflowTimeout, config.get('executions.maxTimeout') as number) * 1000; // as seconds + executionTimeout = setTimeout(() => { + this.activeExecutions.stopExecution(executionId, 'timeout') + + executionTimeout = setTimeout(() => subprocess.kill(), Math.max(timeout * 0.2, 5000)) // minimum 5 seconds + }, timeout) + } + + // Listen to data from the subprocess subprocess.on('message', (message: IProcessMessage) => { if (message.type === 'end') { + clearTimeout(executionTimeout); this.activeExecutions.remove(executionId!, message.data.runData); + } else if (message.type === 'processError') { - + clearTimeout(executionTimeout); const executionError = message.data.executionError as IExecutionError; - this.processError(executionError, startedAt, data.executionMode, executionId); } else if (message.type === 'processHook') { this.processHookMessage(workflowHooks, message.data as IProcessMessageDataHook); + } else if (message.type === 'timeout') { + // Execution timed out and its process has been terminated + const timeoutError = { message: 'Workflow execution timed out!' } as IExecutionError; + + this.processError(timeoutError, startedAt, data.executionMode, executionId); } }); - // Also get informed when the processes does exit especially when it did crash + // Also get informed when the processes does exit especially when it did crash or timed out subprocess.on('exit', (code, signal) => { - if (code !== 0) { + if (signal === 'SIGTERM'){ + // Execution timed out and its process has been terminated + const timeoutError = { + message: 'Workflow execution timed out!', + } as IExecutionError; + + this.processError(timeoutError, startedAt, data.executionMode, executionId); + } else if (code !== 0) { // Process did exit with error code, so something went wrong. const executionError = { message: 'Workflow execution process did crash for an unknown reason!', @@ -243,6 +290,7 @@ export class WorkflowRunner { this.processError(executionError, startedAt, data.executionMode, executionId); } + clearTimeout(executionTimeout); }); return executionId; diff --git a/packages/cli/src/WorkflowRunnerProcess.ts b/packages/cli/src/WorkflowRunnerProcess.ts index 5748038a4f..a0e9d0493f 100644 --- a/packages/cli/src/WorkflowRunnerProcess.ts +++ b/packages/cli/src/WorkflowRunnerProcess.ts @@ -190,17 +190,18 @@ process.on('message', async (message: IProcessMessage) => { // Once the workflow got executed make sure the process gets killed again process.exit(); - } else if (message.type === 'stopExecution') { + } else if (message.type === 'stopExecution' || message.type === 'timeout') { // The workflow execution should be stopped let runData: IRun; if (workflowRunner.workflowExecute !== undefined) { // Workflow started already executing - runData = workflowRunner.workflowExecute.getFullRunData(workflowRunner.startedAt); - // If there is any data send it to parent process - await workflowRunner.workflowExecute.processSuccessExecution(workflowRunner.startedAt, workflowRunner.workflow!); + const timeOutError = message.type === 'timeout' ? { message: 'Workflow execution timed out!' } as IExecutionError : undefined + + // If there is any data send it to parent process, if execution timedout add the error + await workflowRunner.workflowExecute.processSuccessExecution(workflowRunner.startedAt, workflowRunner.workflow!, timeOutError); } else { // Workflow did not get started yet runData = { @@ -209,7 +210,7 @@ process.on('message', async (message: IProcessMessage) => { runData: {}, }, }, - finished: true, + finished: message.type !== 'timeout', mode: workflowRunner.data!.executionMode, startedAt: workflowRunner.startedAt, stoppedAt: new Date(), @@ -218,7 +219,7 @@ process.on('message', async (message: IProcessMessage) => { workflowRunner.sendHookToParentProcess('workflowExecuteAfter', [runData]); } - await sendToParentProcess('end', { + await sendToParentProcess(message.type === 'timeout' ? message.type : 'end', { runData, }); diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index c30be39ca1..072689acfd 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -708,6 +708,9 @@ export class WorkflowExecute { return Promise.resolve(); })() .then(async () => { + if (gotCancel && executionError === undefined) { + return this.processSuccessExecution(startedAt, workflow, { message: 'Workflow has been canceled!' } as IExecutionError); + } return this.processSuccessExecution(startedAt, workflow, executionError); }) .catch(async (error) => { diff --git a/packages/editor-ui/src/Interface.ts b/packages/editor-ui/src/Interface.ts index 63ea4223a9..881e938b60 100644 --- a/packages/editor-ui/src/Interface.ts +++ b/packages/editor-ui/src/Interface.ts @@ -397,6 +397,8 @@ export interface IN8nUISettings { saveDataSuccessExecution: string; saveManualExecutions: boolean; timezone: string; + executionTimeout: number; + maxExecutionTimeout: number; urlBaseWebhook: string; versionCli: string; } @@ -407,4 +409,11 @@ export interface IWorkflowSettings extends IWorkflowSettingsWorkflow { saveDataSuccessExecution?: string; saveManualExecutions?: boolean; timezone?: string; + executionTimeout?: number; +} + +export interface ITimeoutHMS { + hours: number; + minutes: number; + seconds: number; } diff --git a/packages/editor-ui/src/components/WorkflowSettings.vue b/packages/editor-ui/src/components/WorkflowSettings.vue index 178d51a813..ff5e8e57d7 100644 --- a/packages/editor-ui/src/components/WorkflowSettings.vue +++ b/packages/editor-ui/src/components/WorkflowSettings.vue @@ -1,6 +1,6 @@