✨ Add max execution time for Workflows (#755)

* 🎉 basic setup and execution stopping

* 🚧 soft timeout for own process executions

* 🚧 add hard timeout for subprocesses

* 🚧 add soft timeout to main thread

* 🔧 set default timeout to 5 mins --> 500s

* 💡 adding documentation to configs

* 🚧 deactivate timeout by default

* 🚧 add logic of max execution timeout

* ⚡ adding timeout to settings in frontend and server

* 🎨 improve naming

* 💡 fix change in config docs

* ✔️ fixing compilation issue

* 🎨 add format for new config variables

* 👌 type cast before checking equality

* ⚡ Improve error message if NodeType is not known

* 🐳 Tag also rpi latest image

* 🐛 Fix Postgres issue with Node.js 14 #776

* 🚧 add toggle to activate workflow timeout

* 💄 improving UX of setting a timeout and its duration

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>

.github/workflows/docker-images-rpi.yml

@@ -26,3 +26,7 @@ jobs:
         if: success()
         run: |
           docker buildx build --platform linux/arm/v7 --build-arg N8N_VERSION=${{steps.vars.outputs.tag}} -t n8nio/n8n:${{steps.vars.outputs.tag}}-rpi --output type=image,push=true docker/images/n8n-rpi
+      - name: Tag Docker image with latest
+        run: docker tag n8nio/n8n:${{steps.vars.outputs.tag}}-rpi n8nio/n8n:latest-rpi
+      - name: Push docker images of latest
+        run: docker push n8nio/n8n:latest-rpi

packages/cli/config/index.ts

@@ -159,6 +159,30 @@ const config = convict({
 			env: 'EXECUTIONS_PROCESS'
 		},
 
+		// A Workflow times out and gets canceled after this time (seconds).
+		// If the workflow is executed in the main process a soft timeout
+		// is executed (takes effect after the current node finishes).
+		// If a workflow is running in its own process is a soft timeout
+		// tried first, before killing the process after waiting for an
+		// additional fifth of the given timeout duration.
+		//
+		// To deactivate timeout set it to -1
+		//
+		// Timeout is currently not activated by default which will change
+		// in a future version.
+		timeout: {
+			doc: 'Max run time (seconds) before stopping the workflow execution',
+			format: Number,
+			default: -1,
+			env: 'EXECUTIONS_TIMEOUT'
+		},
+		maxTimeout: {
+			doc: 'Max execution time (seconds) that can be set for a workflow individually',
+			format: Number,
+			default: 3600,
+			env: 'EXECUTIONS_TIMEOUT_MAX'
+		},
+
 		// If a workflow executes all the data gets saved by default. This
 		// could be a problem when a workflow gets executed a lot and processes
 		// a lot of data. To not exceed the database's capacity it is possible to

packages/cli/src/ActiveExecutions.ts

@@ -88,10 +88,11 @@ export class ActiveExecutions {
 	 * Forces an execution to stop
 	 *
 	 * @param {string} executionId The id of the execution to stop
+	 * @param {string} timeout String 'timeout' given if stop due to timeout
 	 * @returns {(Promise<IRun | undefined>)}
 	 * @memberof ActiveExecutions
 	 */
-	async stopExecution(executionId: string): Promise<IRun | undefined> {
+	async stopExecution(executionId: string, timeout?: string): Promise<IRun | undefined> {
 		if (this.activeExecutions[executionId] === undefined) {
 			// There is no execution running with that id
 			return;
@@ -101,17 +102,17 @@ export class ActiveExecutions {
 		// returned that it gets then also resolved correctly.
 		if (this.activeExecutions[executionId].process !== undefined) {
 			// Workflow is running in subprocess
-			setTimeout(() => {
-				if (this.activeExecutions[executionId].process!.connected) {
+			if (this.activeExecutions[executionId].process!.connected) {
+				setTimeout(() => {
+				// execute on next event loop tick;
 					this.activeExecutions[executionId].process!.send({
-						type: 'stopExecution'
+						type: timeout ? timeout : 'stopExecution',
 					});
-				}
-
-			}, 1);
+				}, 1)
+			}
 		} else {
 			// Workflow is running in current process
-			this.activeExecutions[executionId].workflowExecution!.cancel('Canceled by user');
+			this.activeExecutions[executionId].workflowExecution!.cancel();
 		}
 
 		return this.getPostExecutePromise(executionId);

packages/cli/src/Interfaces.ts

@@ -286,17 +286,17 @@ export interface IN8nUISettings {
 	saveDataErrorExecution: string;
 	saveDataSuccessExecution: string;
 	saveManualExecutions: boolean;
+	executionTimeout: number;
+	maxExecutionTimeout: number;
 	timezone: string;
 	urlBaseWebhook: string;
 	versionCli: string;
 }
 
-
 export interface IPackageVersions {
 	cli: string;
 }
 
-
 export interface IPushData {
 	data: IPushDataExecutionFinished | IPushDataNodeExecuteAfter | IPushDataNodeExecuteBefore | IPushDataTestWebhook;
 	type: IPushDataType;
@@ -304,7 +304,6 @@ export interface IPushData {
 
 export type IPushDataType = 'executionFinished' | 'executionStarted' | 'nodeExecuteAfter' | 'nodeExecuteBefore' | 'testWebhookDeleted' | 'testWebhookReceived';
 
-
 export interface IPushDataExecutionFinished {
 	data: IRun;
 	executionIdActive: string;

packages/cli/src/NodeTypes.ts

@@ -29,6 +29,9 @@ class NodeTypesClass implements INodeTypes {
 	}
 
 	getByName(nodeType: string): INodeType | undefined {
+		if (this.nodeTypes[nodeType] === undefined) {
+			throw new Error(`The node-type "${nodeType}" is not known!`);
+		}
 		return this.nodeTypes[nodeType].type;
 	}
 }

packages/cli/src/Server.ts

@@ -113,6 +113,8 @@ class App {
 	saveDataErrorExecution: string;
 	saveDataSuccessExecution: string;
 	saveManualExecutions: boolean;
+	executionTimeout: number;
+	maxExecutionTimeout: number;
 	timezone: string;
 	activeExecutionsInstance: ActiveExecutions.ActiveExecutions;
 	push: Push.Push;
@@ -133,6 +135,8 @@ class App {
 		this.saveDataErrorExecution = config.get('executions.saveDataOnError') as string;
 		this.saveDataSuccessExecution = config.get('executions.saveDataOnSuccess') as string;
 		this.saveManualExecutions = config.get('executions.saveDataManualExecutions') as boolean;
+		this.executionTimeout = config.get('executions.timeout') as number;
+		this.maxExecutionTimeout = config.get('executions.maxTimeout') as number;
 		this.timezone = config.get('generic.timezone') as string;
 		this.restEndpoint = config.get('endpoints.rest') as string;
 
@@ -482,9 +486,12 @@ class App {
 					// Do not save when default got set
 					delete newWorkflowData.settings.saveManualExecutions;
 				}
+				if (parseInt(newWorkflowData.settings.executionTimeout as string) === this.executionTimeout) {
+					// Do not save when default got set
+					delete newWorkflowData.settings.executionTimeout
+				}
 			}
 
-
 			newWorkflowData.updatedAt = this.getCurrentDate();
 
 			await Db.collections.Workflow!.update(id, newWorkflowData);
@@ -1534,6 +1541,8 @@ class App {
 				saveDataErrorExecution: this.saveDataErrorExecution,
 				saveDataSuccessExecution: this.saveDataSuccessExecution,
 				saveManualExecutions: this.saveManualExecutions,
+				executionTimeout: this.executionTimeout,
+				maxExecutionTimeout: this.maxExecutionTimeout,
 				timezone: this.timezone,
 				urlBaseWebhook: WebhookHelpers.getWebhookBaseUrl(),
 				versionCli: this.versions!.cli,

packages/cli/src/WorkflowRunner.ts

@@ -90,7 +90,6 @@ export class WorkflowRunner {
 		WorkflowExecuteAdditionalData.pushExecutionFinished(executionMode, fullRunData, executionId);
 	}
 
-
 	/**
 	 * Run the workflow
 	 *
@@ -155,9 +154,27 @@ export class WorkflowRunner {
 
 		this.activeExecutions.attachWorkflowExecution(executionId, workflowExecution);
 
+		// Soft timeout to stop workflow execution after current running node
+		let executionTimeout: NodeJS.Timeout;
+		let workflowTimeout = config.get('executions.timeout') as number > 0 && config.get('executions.timeout') as number; // initialize with default
+		if (data.workflowData.settings && data.workflowData.settings.executionTimeout) {
+			workflowTimeout = data.workflowData.settings!.executionTimeout as number > 0 && data.workflowData.settings!.executionTimeout as number // preference on workflow setting
+		}
+
+		if (workflowTimeout) {
+			const timeout = Math.min(workflowTimeout, config.get('executions.maxTimeout') as number) * 1000; // as seconds
+			executionTimeout = setTimeout(() => {
+				this.activeExecutions.stopExecution(executionId, 'timeout')
+			}, timeout)
+		}
+
 		workflowExecution.then((fullRunData) => {
+			clearTimeout(executionTimeout);
+			if (workflowExecution.isCanceled) {
+				fullRunData.finished = false;
+			}
 			this.activeExecutions.remove(executionId, fullRunData);
-		});
+		})
 
 		return executionId;
 	}
@@ -218,24 +235,54 @@ export class WorkflowRunner {
 		// Send all data to subprocess it needs to run the workflow
 		subprocess.send({ type: 'startWorkflow', data } as IProcessMessage);
 
+		// Start timeout for the execution
+		let executionTimeout: NodeJS.Timeout;
+		let workflowTimeout = config.get('executions.timeout') as number > 0 && config.get('executions.timeout') as number; // initialize with default
+		if (data.workflowData.settings && data.workflowData.settings.executionTimeout) {
+			workflowTimeout = data.workflowData.settings!.executionTimeout as number > 0 && data.workflowData.settings!.executionTimeout as number // preference on workflow setting
+		}
+
+		if (workflowTimeout) {
+			const timeout = Math.min(workflowTimeout, config.get('executions.maxTimeout') as number) * 1000; // as seconds
+			executionTimeout = setTimeout(() => {
+				this.activeExecutions.stopExecution(executionId, 'timeout')
+
+				executionTimeout = setTimeout(() => subprocess.kill(), Math.max(timeout * 0.2, 5000)) // minimum 5 seconds
+			}, timeout)
+		}
+
+
 		// Listen to data from the subprocess
 		subprocess.on('message', (message: IProcessMessage) => {
 			if (message.type === 'end') {
+				clearTimeout(executionTimeout);
 				this.activeExecutions.remove(executionId!, message.data.runData);
+
 			} else if (message.type === 'processError') {
-
+				clearTimeout(executionTimeout);
 				const executionError = message.data.executionError as IExecutionError;
-
 				this.processError(executionError, startedAt, data.executionMode, executionId);
 
 			} else if (message.type === 'processHook') {
 				this.processHookMessage(workflowHooks, message.data as IProcessMessageDataHook);
+			} else if (message.type === 'timeout') {
+				// Execution timed out and its process has been terminated
+				const timeoutError = { message: 'Workflow execution timed out!' } as IExecutionError;
+
+				this.processError(timeoutError, startedAt, data.executionMode, executionId);
 			}
 		});
 
-		// Also get informed when the processes does exit especially when it did crash
+		// Also get informed when the processes does exit especially when it did crash or timed out
 		subprocess.on('exit', (code, signal) => {
-			if (code !== 0) {
+			if (signal === 'SIGTERM'){
+				// Execution timed out and its process has been terminated
+				const timeoutError = {
+					message: 'Workflow execution timed out!',
+				} as IExecutionError;
+
+				this.processError(timeoutError, startedAt, data.executionMode, executionId);
+			} else if (code !== 0) {
 				// Process did exit with error code, so something went wrong.
 				const executionError = {
 					message: 'Workflow execution process did crash for an unknown reason!',
@@ -243,6 +290,7 @@ export class WorkflowRunner {
 
 				this.processError(executionError, startedAt, data.executionMode, executionId);
 			}
+			clearTimeout(executionTimeout);
 		});
 
 		return executionId;

packages/cli/src/WorkflowRunnerProcess.ts

@@ -190,17 +190,18 @@ process.on('message', async (message: IProcessMessage) => {
 
 			// Once the workflow got executed make sure the process gets killed again
 			process.exit();
-		} else if (message.type === 'stopExecution') {
+		} else if (message.type === 'stopExecution' || message.type === 'timeout') {
 			// The workflow execution should be stopped
 			let runData: IRun;
 
 			if (workflowRunner.workflowExecute !== undefined) {
 				// Workflow started already executing
-
 				runData = workflowRunner.workflowExecute.getFullRunData(workflowRunner.startedAt);
 
-				// If there is any data send it to parent process
-				await workflowRunner.workflowExecute.processSuccessExecution(workflowRunner.startedAt, workflowRunner.workflow!);
+				const timeOutError = message.type === 'timeout' ? { message: 'Workflow execution timed out!' } as IExecutionError : undefined
+
+				// If there is any data send it to parent process, if execution timedout add the error
+				await workflowRunner.workflowExecute.processSuccessExecution(workflowRunner.startedAt, workflowRunner.workflow!, timeOutError);
 			} else {
 				// Workflow did not get started yet
 				runData = {
@@ -209,7 +210,7 @@ process.on('message', async (message: IProcessMessage) => {
 							runData: {},
 						},
 					},
-					finished: true,
+					finished: message.type !== 'timeout',
 					mode: workflowRunner.data!.executionMode,
 					startedAt: workflowRunner.startedAt,
 					stoppedAt: new Date(),
@@ -218,7 +219,7 @@ process.on('message', async (message: IProcessMessage) => {
 				workflowRunner.sendHookToParentProcess('workflowExecuteAfter', [runData]);
 			}
 
-			await sendToParentProcess('end', {
+			await sendToParentProcess(message.type === 'timeout' ? message.type : 'end', {
 				runData,
 			});
 

packages/core/src/WorkflowExecute.ts

@@ -708,6 +708,9 @@ export class WorkflowExecute {
 				return Promise.resolve();
 			})()
 			.then(async () => {
+				if (gotCancel && executionError === undefined) {
+					return this.processSuccessExecution(startedAt, workflow, { message: 'Workflow has been canceled!' } as IExecutionError);
+				}
 				return this.processSuccessExecution(startedAt, workflow, executionError);
 			})
 			.catch(async (error) => {

packages/editor-ui/src/Interface.ts

@@ -397,6 +397,8 @@ export interface IN8nUISettings {
 	saveDataSuccessExecution: string;
 	saveManualExecutions: boolean;
 	timezone: string;
+	executionTimeout: number;
+	maxExecutionTimeout: number;
 	urlBaseWebhook: string;
 	versionCli: string;
 }
@@ -407,4 +409,11 @@ export interface IWorkflowSettings extends IWorkflowSettingsWorkflow {
 	saveDataSuccessExecution?: string;
 	saveManualExecutions?: boolean;
 	timezone?: string;
+	executionTimeout?: number;
+}
+
+export interface ITimeoutHMS {
+	hours: number;
+	minutes: number;
+	seconds: number;
 }

packages/editor-ui/src/components/WorkflowSettings.vue

@@ -1,6 +1,6 @@
 <template>
 	<span>
-		<el-dialog class="workflow-settings" :visible="dialogVisible" append-to-body width="50%" title="Workflow Settings" :before-close="closeDialog">
+		<el-dialog class="workflow-settings" :visible="dialogVisible" append-to-body width="65%" title="Workflow Settings" :before-close="closeDialog">
 			<div v-loading="isLoading">
 				<el-row>
 					<el-col :span="10" class="setting-name">
@@ -97,6 +97,44 @@
 						</el-select>
 					</el-col>
 				</el-row>
+				<el-row>
+					<el-col :span="10" class="setting-name">
+						Timeout Workflow:
+						<el-tooltip class="setting-info" placement="top" effect="light">
+							<div slot="content" v-html="helpTexts.executionTimeoutToggle"></div>
+							<font-awesome-icon icon="question-circle" />
+						</el-tooltip>
+					</el-col>
+					<el-col :span="14">
+						<div>
+							<el-switch ref="inputField" :value="workflowSettings.executionTimeout > -1" @change="toggleTimeout" active-color="#13ce66"></el-switch>
+							<div class="expression-info clickable" @click="expressionEditDialogVisible = true">Edit Expression</div>
+						</div>
+					</el-col>
+				</el-row>
+				<div v-if="workflowSettings.executionTimeout > -1">
+					<el-row>
+						<el-col :span="10" class="setting-name">
+							Timeout After:
+							<el-tooltip class="setting-info" placement="top" effect="light">
+								<div slot="content" v-html="helpTexts.executionTimeout"></div>
+								<font-awesome-icon icon="question-circle" />
+							</el-tooltip>
+						</el-col>
+						<el-col :span="4">
+							<el-input-number size="small" v-model="timeoutHMS.hours" :min="0" placeholder="hours" type="number" class="el-input_inner"></el-input-number></br>
+							<div class="timeout-setting-name">hours</div>
+						</el-col>
+						<el-col :span="4">
+							<el-input-number size="small" v-model="timeoutHMS.minutes" :min="0" placeholder="minutes" type="number" class="el-input_inner"></el-input-number></br>
+							<div class="timeout-setting-name">minutes</div>
+						</el-col>
+						<el-col :span="4">
+							<el-input-number size="small" v-model="timeoutHMS.seconds" :min="0" placeholder="seconds" type="number" class="el-input_inner"></el-input-number></br>
+							<div class="timeout-setting-name">seconds</div>
+						</el-col>
+					</el-row>
+				</div>
 
 				<div class="action-buttons">
 					<el-button type="success" @click="saveSettings">
@@ -115,6 +153,7 @@ import { restApi } from '@/components/mixins/restApi';
 import { genericHelpers } from '@/components/mixins/genericHelpers';
 import { showMessage } from '@/components/mixins/showMessage';
 import {
+	ITimeoutHMS,
 	IWorkflowDataUpdate,
 	IWorkflowSettings,
 	IWorkflowShortResponse,
@@ -140,6 +179,8 @@ export default mixins(
 				saveDataErrorExecution: 'If data data of executions should be saved in case they failed.',
 				saveDataSuccessExecution: 'If data data of executions should be saved in case they succeed.',
 				saveManualExecutions: 'If data data of executions should be saved when started manually from the editor.',
+				executionTimeoutToggle: 'Cancel workflow execution after defined time',
+				executionTimeout: 'After what time the workflow should timeout.',
 			},
 			defaultValues: {
 				timezone: 'America/New_York',
@@ -153,6 +194,9 @@ export default mixins(
 			timezones: [] as Array<{ key: string, value: string }>,
 			workflowSettings: {} as IWorkflowSettings,
 			workflows: [] as IWorkflowShortResponse[],
+			executionTimeout: this.$store.getters.executionTimeout,
+			maxExecutionTimeout: this.$store.getters.maxExecutionTimeout,
+			timeoutHMS: { hours: 0, minutes: 0, seconds: 0 } as ITimeoutHMS,
 		};
 	},
 	watch: {
@@ -313,8 +357,15 @@ export default mixins(
 			if (workflowSettings.saveManualExecutions === undefined) {
 				workflowSettings.saveManualExecutions = 'DEFAULT';
 			}
+			if (workflowSettings.executionTimeout === undefined) {
+				workflowSettings.executionTimeout = this.$store.getters.executionTimeout;
+			}
+			if (workflowSettings.maxExecutionTimeout === undefined) {
+				workflowSettings.maxExecutionTimeout = this.$store.getters.maxExecutionTimeout;
+			}
 
 			Vue.set(this, 'workflowSettings', workflowSettings);
+			this.timeoutHMS = this.convertToHMS(workflowSettings.executionTimeout);
 			this.isLoading = false;
 		},
 		async saveSettings () {
@@ -323,6 +374,26 @@ export default mixins(
 				settings: this.workflowSettings,
 			};
 
+			// Convert hours, minutes, seconds into seconds for the workflow timeout
+			const { hours, minutes, seconds } = this.timeoutHMS;
+			data.settings!.executionTimeout =
+				data.settings!.executionTimeout !== -1
+					? hours * 3600 + minutes * 60 + seconds
+					: -1;
+
+			if (data.settings!.executionTimeout === 0) {
+				this.$showError(new Error('timeout is activated but set to 0'), 'Problem saving settings', 'There was a problem saving the settings:');
+				return;
+			}
+
+			// @ts-ignore
+			if (data.settings!.executionTimeout > this.workflowSettings.maxExecutionTimeout) {
+				const { hours, minutes, seconds } = this.convertToHMS(this.workflowSettings.maxExecutionTimeout as number);
+				this.$showError(new Error(`Maximum Timeout is: ${hours} hours, ${minutes} minutes, ${seconds} seconds`), 'Problem saving settings', 'Set timeout is exceeding the maximum timeout!');
+				return;
+			}
+			delete data.settings!.maxExecutionTimeout;
+
 			this.isLoading = true;
 
 			try {
@@ -353,6 +424,20 @@ export default mixins(
 
 			this.closeDialog();
 		},
+		toggleTimeout() {
+			this.workflowSettings.executionTimeout = this.workflowSettings.executionTimeout === -1 ? 0 : -1;
+		},
+		convertToHMS(num: number): ITimeoutHMS {
+			if (num > 0) {
+				let remainder: number;
+				const hours = Math.floor(num / 3600);
+				remainder = num % 3600;
+				const minutes = Math.floor(remainder / 60);
+				const seconds = remainder % 60;
+				return { hours, minutes, seconds };
+			}
+			return { hours: 0, minutes: 0, seconds: 0 };
+		},
 	},
 });
 </script>
@@ -383,4 +468,9 @@ export default mixins(
 	}
 }
 
+.timeout-setting-name {
+	text-align: center;
+	width: calc(100% - 20px);
+}
+
 </style>

packages/editor-ui/src/store.ts

@@ -52,6 +52,8 @@ export const store = new Vuex.Store({
 		saveDataSuccessExecution: 'all',
 		saveManualExecutions: false,
 		timezone: 'America/New_York',
+		executionTimeout: -1,
+		maxExecutionTimeout: Number.MAX_SAFE_INTEGER,
 		versionCli: '0.0.0',
 		workflowExecutionData: null as IExecutionResponse | null,
 		lastSelectedNode: null as string | null,
@@ -479,6 +481,12 @@ export const store = new Vuex.Store({
 		setTimezone (state, timezone: string) {
 			Vue.set(state, 'timezone', timezone);
 		},
+		setExecutionTimeout (state, executionTimeout: number) {
+			Vue.set(state, 'executionTimeout', executionTimeout);
+		},
+		setMaxExecutionTimeout (state, maxExecutionTimeout: number) {
+			Vue.set(state, 'maxExecutionTimeout', maxExecutionTimeout);
+		},
 		setVersionCli (state, version: string) {
 			Vue.set(state, 'versionCli', version);
 		},
@@ -592,6 +600,12 @@ export const store = new Vuex.Store({
 		timezone: (state): string => {
 			return state.timezone;
 		},
+		executionTimeout: (state): number => {
+			return state.executionTimeout;
+		},
+		maxExecutionTimeout: (state): number => {
+			return state.maxExecutionTimeout;
+		},
 		versionCli: (state): string => {
 			return state.versionCli;
 		},

packages/editor-ui/src/views/NodeView.vue

@@ -1865,6 +1865,8 @@ export default mixins(
 				this.$store.commit('setSaveDataSuccessExecution', settings.saveDataSuccessExecution);
 				this.$store.commit('setSaveManualExecutions', settings.saveManualExecutions);
 				this.$store.commit('setTimezone', settings.timezone);
+				this.$store.commit('setExecutionTimeout', settings.executionTimeout);
+				this.$store.commit('setMaxExecutionTimeout', settings.maxExecutionTimeout);
 				this.$store.commit('setVersionCli', settings.versionCli);
 			},
 			async loadNodeTypes (): Promise<void> {