🔀 Merge branch 'WorkflowExecutionMaintainance'

packages/cli/config/index.ts

@@ -161,8 +161,8 @@ const config = convict({
 
 		// If a workflow executes all the data gets saved by default. This
 		// could be a problem when a workflow gets executed a lot and processes
-		// a lot of data. To not write the database full it is possible to
+		// a lot of data. To not exceed the database's capacity it is possible to
-		// not save the execution at all.
+		// prune the database regularly or to not save the execution at all.
 		// Depending on if the execution did succeed or error a different
 		// save behaviour can be set.
 		saveDataOnError: {
@@ -188,6 +188,27 @@ const config = convict({
 			default: false,
 			env: 'EXECUTIONS_DATA_SAVE_MANUAL_EXECUTIONS'
 		},
+
+		// To not exceed the database's capacity and keep its size moderate
+		// the execution data gets pruned regularly (default: 1 hour interval).
+		// All saved execution data older than the max age will be deleted.
+		// Pruning is currently not activated by default, which will change in
+		// a future version.
+		pruneData: {
+			doc: 'Delete data of past executions on a rolling basis',
+			default: false,
+			env: 'EXECUTIONS_DATA_PRUNE'
+		},
+		pruneDataMaxAge: {
+			doc: 'How old (hours) the execution data has to be to get deleted',
+			default: 336,
+			env: 'EXECUTIONS_DATA_MAX_AGE'
+		},
+		pruneDataTimeout: {
+			doc: 'Timeout (seconds) after execution data has been pruned',
+			default: 3600,
+			env: 'EXECUTIONS_DATA_PRUNE_TIMEOUT'
+		},
 	},
 
 	generic: {

packages/cli/src/WorkflowExecuteAdditionalData.ts

@@ -41,6 +41,8 @@ import {
 
 import * as config from '../config';
 
+import { LessThanOrEqual } from "typeorm";
+
 
 /**
  * Checks if there was an error and if errorWorkflow is defined. If so it collects
@@ -79,6 +81,30 @@ function executeErrorWorkflow(workflowData: IWorkflowBase, fullRunData: IRun, mo
 	}
 }
 
+/**
+ * Prunes Saved Execution which are older than configured.
+ * Throttled to be executed just once in configured timeframe.
+ *
+ */
+let throttling = false;
+function pruneExecutionData(): void {
+	if (!throttling) {
+		throttling = true;
+		const timeout = config.get('executions.pruneDataTimeout') as number; // in seconds
+		const maxAge = config.get('executions.pruneDataMaxAge') as number; // in h
+		const date = new Date(); // today
+		date.setHours(date.getHours() - maxAge);
+
+		// throttle just on success to allow for self healing on failure
+		Db.collections.Execution!.delete({ stoppedAt: LessThanOrEqual(date.toISOString()) })
+		.then(data =>
+			setTimeout(() => {
+				throttling = false;
+			}, timeout * 1000)
+		).catch(err => throttling = false);
+	}
+}
+
 
 /**
  * Pushes the execution out to all connected clients
@@ -189,6 +215,11 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks {
 		workflowExecuteAfter: [
 			async function (this: WorkflowHooks, fullRunData: IRun, newStaticData: IDataObject): Promise<void> {
 
+				// Prune old execution data
+				if (config.get('executions.pruneData')) {
+					pruneExecutionData();
+				}
+
 				const isManualMode = [this.mode, parentProcessMode].includes('manual');
 
 				try {