From b956444c0e41c24c426daaac91e597093c7ef129 Mon Sep 17 00:00:00 2001
From: Ben Hesseldieck <b.hesseldieck@gmail.com>
Date: Thu, 9 Jul 2020 16:52:38 +0200
Subject: [PATCH] :racehorse: pruning execution data complete

---
 packages/cli/config/index.ts                  | 25 ++++++++++++--
 .../cli/src/WorkflowExecuteAdditionalData.ts  | 33 ++++++++++++-------
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/packages/cli/config/index.ts b/packages/cli/config/index.ts
index 847587460f..0022c08b7a 100644
--- a/packages/cli/config/index.ts
+++ b/packages/cli/config/index.ts
@@ -161,8 +161,8 @@ const config = convict({
 
 		// If a workflow executes all the data gets saved by default. This
 		// could be a problem when a workflow gets executed a lot and processes
-		// a lot of data. To not write the database full it is possible to
-		// not save the execution at all.
+		// a lot of data. To not exceed the database's capacity it is possible to
+		// prune the database regularly or to not save the execution at all.
 		// Depending on if the execution did succeed or error a different
 		// save behaviour can be set.
 		saveDataOnError: {
@@ -188,6 +188,27 @@ const config = convict({
 			default: false,
 			env: 'EXECUTIONS_DATA_SAVE_MANUAL_EXECUTIONS'
 		},
+
+		// To not exceed the database's capacity and keep its size moderate
+		// the execution data gets pruned regularly (default: 1 hour interval).
+		// All saved execution data older than the max age will be deleted.
+		// Pruning is currently not activated by default, which will change in
+		// a future version.
+		pruneData: {
+			doc: 'Delete data of past executions on a rolling basis',
+			default: false,
+			env: 'EXECUTIONS_DATA_PRUNE'
+		},
+		pruneDataMaxAge: {
+			doc: 'How old (hours) the execution data has to be to get deleted',
+			default: 336,
+			env: 'EXECUTIONS_DATA_MAX_AGE'
+		},
+		pruneDataTimeout: {
+			doc: 'Timeout (ms) after execution data has been pruned',
+			default: 3600000,
+			env: 'EXECUTIONS_DATA_PRUNE_TIMEOUT'
+		},
 	},
 
 	generic: {
diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts
index 86e5c3fe22..9b55f88b58 100644
--- a/packages/cli/src/WorkflowExecuteAdditionalData.ts
+++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts
@@ -86,17 +86,22 @@ function executeErrorWorkflow(workflowData: IWorkflowBase, fullRunData: IRun, mo
  * Throttled to be executed just once in configured timeframe.
  *
  */
-let inThrottle: boolean;
-function pruneSavedExecutions(): void {
-	console.log('THROTTLE:', inThrottle);
-	if (!inThrottle) {
-		inThrottle = true;
-		Db.collections.Execution!.delete({ startedAt: LessThanOrEqual(new Date().toISOString()) });
-		console.log('Deleting logs');
-		setTimeout(() => {
-			console.log('resetting throttle');
-			inThrottle = false;
-		}, 30000);
+let throttling: boolean;
+function pruneExecutionData(): void {
+	if (!throttling) {
+		throttling = true;
+		const timeout = config.get('executions.pruneDataTimeout') as number; // in ms
+		const maxAge = config.get('executions.pruneDataMaxAge') as number; // in h
+		const date = new Date(); // today
+		date.setHours(date.getHours() - maxAge);
+
+		// throttle just on success to allow for self healing on failure
+		Db.collections.Execution!.delete({ startedAt: LessThanOrEqual(date.toISOString()) })
+		.then(data =>
+			setTimeout(() => {
+				throttling = false;
+			}, timeout)
+		).catch(err => throttling = false)
 	}
 }
 
@@ -272,7 +277,6 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks {
 
 					// Save the Execution in DB
 					const executionResult = await Db.collections.Execution!.save(executionData as IExecutionFlattedDb);
-					pruneSavedExecutions()
 
 					if (fullRunData.finished === true && this.retryOf !== undefined) {
 						// If the retry was successful save the reference it on the original execution
@@ -280,6 +284,11 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks {
 						await Db.collections.Execution!.update(this.retryOf, { retrySuccessId: executionResult.id });
 					}
 
+					// Prune old execution data
+					if (config.get('executions.pruneData')) {
+						pruneExecutionData()
+					}
+
 					if (!isManualMode) {
 						executeErrorWorkflow(this.workflowData, fullRunData, this.mode, executionResult ? executionResult.id as string : undefined, this.retryOf);
 					}