mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-03-05 20:50:17 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

🐎 pruning execution data complete

Browse source
This commit is contained in:
Ben Hesseldieck 2020-07-09 16:52:38 +02:00
parent 05507fc19b
commit b956444c0e
2 changed files with 44 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
packages/cli/config/index.ts
View file

@ -161,8 +161,8 @@ const config = convict({
// If a workflow executes all the data gets saved by default. This // If a workflow executes all the data gets saved by default. This
// could be a problem when a workflow gets executed a lot and processes // could be a problem when a workflow gets executed a lot and processes
// a lot of data. To not write the database full it is possible to // a lot of data. To not exceed the database's capacity it is possible to
// not save the execution at all. // prune the database regularly or to not save the execution at all.
// Depending on if the execution did succeed or error a different // Depending on if the execution did succeed or error a different
// save behaviour can be set. // save behaviour can be set.
saveDataOnError: { saveDataOnError: {
@ -188,6 +188,27 @@ const config = convict({
default: false, default: false,
env: 'EXECUTIONS_DATA_SAVE_MANUAL_EXECUTIONS' env: 'EXECUTIONS_DATA_SAVE_MANUAL_EXECUTIONS'
}, },
// To not exceed the database's capacity and keep its size moderate
// the execution data gets pruned regularly (default: 1 hour interval).
// All saved execution data older than the max age will be deleted.
// Pruning is currently not activated by default, which will change in
// a future version.
pruneData: {
doc: 'Delete data of past executions on a rolling basis',
default: false,
env: 'EXECUTIONS_DATA_PRUNE'
},
pruneDataMaxAge: {
doc: 'How old (hours) the execution data has to be to get deleted',
default: 336,
env: 'EXECUTIONS_DATA_MAX_AGE'
},
pruneDataTimeout: {
doc: 'Timeout (ms) after execution data has been pruned',
default: 3600000,
env: 'EXECUTIONS_DATA_PRUNE_TIMEOUT'
},
}, },
generic: { generic: {

33
packages/cli/src/WorkflowExecuteAdditionalData.ts
View file

@ -86,17 +86,22 @@ function executeErrorWorkflow(workflowData: IWorkflowBase, fullRunData: IRun, mo
* Throttled to be executed just once in configured timeframe. * Throttled to be executed just once in configured timeframe.
* *
*/ */
let inThrottle: boolean; let throttling: boolean;
function pruneSavedExecutions(): void { function pruneExecutionData(): void {
console.log('THROTTLE:', inThrottle); if (!throttling) {
if (!inThrottle) { throttling = true;
inThrottle = true; const timeout = config.get('executions.pruneDataTimeout') as number; // in ms
Db.collections.Execution!.delete({ startedAt: LessThanOrEqual(new Date().toISOString()) }); const maxAge = config.get('executions.pruneDataMaxAge') as number; // in h
console.log('Deleting logs'); const date = new Date(); // today
setTimeout(() => { date.setHours(date.getHours() - maxAge);
console.log('resetting throttle');
inThrottle = false; // throttle just on success to allow for self healing on failure
}, 30000); Db.collections.Execution!.delete({ startedAt: LessThanOrEqual(date.toISOString()) })
.then(data =>
setTimeout(() => {
throttling = false;
}, timeout)
).catch(err => throttling = false)
} }
} }
@ -272,7 +277,6 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks {
// Save the Execution in DB // Save the Execution in DB
const executionResult = await Db.collections.Execution!.save(executionData as IExecutionFlattedDb); const executionResult = await Db.collections.Execution!.save(executionData as IExecutionFlattedDb);
pruneSavedExecutions()
if (fullRunData.finished === true && this.retryOf !== undefined) { if (fullRunData.finished === true && this.retryOf !== undefined) {
// If the retry was successful save the reference it on the original execution // If the retry was successful save the reference it on the original execution
@ -280,6 +284,11 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks {
await Db.collections.Execution!.update(this.retryOf, { retrySuccessId: executionResult.id }); await Db.collections.Execution!.update(this.retryOf, { retrySuccessId: executionResult.id });
} }
// Prune old execution data
if (config.get('executions.pruneData')) {
pruneExecutionData()
}
if (!isManualMode) { if (!isManualMode) {
executeErrorWorkflow(this.workflowData, fullRunData, this.mode, executionResult ? executionResult.id as string : undefined, this.retryOf); executeErrorWorkflow(this.workflowData, fullRunData, this.mode, executionResult ? executionResult.id as string : undefined, this.retryOf);
} }