mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2024-12-25 04:34:06 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

🐎 pruning execution data complete

Browse source
This commit is contained in:
Ben Hesseldieck 2020-07-09 16:52:38 +02:00
parent 05507fc19b
commit b956444c0e
2 changed files with 44 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
packages/cli/config/index.ts
View file

@ -161,8 +161,8 @@ const config = convict({
// If a workflow executes all the data gets saved by default. This
// could be a problem when a workflow gets executed a lot and processes
// a lot of data. To not write the database full it is possible to
// not save the execution at all.
// a lot of data. To not exceed the database's capacity it is possible to
// prune the database regularly or to not save the execution at all.
// Depending on if the execution did succeed or error a different
// save behaviour can be set.
saveDataOnError: {
@ -188,6 +188,27 @@ const config = convict({
default: false,
env: 'EXECUTIONS_DATA_SAVE_MANUAL_EXECUTIONS'
},
// To not exceed the database's capacity and keep its size moderate
// the execution data gets pruned regularly (default: 1 hour interval).
// All saved execution data older than the max age will be deleted.
// Pruning is currently not activated by default, which will change in
// a future version.
pruneData: {
doc: 'Delete data of past executions on a rolling basis',
default: false,
env: 'EXECUTIONS_DATA_PRUNE'
},
pruneDataMaxAge: {
doc: 'How old (hours) the execution data has to be to get deleted',
default: 336,
env: 'EXECUTIONS_DATA_MAX_AGE'
},
pruneDataTimeout: {
doc: 'Timeout (ms) after execution data has been pruned',
default: 3600000,
env: 'EXECUTIONS_DATA_PRUNE_TIMEOUT'
},
},
generic: {

33
packages/cli/src/WorkflowExecuteAdditionalData.ts
View file

@ -86,17 +86,22 @@ function executeErrorWorkflow(workflowData: IWorkflowBase, fullRunData: IRun, mo
* Throttled to be executed just once in configured timeframe.
*
*/
let inThrottle: boolean;
function pruneSavedExecutions(): void {
console.log('THROTTLE:', inThrottle);
if (!inThrottle) {
inThrottle = true;
Db.collections.Execution!.delete({ startedAt: LessThanOrEqual(new Date().toISOString()) });
console.log('Deleting logs');
setTimeout(() => {
console.log('resetting throttle');
inThrottle = false;
}, 30000);
let throttling: boolean;
function pruneExecutionData(): void {
if (!throttling) {
throttling = true;
const timeout = config.get('executions.pruneDataTimeout') as number; // in ms
const maxAge = config.get('executions.pruneDataMaxAge') as number; // in h
const date = new Date(); // today
date.setHours(date.getHours() - maxAge);
// throttle just on success to allow for self healing on failure
Db.collections.Execution!.delete({ startedAt: LessThanOrEqual(date.toISOString()) })
.then(data =>
setTimeout(() => {
throttling = false;
}, timeout)
).catch(err => throttling = false)
}
}
@ -272,7 +277,6 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks {
// Save the Execution in DB
const executionResult = await Db.collections.Execution!.save(executionData as IExecutionFlattedDb);
pruneSavedExecutions()
if (fullRunData.finished === true && this.retryOf !== undefined) {
// If the retry was successful save the reference it on the original execution
@ -280,6 +284,11 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks {
await Db.collections.Execution!.update(this.retryOf, { retrySuccessId: executionResult.id });
}
// Prune old execution data
if (config.get('executions.pruneData')) {
pruneExecutionData()
}
if (!isManualMode) {
executeErrorWorkflow(this.workflowData, fullRunData, this.mode, executionResult ? executionResult.id as string : undefined, this.retryOf);
}