mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-03-05 20:50:17 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(core): Fix execution pruning queries (#5562)

Browse source 
* fix(core): Execution pruning should delete query should use the `OR` operator

* fix(core): Prune executions in a chunk to avoid sqlite error "Expression tree is too large"

* reduce the memory usage during execution pruning
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™ 2023-02-24 18:02:34 +01:00 committed by कारतोफ्फेलस्क्रिप्ट™
parent 9735188195
commit 2137ae23d7
1 changed files with 19 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
packages/cli/src/WorkflowExecuteAdditionalData.ts
View file

@ -44,7 +44,7 @@ import {
import pick from 'lodash.pick'; import pick from 'lodash.pick';
import type { FindOptionsWhere } from 'typeorm'; import type { FindOptionsWhere } from 'typeorm';
import { LessThanOrEqual } from 'typeorm'; import { LessThanOrEqual, In } from 'typeorm';
import { DateUtils } from 'typeorm/util/DateUtils'; import { DateUtils } from 'typeorm/util/DateUtils';
import config from '@/config'; import config from '@/config';
import * as Db from '@/Db'; import * as Db from '@/Db';
@ -212,7 +212,9 @@ async function pruneExecutionData(this: WorkflowHooks): Promise<void> {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const utcDate = DateUtils.mixedDateToUtcDatetimeString(date); const utcDate = DateUtils.mixedDateToUtcDatetimeString(date);
const toPrune: FindOptionsWhere<IExecutionFlattedDb> = { stoppedAt: LessThanOrEqual(utcDate) }; const toPrune: Array<FindOptionsWhere<IExecutionFlattedDb>> = [
{ stoppedAt: LessThanOrEqual(utcDate) },
];
if (maxCount > 0) { if (maxCount > 0) {
const executions = await Db.collections.Execution.find({ const executions = await Db.collections.Execution.find({
@ -223,27 +225,29 @@ async function pruneExecutionData(this: WorkflowHooks): Promise<void> {
}); });
if (executions[0]) { if (executions[0]) {
toPrune.id = LessThanOrEqual(executions[0].id); toPrune.push({ id: LessThanOrEqual(executions[0].id) });
} }
} }
const isBinaryModeDefaultMode = config.getEnv('binaryDataManager.mode') === 'default'; const isBinaryModeDefaultMode = config.getEnv('binaryDataManager.mode') === 'default';
try { try {
const executions = isBinaryModeDefaultMode
? []
: await Db.collections.Execution.find({
select: ['id'],
where: toPrune,
});
await Db.collections.Execution.delete(toPrune);
setTimeout(() => { setTimeout(() => {
throttling = false; throttling = false;
}, timeout * 1000); }, timeout * 1000);
let executionIds: Array<IExecutionFlattedDb['id']>;
do {
executionIds = (
await Db.collections.Execution.find({
select: ['id'],
where: toPrune,
take: 100,
})
).map(({ id }) => id);
await Db.collections.Execution.delete({ id: In(executionIds) });
// Mark binary data for deletion for all executions // Mark binary data for deletion for all executions
if (!isBinaryModeDefaultMode) if (!isBinaryModeDefaultMode)
await BinaryDataManager.getInstance().markDataForDeletionByExecutionIds( await BinaryDataManager.getInstance().markDataForDeletionByExecutionIds(executionIds);
executions.map(({ id }) => id), } while (executionIds.length > 0);
);
} catch (error) { } catch (error) {
ErrorReporter.error(error); ErrorReporter.error(error);
throttling = false; throttling = false;