From ce963e88240a3ec92121577660a2c4e38e7cc85a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Mon, 4 Nov 2024 09:09:12 +0100 Subject: [PATCH] refactor(core): Port `pruning` config (#11507) --- .../@n8n/config/src/configs/pruning.config.ts | 35 ++++++++++++++ packages/@n8n/config/src/index.ts | 5 ++ packages/@n8n/config/test/config.test.ts | 8 ++++ packages/cli/src/config/schema.ts | 48 ------------------- .../repositories/execution.repository.ts | 6 +-- .../events/relays/telemetry.event-relay.ts | 4 +- packages/cli/src/services/frontend.service.ts | 6 +-- packages/cli/src/services/pruning.service.ts | 7 ++- .../test/integration/pruning.service.test.ts | 22 ++++----- 9 files changed, 68 insertions(+), 73 deletions(-) create mode 100644 packages/@n8n/config/src/configs/pruning.config.ts diff --git a/packages/@n8n/config/src/configs/pruning.config.ts b/packages/@n8n/config/src/configs/pruning.config.ts new file mode 100644 index 0000000000..109dffc462 --- /dev/null +++ b/packages/@n8n/config/src/configs/pruning.config.ts @@ -0,0 +1,35 @@ +import { Config, Env } from '../decorators'; + +@Config +export class PruningConfig { + /** Whether to delete past executions on a rolling basis. */ + @Env('EXECUTIONS_DATA_PRUNE') + isEnabled: boolean = true; + + /** How old (hours) a finished execution must be to qualify for soft-deletion. */ + @Env('EXECUTIONS_DATA_MAX_AGE') + maxAge: number = 336; + + /** + * Max number of finished executions to keep in database. Does not necessarily + * prune to the exact max number. `0` for unlimited. + */ + @Env('EXECUTIONS_DATA_PRUNE_MAX_COUNT') + maxCount: number = 10_000; + + /** + * How old (hours) a finished execution must be to qualify for hard-deletion. + * This buffer by default excludes recent executions as the user may need + * them while building a workflow. + */ + @Env('EXECUTIONS_DATA_HARD_DELETE_BUFFER') + hardDeleteBuffer: number = 1; + + /** How often (minutes) execution data should be hard-deleted. */ + @Env('EXECUTIONS_DATA_PRUNE_HARD_DELETE_INTERVAL') + hardDeleteInterval: number = 15; + + /** How often (minutes) execution data should be soft-deleted */ + @Env('EXECUTIONS_DATA_PRUNE_SOFT_DELETE_INTERVAL') + softDeleteInterval: number = 60; +} diff --git a/packages/@n8n/config/src/index.ts b/packages/@n8n/config/src/index.ts index c056a1090c..0a89535ee3 100644 --- a/packages/@n8n/config/src/index.ts +++ b/packages/@n8n/config/src/index.ts @@ -10,6 +10,7 @@ import { LicenseConfig } from './configs/license.config'; import { LoggingConfig } from './configs/logging.config'; import { MultiMainSetupConfig } from './configs/multi-main-setup.config'; import { NodesConfig } from './configs/nodes.config'; +import { PruningConfig } from './configs/pruning.config'; import { PublicApiConfig } from './configs/public-api.config'; import { TaskRunnersConfig } from './configs/runners.config'; import { ScalingModeConfig } from './configs/scaling-mode.config'; @@ -24,6 +25,7 @@ import { Config, Env, Nested } from './decorators'; export { Config, Env, Nested } from './decorators'; export { TaskRunnersConfig } from './configs/runners.config'; export { SecurityConfig } from './configs/security.config'; +export { PruningConfig } from './configs/pruning.config'; export { FrontendBetaFeatures, FrontendConfig } from './configs/frontend.config'; export { LOG_SCOPES } from './configs/logging.config'; export type { LogScope } from './configs/logging.config'; @@ -112,4 +114,7 @@ export class GlobalConfig { @Nested security: SecurityConfig; + + @Nested + pruning: PruningConfig; } diff --git a/packages/@n8n/config/test/config.test.ts b/packages/@n8n/config/test/config.test.ts index 07af2c0a0b..bc10028f36 100644 --- a/packages/@n8n/config/test/config.test.ts +++ b/packages/@n8n/config/test/config.test.ts @@ -271,6 +271,14 @@ describe('GlobalConfig', () => { blockFileAccessToN8nFiles: true, daysAbandonedWorkflow: 90, }, + pruning: { + isEnabled: true, + maxAge: 336, + maxCount: 10_000, + hardDeleteBuffer: 1, + hardDeleteInterval: 15, + softDeleteInterval: 60, + }, }; it('should use all default values when no env variables are defined', () => { diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index 8bece9199a..9f8bc45232 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -98,54 +98,6 @@ export const schema = { env: 'EXECUTIONS_DATA_SAVE_MANUAL_EXECUTIONS', }, - // To not exceed the database's capacity and keep its size moderate - // the execution data gets pruned regularly (default: 15 minute interval). - // All saved execution data older than the max age will be deleted. - // Pruning is currently not activated by default, which will change in - // a future version. - pruneData: { - doc: 'Delete data of past executions on a rolling basis', - format: Boolean, - default: true, - env: 'EXECUTIONS_DATA_PRUNE', - }, - pruneDataMaxAge: { - doc: 'How old (hours) the finished execution data has to be to get soft-deleted', - format: Number, - default: 336, - env: 'EXECUTIONS_DATA_MAX_AGE', - }, - pruneDataHardDeleteBuffer: { - doc: 'How old (hours) the finished execution data has to be to get hard-deleted. By default, this buffer excludes recent executions as the user may need them while building a workflow.', - format: Number, - default: 1, - env: 'EXECUTIONS_DATA_HARD_DELETE_BUFFER', - }, - pruneDataIntervals: { - hardDelete: { - doc: 'How often (minutes) execution data should be hard-deleted', - format: Number, - default: 15, - env: 'EXECUTIONS_DATA_PRUNE_HARD_DELETE_INTERVAL', - }, - softDelete: { - doc: 'How often (minutes) execution data should be soft-deleted', - format: Number, - default: 60, - env: 'EXECUTIONS_DATA_PRUNE_SOFT_DELETE_INTERVAL', - }, - }, - - // Additional pruning option to delete executions if total count exceeds the configured max. - // Deletes the oldest entries first - // Set to 0 for No limit - pruneDataMaxCount: { - doc: "Maximum number of finished executions to keep in DB. Doesn't necessarily prune exactly to max number. 0 = no limit", - format: Number, - default: 10000, - env: 'EXECUTIONS_DATA_PRUNE_MAX_COUNT', - }, - queueRecovery: { interval: { doc: 'How often (minutes) to check for queue recovery', diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index ce4bedac2a..39f5e92cad 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -35,7 +35,6 @@ import type { } from 'n8n-workflow'; import { Service } from 'typedi'; -import config from '@/config'; import { AnnotationTagEntity } from '@/databases/entities/annotation-tag-entity.ee'; import { AnnotationTagMapping } from '@/databases/entities/annotation-tag-mapping.ee'; import { ExecutionAnnotation } from '@/databases/entities/execution-annotation.ee'; @@ -460,8 +459,7 @@ export class ExecutionRepository extends Repository { } async softDeletePrunableExecutions() { - const maxAge = config.getEnv('executions.pruneDataMaxAge'); // in h - const maxCount = config.getEnv('executions.pruneDataMaxCount'); + const { maxAge, maxCount } = this.globalConfig.pruning; // Sub-query to exclude executions having annotations const annotatedExecutionsSubQuery = this.manager @@ -517,7 +515,7 @@ export class ExecutionRepository extends Repository { async hardDeleteSoftDeletedExecutions() { const date = new Date(); - date.setHours(date.getHours() - config.getEnv('executions.pruneDataHardDeleteBuffer')); + date.setHours(date.getHours() - this.globalConfig.pruning.hardDeleteBuffer); const workflowIdsAndExecutionIds = ( await this.find({ diff --git a/packages/cli/src/events/relays/telemetry.event-relay.ts b/packages/cli/src/events/relays/telemetry.event-relay.ts index fc5cf0a53d..88f954ab93 100644 --- a/packages/cli/src/events/relays/telemetry.event-relay.ts +++ b/packages/cli/src/events/relays/telemetry.event-relay.ts @@ -771,8 +771,8 @@ export class TelemetryEventRelay extends EventRelay { executions_data_save_manual_executions: config.getEnv( 'executions.saveDataManualExecutions', ), - executions_data_prune: config.getEnv('executions.pruneData'), - executions_data_max_age: config.getEnv('executions.pruneDataMaxAge'), + executions_data_prune: this.globalConfig.pruning.isEnabled, + executions_data_max_age: this.globalConfig.pruning.maxAge, }, n8n_deployment_type: config.getEnv('deployment.type'), n8n_binary_data_mode: binaryDataConfig.mode, diff --git a/packages/cli/src/services/frontend.service.ts b/packages/cli/src/services/frontend.service.ts index 6cad4a4f24..7d9fe7d2b4 100644 --- a/packages/cli/src/services/frontend.service.ts +++ b/packages/cli/src/services/frontend.service.ts @@ -222,9 +222,9 @@ export class FrontendService { licensePruneTime: -1, }, pruning: { - isEnabled: config.getEnv('executions.pruneData'), - maxAge: config.getEnv('executions.pruneDataMaxAge'), - maxCount: config.getEnv('executions.pruneDataMaxCount'), + isEnabled: this.globalConfig.pruning.isEnabled, + maxAge: this.globalConfig.pruning.maxAge, + maxCount: this.globalConfig.pruning.maxCount, }, security: { blockFileAccessToN8nFiles: this.securityConfig.blockFileAccessToN8nFiles, diff --git a/packages/cli/src/services/pruning.service.ts b/packages/cli/src/services/pruning.service.ts index 0859dddd39..943f8d30ca 100644 --- a/packages/cli/src/services/pruning.service.ts +++ b/packages/cli/src/services/pruning.service.ts @@ -3,7 +3,6 @@ import { BinaryDataService, InstanceSettings } from 'n8n-core'; import { jsonStringify } from 'n8n-workflow'; import { Service } from 'typedi'; -import config from '@/config'; import { inTest, TIME } from '@/constants'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; import { OnShutdown } from '@/decorators/on-shutdown'; @@ -16,8 +15,8 @@ export class PruningService { private hardDeletionBatchSize = 100; private rates: Record = { - softDeletion: config.getEnv('executions.pruneDataIntervals.softDelete') * TIME.MINUTE, - hardDeletion: config.getEnv('executions.pruneDataIntervals.hardDelete') * TIME.MINUTE, + softDeletion: this.globalConfig.pruning.softDeleteInterval * TIME.MINUTE, + hardDeletion: this.globalConfig.pruning.hardDeleteInterval * TIME.MINUTE, }; public softDeletionInterval: NodeJS.Timer | undefined; @@ -52,7 +51,7 @@ export class PruningService { private isPruningEnabled() { const { instanceType, isFollower } = this.instanceSettings; - if (!config.getEnv('executions.pruneData') || inTest || instanceType !== 'main') { + if (!this.globalConfig.pruning.isEnabled || inTest || instanceType !== 'main') { return false; } diff --git a/packages/cli/test/integration/pruning.service.test.ts b/packages/cli/test/integration/pruning.service.test.ts index 5a53d70315..7ade1d3fe5 100644 --- a/packages/cli/test/integration/pruning.service.test.ts +++ b/packages/cli/test/integration/pruning.service.test.ts @@ -1,9 +1,9 @@ +import { GlobalConfig } from '@n8n/config'; import { mock } from 'jest-mock-extended'; import { BinaryDataService, InstanceSettings } from 'n8n-core'; import type { ExecutionStatus } from 'n8n-workflow'; import Container from 'typedi'; -import config from '@/config'; import { TIME } from '@/constants'; import type { ExecutionEntity } from '@/databases/entities/execution-entity'; import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; @@ -28,17 +28,19 @@ describe('softDeleteOnPruningCycle()', () => { const now = new Date(); const yesterday = new Date(Date.now() - TIME.DAY); let workflow: WorkflowEntity; + let globalConfig: GlobalConfig; beforeAll(async () => { await testDb.init(); + globalConfig = Container.get(GlobalConfig); pruningService = new PruningService( mockInstance(Logger), instanceSettings, Container.get(ExecutionRepository), mockInstance(BinaryDataService), mock(), - mock(), + globalConfig, ); workflow = await createWorkflow(); @@ -52,10 +54,6 @@ describe('softDeleteOnPruningCycle()', () => { await testDb.terminate(); }); - afterEach(() => { - config.load(config.default); - }); - async function findAllExecutions() { return await Container.get(ExecutionRepository).find({ order: { id: 'asc' }, @@ -64,9 +62,9 @@ describe('softDeleteOnPruningCycle()', () => { } describe('when EXECUTIONS_DATA_PRUNE_MAX_COUNT is set', () => { - beforeEach(() => { - config.set('executions.pruneDataMaxCount', 1); - config.set('executions.pruneDataMaxAge', 336); + beforeAll(() => { + globalConfig.pruning.maxAge = 336; + globalConfig.pruning.maxCount = 1; }); test('should mark as deleted based on EXECUTIONS_DATA_PRUNE_MAX_COUNT', async () => { @@ -165,9 +163,9 @@ describe('softDeleteOnPruningCycle()', () => { }); describe('when EXECUTIONS_DATA_MAX_AGE is set', () => { - beforeEach(() => { - config.set('executions.pruneDataMaxAge', 1); // 1h - config.set('executions.pruneDataMaxCount', 0); + beforeAll(() => { + globalConfig.pruning.maxAge = 1; + globalConfig.pruning.maxCount = 0; }); test('should mark as deleted based on EXECUTIONS_DATA_MAX_AGE', async () => {