fix(core): Restore old names for pruning config keys (#11782)

This commit is contained in:
Iván Ovejero 2024-11-20 09:56:37 +01:00 committed by GitHub
parent e1dacb4d57
commit d15b8d0509
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 61 additions and 53 deletions

View file

@ -1,21 +1,32 @@
import { Config, Env } from '../decorators';
import { Config, Env, Nested } from '../decorators';
@Config
export class PruningConfig {
class PruningIntervalsConfig {
/** How often (minutes) execution data should be hard-deleted. */
@Env('EXECUTIONS_DATA_PRUNE_HARD_DELETE_INTERVAL')
hardDelete: number = 15;
/** How often (minutes) execution data should be soft-deleted */
@Env('EXECUTIONS_DATA_PRUNE_SOFT_DELETE_INTERVAL')
softDelete: number = 60;
}
@Config
export class ExecutionsConfig {
/** Whether to delete past executions on a rolling basis. */
@Env('EXECUTIONS_DATA_PRUNE')
isEnabled: boolean = true;
pruneData: boolean = true;
/** How old (hours) a finished execution must be to qualify for soft-deletion. */
@Env('EXECUTIONS_DATA_MAX_AGE')
maxAge: number = 336;
pruneDataMaxAge: number = 336;
/**
* Max number of finished executions to keep in database. Does not necessarily
* prune to the exact max number. `0` for unlimited.
*/
@Env('EXECUTIONS_DATA_PRUNE_MAX_COUNT')
maxCount: number = 10_000;
pruneDataMaxCount: number = 10_000;
/**
* How old (hours) a finished execution must be to qualify for hard-deletion.
@ -23,13 +34,8 @@ export class PruningConfig {
* them while building a workflow.
*/
@Env('EXECUTIONS_DATA_HARD_DELETE_BUFFER')
hardDeleteBuffer: number = 1;
pruneDataHardDeleteBuffer: number = 1;
/** How often (minutes) execution data should be hard-deleted. */
@Env('EXECUTIONS_DATA_PRUNE_HARD_DELETE_INTERVAL')
hardDeleteInterval: number = 15;
/** How often (minutes) execution data should be soft-deleted */
@Env('EXECUTIONS_DATA_PRUNE_SOFT_DELETE_INTERVAL')
softDeleteInterval: number = 60;
@Nested
pruneDataIntervals: PruningIntervalsConfig;
}

View file

@ -4,6 +4,7 @@ import { DatabaseConfig } from './configs/database.config';
import { DiagnosticsConfig } from './configs/diagnostics.config';
import { EndpointsConfig } from './configs/endpoints.config';
import { EventBusConfig } from './configs/event-bus.config';
import { ExecutionsConfig } from './configs/executions.config';
import { ExternalSecretsConfig } from './configs/external-secrets.config';
import { ExternalStorageConfig } from './configs/external-storage.config';
import { GenericConfig } from './configs/generic.config';
@ -11,7 +12,6 @@ import { LicenseConfig } from './configs/license.config';
import { LoggingConfig } from './configs/logging.config';
import { MultiMainSetupConfig } from './configs/multi-main-setup.config';
import { NodesConfig } from './configs/nodes.config';
import { PruningConfig } from './configs/pruning.config';
import { PublicApiConfig } from './configs/public-api.config';
import { TaskRunnersConfig } from './configs/runners.config';
import { ScalingModeConfig } from './configs/scaling-mode.config';
@ -26,7 +26,7 @@ import { Config, Env, Nested } from './decorators';
export { Config, Env, Nested } from './decorators';
export { TaskRunnersConfig } from './configs/runners.config';
export { SecurityConfig } from './configs/security.config';
export { PruningConfig } from './configs/pruning.config';
export { ExecutionsConfig } from './configs/executions.config';
export { FrontendBetaFeatures, FrontendConfig } from './configs/frontend.config';
export { LOG_SCOPES } from './configs/logging.config';
export type { LogScope } from './configs/logging.config';
@ -117,7 +117,7 @@ export class GlobalConfig {
security: SecurityConfig;
@Nested
pruning: PruningConfig;
executions: ExecutionsConfig;
@Nested
diagnostics: DiagnosticsConfig;

View file

@ -272,13 +272,15 @@ describe('GlobalConfig', () => {
blockFileAccessToN8nFiles: true,
daysAbandonedWorkflow: 90,
},
pruning: {
isEnabled: true,
maxAge: 336,
maxCount: 10_000,
hardDeleteBuffer: 1,
hardDeleteInterval: 15,
softDeleteInterval: 60,
executions: {
pruneData: true,
pruneDataMaxAge: 336,
pruneDataMaxCount: 10_000,
pruneDataHardDeleteBuffer: 1,
pruneDataIntervals: {
hardDelete: 15,
softDelete: 60,
},
},
diagnostics: {
enabled: false,

View file

@ -459,7 +459,7 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
}
async softDeletePrunableExecutions() {
const { maxAge, maxCount } = this.globalConfig.pruning;
const { pruneDataMaxAge, pruneDataMaxCount } = this.globalConfig.executions;
// Sub-query to exclude executions having annotations
const annotatedExecutionsSubQuery = this.manager
@ -470,18 +470,18 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
// Find ids of all executions that were stopped longer that pruneDataMaxAge ago
const date = new Date();
date.setHours(date.getHours() - maxAge);
date.setHours(date.getHours() - pruneDataMaxAge);
const toPrune: Array<FindOptionsWhere<ExecutionEntity>> = [
// date reformatting needed - see https://github.com/typeorm/typeorm/issues/2286
{ stoppedAt: LessThanOrEqual(DateUtils.mixedDateToUtcDatetimeString(date)) },
];
if (maxCount > 0) {
if (pruneDataMaxCount > 0) {
const executions = await this.createQueryBuilder('execution')
.select('execution.id')
.where('execution.id NOT IN ' + annotatedExecutionsSubQuery.getQuery())
.skip(maxCount)
.skip(pruneDataMaxCount)
.take(1)
.orderBy('execution.id', 'DESC')
.getMany();
@ -515,7 +515,7 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
async findSoftDeletedExecutions() {
const date = new Date();
date.setHours(date.getHours() - this.globalConfig.pruning.hardDeleteBuffer);
date.setHours(date.getHours() - this.globalConfig.executions.pruneDataHardDeleteBuffer);
const workflowIdsAndExecutionIds = (
await this.find({

View file

@ -771,8 +771,8 @@ export class TelemetryEventRelay extends EventRelay {
executions_data_save_manual_executions: config.getEnv(
'executions.saveDataManualExecutions',
),
executions_data_prune: this.globalConfig.pruning.isEnabled,
executions_data_max_age: this.globalConfig.pruning.maxAge,
executions_data_prune: this.globalConfig.executions.pruneData,
executions_data_max_age: this.globalConfig.executions.pruneDataMaxAge,
},
n8n_deployment_type: config.getEnv('deployment.type'),
n8n_binary_data_mode: binaryDataConfig.mode,

View file

@ -223,9 +223,9 @@ export class FrontendService {
licensePruneTime: -1,
},
pruning: {
isEnabled: this.globalConfig.pruning.isEnabled,
maxAge: this.globalConfig.pruning.maxAge,
maxCount: this.globalConfig.pruning.maxCount,
isEnabled: this.globalConfig.executions.pruneData,
maxAge: this.globalConfig.executions.pruneDataMaxAge,
maxCount: this.globalConfig.executions.pruneDataMaxCount,
},
security: {
blockFileAccessToN8nFiles: this.securityConfig.blockFileAccessToN8nFiles,

View file

@ -1,4 +1,4 @@
import type { PruningConfig } from '@n8n/config';
import type { ExecutionsConfig } from '@n8n/config';
import { mock } from 'jest-mock-extended';
import type { InstanceSettings } from 'n8n-core';
@ -92,7 +92,7 @@ describe('PruningService', () => {
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock<PruningConfig>({ isEnabled: true }),
mock<ExecutionsConfig>({ pruneData: true }),
);
expect(pruningService.isEnabled).toBe(true);
@ -108,7 +108,7 @@ describe('PruningService', () => {
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock<PruningConfig>({ isEnabled: false }),
mock<ExecutionsConfig>({ pruneData: false }),
);
expect(pruningService.isEnabled).toBe(false);
@ -124,7 +124,7 @@ describe('PruningService', () => {
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock<PruningConfig>({ isEnabled: true }),
mock<ExecutionsConfig>({ pruneData: true }),
);
expect(pruningService.isEnabled).toBe(false);
@ -140,7 +140,7 @@ describe('PruningService', () => {
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock<PruningConfig>({ isEnabled: true }),
mock<ExecutionsConfig>({ pruneData: true }),
);
expect(pruningService.isEnabled).toBe(false);
@ -158,7 +158,7 @@ describe('PruningService', () => {
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock<PruningConfig>({ isEnabled: false }),
mock<ExecutionsConfig>({ pruneData: false }),
);
const scheduleRollingSoftDeletionsSpy = jest.spyOn(
@ -186,7 +186,7 @@ describe('PruningService', () => {
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock<PruningConfig>({ isEnabled: true }),
mock<ExecutionsConfig>({ pruneData: true }),
);
const scheduleRollingSoftDeletionsSpy = jest

View file

@ -1,4 +1,4 @@
import { PruningConfig } from '@n8n/config';
import { ExecutionsConfig } from '@n8n/config';
import { BinaryDataService, InstanceSettings } from 'n8n-core';
import { ensureError } from 'n8n-workflow';
import { strict } from 'node:assert';
@ -26,8 +26,8 @@ export class PruningService {
private hardDeletionTimeout: NodeJS.Timeout | undefined;
private readonly rates = {
softDeletion: this.pruningConfig.softDeleteInterval * Time.minutes.toMilliseconds,
hardDeletion: this.pruningConfig.hardDeleteInterval * Time.minutes.toMilliseconds,
softDeletion: this.executionsConfig.pruneDataIntervals.softDelete * Time.minutes.toMilliseconds,
hardDeletion: this.executionsConfig.pruneDataIntervals.hardDelete * Time.minutes.toMilliseconds,
};
/** Max number of executions to hard-delete in a cycle. */
@ -41,7 +41,7 @@ export class PruningService {
private readonly executionRepository: ExecutionRepository,
private readonly binaryDataService: BinaryDataService,
private readonly orchestrationService: OrchestrationService,
private readonly pruningConfig: PruningConfig,
private readonly executionsConfig: ExecutionsConfig,
) {
this.logger = this.logger.scoped('pruning');
}
@ -59,7 +59,7 @@ export class PruningService {
get isEnabled() {
return (
this.pruningConfig.isEnabled &&
this.executionsConfig.pruneData &&
this.instanceSettings.instanceType === 'main' &&
this.instanceSettings.isLeader
);

View file

@ -1,4 +1,4 @@
import { PruningConfig } from '@n8n/config';
import { ExecutionsConfig } from '@n8n/config';
import { mock } from 'jest-mock-extended';
import { BinaryDataService, InstanceSettings } from 'n8n-core';
import type { ExecutionStatus } from 'n8n-workflow';
@ -27,19 +27,19 @@ describe('softDeleteOnPruningCycle()', () => {
const now = new Date();
const yesterday = new Date(Date.now() - 1 * Time.days.toMilliseconds);
let workflow: WorkflowEntity;
let pruningConfig: PruningConfig;
let executionsConfig: ExecutionsConfig;
beforeAll(async () => {
await testDb.init();
pruningConfig = Container.get(PruningConfig);
executionsConfig = Container.get(ExecutionsConfig);
pruningService = new PruningService(
mockLogger(),
instanceSettings,
Container.get(ExecutionRepository),
mockInstance(BinaryDataService),
mock(),
pruningConfig,
executionsConfig,
);
workflow = await createWorkflow();
@ -62,8 +62,8 @@ describe('softDeleteOnPruningCycle()', () => {
describe('when EXECUTIONS_DATA_PRUNE_MAX_COUNT is set', () => {
beforeAll(() => {
pruningConfig.maxAge = 336;
pruningConfig.maxCount = 1;
executionsConfig.pruneDataMaxAge = 336;
executionsConfig.pruneDataMaxCount = 1;
});
test('should mark as deleted based on EXECUTIONS_DATA_PRUNE_MAX_COUNT', async () => {
@ -163,8 +163,8 @@ describe('softDeleteOnPruningCycle()', () => {
describe('when EXECUTIONS_DATA_MAX_AGE is set', () => {
beforeAll(() => {
pruningConfig.maxAge = 1;
pruningConfig.maxCount = 0;
executionsConfig.pruneDataMaxAge = 1;
executionsConfig.pruneDataMaxCount = 0;
});
test('should mark as deleted based on EXECUTIONS_DATA_MAX_AGE', async () => {