mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2024-12-24 04:04:06 -08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

test(core): Expand test coverage for pruning (#11567)

Browse source
This commit is contained in:
Iván Ovejero 2024-11-05 18:21:56 +01:00 committed by GitHub
parent 46eceabc27
commit 19d55da4ad
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 242 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
packages/@n8n/config/src/configs/logging.config.ts
View file

@ -7,6 +7,7 @@ export const LOG_SCOPES = [
'external-secrets',
'license',
'multi-main-setup',
'pruning',
'pubsub',
'redis',
'scaling',

2
packages/cli/src/commands/start.ts
View file

@ -27,7 +27,7 @@ import { Subscriber } from '@/scaling/pubsub/subscriber.service';
import { Server } from '@/server';
import { OrchestrationService } from '@/services/orchestration.service';
import { OwnershipService } from '@/services/ownership.service';
import { PruningService } from '@/services/pruning.service';
import { PruningService } from '@/services/pruning/pruning.service';
import { UrlService } from '@/services/url.service';
import { WaitTracker } from '@/wait-tracker';
import { WorkflowRunner } from '@/workflow-runner';

2
packages/cli/src/databases/repositories/execution.repository.ts
View file

@ -513,7 +513,7 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
.execute();
}
async hardDeleteSoftDeletedExecutions() {
async findSoftDeletedExecutions() {
const date = new Date();
date.setHours(date.getHours() - this.globalConfig.pruning.hardDeleteBuffer);

213
packages/cli/src/services/pruning/__tests__/pruning.service.test.ts Normal file
View file

@ -0,0 +1,213 @@
import type { GlobalConfig } from '@n8n/config';
import { mock } from 'jest-mock-extended';
import type { InstanceSettings } from 'n8n-core';
import type { MultiMainSetup } from '@/scaling/multi-main-setup.ee';
import type { OrchestrationService } from '@/services/orchestration.service';
import { mockLogger } from '@test/mocking';
import { PruningService } from '../pruning.service';
describe('PruningService', () => {
describe('init', () => {
it('should start pruning if leader', () => {
const pruningService = new PruningService(
mockLogger(),
mock<InstanceSettings>({ isLeader: true }),
mock(),
mock(),
mock<OrchestrationService>({
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock(),
);
const startPruningSpy = jest.spyOn(pruningService, 'startPruning');
pruningService.init();
expect(startPruningSpy).toHaveBeenCalled();
});
it('should not start pruning if follower', () => {
const pruningService = new PruningService(
mockLogger(),
mock<InstanceSettings>({ isLeader: false }),
mock(),
mock(),
mock<OrchestrationService>({
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock(),
);
const startPruningSpy = jest.spyOn(pruningService, 'startPruning');
pruningService.init();
expect(startPruningSpy).not.toHaveBeenCalled();
});
it('should register leadership events if multi-main setup is enabled', () => {
const pruningService = new PruningService(
mockLogger(),
mock<InstanceSettings>({ isLeader: true }),
mock(),
mock(),
mock<OrchestrationService>({
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>({ on: jest.fn() }),
}),
mock(),
);
pruningService.init();
// @ts-expect-error Private method
expect(pruningService.orchestrationService.multiMainSetup.on).toHaveBeenCalledWith(
'leader-takeover',
expect.any(Function),
);
// @ts-expect-error Private method
expect(pruningService.orchestrationService.multiMainSetup.on).toHaveBeenCalledWith(
'leader-stepdown',
expect.any(Function),
);
});
});
describe('isEnabled', () => {
it('should return `true` based on config if leader main', () => {
const pruningService = new PruningService(
mockLogger(),
mock<InstanceSettings>({ isLeader: true, instanceType: 'main' }),
mock(),
mock(),
mock<OrchestrationService>({
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock<GlobalConfig>({ pruning: { isEnabled: true } }),
);
// @ts-expect-error Private method
const isEnabled = pruningService.isEnabled();
expect(isEnabled).toBe(true);
});
it('should return `false` based on config if leader main', () => {
const pruningService = new PruningService(
mockLogger(),
mock<InstanceSettings>({ isLeader: true, instanceType: 'main' }),
mock(),
mock(),
mock<OrchestrationService>({
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock<GlobalConfig>({ pruning: { isEnabled: false } }),
);
// @ts-expect-error Private method
const isEnabled = pruningService.isEnabled();
expect(isEnabled).toBe(false);
});
it('should return `false` if non-main even if enabled', () => {
const pruningService = new PruningService(
mockLogger(),
mock<InstanceSettings>({ isLeader: false, instanceType: 'worker' }),
mock(),
mock(),
mock<OrchestrationService>({
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock<GlobalConfig>({ pruning: { isEnabled: true } }),
);
// @ts-expect-error Private method
const isEnabled = pruningService.isEnabled();
expect(isEnabled).toBe(false);
});
it('should return `false` if follower main even if enabled', () => {
const pruningService = new PruningService(
mockLogger(),
mock<InstanceSettings>({ isLeader: false, isFollower: true, instanceType: 'main' }),
mock(),
mock(),
mock<OrchestrationService>({
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock<GlobalConfig>({ pruning: { isEnabled: true }, multiMainSetup: { enabled: true } }),
);
// @ts-expect-error Private method
const isEnabled = pruningService.isEnabled();
expect(isEnabled).toBe(false);
});
});
describe('startPruning', () => {
it('should not start pruning if service is disabled', () => {
const pruningService = new PruningService(
mockLogger(),
mock<InstanceSettings>({ isLeader: true, instanceType: 'main' }),
mock(),
mock(),
mock<OrchestrationService>({
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock<GlobalConfig>({ pruning: { isEnabled: false } }),
);
// @ts-expect-error Private method
const setSoftDeletionInterval = jest.spyOn(pruningService, 'setSoftDeletionInterval');
// @ts-expect-error Private method
const scheduleHardDeletion = jest.spyOn(pruningService, 'scheduleHardDeletion');
pruningService.startPruning();
expect(setSoftDeletionInterval).not.toHaveBeenCalled();
expect(scheduleHardDeletion).not.toHaveBeenCalled();
});
it('should start pruning if service is enabled', () => {
const pruningService = new PruningService(
mockLogger(),
mock<InstanceSettings>({ isLeader: true, instanceType: 'main' }),
mock(),
mock(),
mock<OrchestrationService>({
isMultiMainSetupEnabled: true,
multiMainSetup: mock<MultiMainSetup>(),
}),
mock<GlobalConfig>({ pruning: { isEnabled: true } }),
);
const setSoftDeletionInterval = jest
// @ts-expect-error Private method
.spyOn(pruningService, 'setSoftDeletionInterval')
.mockImplementation();
const scheduleHardDeletion = jest
// @ts-expect-error Private method
.spyOn(pruningService, 'scheduleHardDeletion')
.mockImplementation();
pruningService.startPruning();
expect(setSoftDeletionInterval).toHaveBeenCalled();
expect(scheduleHardDeletion).toHaveBeenCalled();
});
});
});

44
packages/cli/src/services/pruning.service.ts → packages/cli/src/services/pruning/pruning.service.ts
View file

@ -3,12 +3,12 @@ import { BinaryDataService, InstanceSettings } from 'n8n-core';
import { jsonStringify } from 'n8n-workflow';
import { Service } from 'typedi';
import { inTest, TIME } from '@/constants';
import { TIME } from '@/constants';
import { ExecutionRepository } from '@/databases/repositories/execution.repository';
import { OnShutdown } from '@/decorators/on-shutdown';
import { Logger } from '@/logging/logger.service';
import { OrchestrationService } from './orchestration.service';
import { OrchestrationService } from '../orchestration.service';
@Service()
export class PruningService {
@ -32,7 +32,9 @@ export class PruningService {
private readonly binaryDataService: BinaryDataService,
private readonly orchestrationService: OrchestrationService,
private readonly globalConfig: GlobalConfig,
) {}
) {
this.logger = this.logger.scoped('pruning');
}
/**
* @important Requires `OrchestrationService` to be initialized.
@ -49,9 +51,9 @@ export class PruningService {
}
}
private isPruningEnabled() {
private isEnabled() {
const { instanceType, isFollower } = this.instanceSettings;
if (!this.globalConfig.pruning.isEnabled || inTest || instanceType !== 'main') {
if (!this.globalConfig.pruning.isEnabled || instanceType !== 'main') {
return false;
}
@ -66,23 +68,23 @@ export class PruningService {
* @important Call this method only after DB migrations have completed.
*/
startPruning() {
if (!this.isPruningEnabled()) return;
if (!this.isEnabled()) return;
if (this.isShuttingDown) {
this.logger.warn('[Pruning] Cannot start pruning while shutting down');
this.logger.warn('Cannot start pruning while shutting down');
return;
}
this.logger.debug('[Pruning] Starting soft-deletion and hard-deletion timers');
this.logger.debug('Starting soft-deletion and hard-deletion timers');
this.setSoftDeletionInterval();
this.scheduleHardDeletion();
}
stopPruning() {
if (!this.isPruningEnabled()) return;
if (!this.isEnabled()) return;
this.logger.debug('[Pruning] Removing soft-deletion and hard-deletion timers');
this.logger.debug('Removing soft-deletion and hard-deletion timers');
clearInterval(this.softDeletionInterval);
clearTimeout(this.hardDeletionTimeout);
@ -96,7 +98,7 @@ export class PruningService {
this.rates.softDeletion,
);
this.logger.debug(`[Pruning] Soft-deletion scheduled every ${when}`);
this.logger.debug(`Soft-deletion scheduled every ${when}`);
}
private scheduleHardDeletion(rateMs = this.rates.hardDeletion) {
@ -113,27 +115,27 @@ export class PruningService {
? error.message
: jsonStringify(error, { replaceCircularRefs: true });
this.logger.error('[Pruning] Failed to hard-delete executions', { errorMessage });
this.logger.error('Failed to hard-delete executions', { errorMessage });
});
}, rateMs);
this.logger.debug(`[Pruning] Hard-deletion scheduled for next ${when}`);
this.logger.debug(`Hard-deletion scheduled for next ${when}`);
}
/**
* Mark executions as deleted based on age and count, in a pruning cycle.
*/
async softDeleteOnPruningCycle() {
this.logger.debug('[Pruning] Starting soft-deletion of executions');
this.logger.debug('Starting soft-deletion of executions');
const result = await this.executionRepository.softDeletePrunableExecutions();
if (result.affected === 0) {
this.logger.debug('[Pruning] Found no executions to soft-delete');
this.logger.debug('Found no executions to soft-delete');
return;
}
this.logger.debug('[Pruning] Soft-deleted executions', { count: result.affected });
this.logger.debug('Soft-deleted executions', { count: result.affected });
}
@OnShutdown()
@ -147,26 +149,26 @@ export class PruningService {
* @return Delay in ms after which the next cycle should be started
*/
private async hardDeleteOnPruningCycle() {
const ids = await this.executionRepository.hardDeleteSoftDeletedExecutions();
const ids = await this.executionRepository.findSoftDeletedExecutions();
const executionIds = ids.map((o) => o.executionId);
if (executionIds.length === 0) {
this.logger.debug('[Pruning] Found no executions to hard-delete');
this.logger.debug('Found no executions to hard-delete');
return this.rates.hardDeletion;
}
try {
this.logger.debug('[Pruning] Starting hard-deletion of executions', { executionIds });
this.logger.debug('Starting hard-deletion of executions', { executionIds });
await this.binaryDataService.deleteMany(ids);
await this.executionRepository.deleteByIds(executionIds);
this.logger.debug('[Pruning] Hard-deleted executions', { executionIds });
this.logger.debug('Hard-deleted executions', { executionIds });
} catch (error) {
this.logger.error('[Pruning] Failed to hard-delete executions', {
this.logger.error('Failed to hard-delete executions', {
executionIds,
error: error instanceof Error ? error.message : `${error}`,
});

7
packages/cli/test/integration/pruning.service.test.ts
View file

@ -8,8 +8,7 @@ import { TIME } from '@/constants';
import type { ExecutionEntity } from '@/databases/entities/execution-entity';
import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
import { ExecutionRepository } from '@/databases/repositories/execution.repository';
import { Logger } from '@/logging/logger.service';
import { PruningService } from '@/services/pruning.service';
import { PruningService } from '@/services/pruning/pruning.service';
import {
annotateExecution,
@ -18,7 +17,7 @@ import {
} from './shared/db/executions';
import { createWorkflow } from './shared/db/workflows';
import * as testDb from './shared/test-db';
import { mockInstance } from '../shared/mocking';
import { mockInstance, mockLogger } from '../shared/mocking';
describe('softDeleteOnPruningCycle()', () => {
let pruningService: PruningService;
@ -35,7 +34,7 @@ describe('softDeleteOnPruningCycle()', () => {
globalConfig = Container.get(GlobalConfig);
pruningService = new PruningService(
mockInstance(Logger),
mockLogger(),
instanceSettings,
Container.get(ExecutionRepository),
mockInstance(BinaryDataService),