refactor(core): Support multiple log scopes (#11318)

packages/@n8n/config/src/configs/logging.config.ts

@@ -1,14 +1,16 @@
 import { Config, Env, Nested } from '../decorators';
 import { StringArray } from '../utils';
 
-/**
+/** Scopes (areas of functionality) to filter logs by. */
- * Scopes (areas of functionality) to filter logs by.
+export const LOG_SCOPES = [
- *
+	'concurrency',
- * `executions` -> execution lifecycle
+	'license',
- * `license` -> license SDK
+	'multi-main-setup',
- * `scaling` -> scaling mode
+	'pubsub',
- */
+	'redis',
-export const LOG_SCOPES = ['executions', 'license', 'scaling'] as const;
+	'scaling',
+	'waiting-executions',
+] as const;
 
 export type LogScope = (typeof LOG_SCOPES)[number];
 
@@ -59,14 +61,19 @@ export class LoggingConfig {
 	/**
 	 * Scopes to filter logs by. Nothing is filtered by default.
 	 *
-	 * Currently supported log scopes:
+	 * Supported log scopes:
-	 * - `executions`
+	 *
+	 * - `concurrency`
 	 * - `license`
+	 * - `multi-main-setup`
+	 * - `pubsub`
+	 * - `redis`
 	 * - `scaling`
+	 * - `waiting-executions`
 	 *
 	 * @example
 	 * `N8N_LOG_SCOPES=license`
-	 * `N8N_LOG_SCOPES=license,executions`
+	 * `N8N_LOG_SCOPES=license,waiting-executions`
 	 */
 	@Env('N8N_LOG_SCOPES')
 	scopes: StringArray<LogScope> = [];

packages/cli/src/commands/start.ts

@@ -170,7 +170,7 @@ export class Start extends BaseCommand {
 
 		this.logger.info('Initializing n8n process');
 		if (config.getEnv('executions.mode') === 'queue') {
-			const scopedLogger = this.logger.withScope('scaling');
+			const scopedLogger = this.logger.scoped('scaling');
 			scopedLogger.debug('Starting main instance in scaling mode');
 			scopedLogger.debug(`Host ID: ${this.instanceSettings.hostId}`);
 		}
@@ -263,7 +263,7 @@ export class Start extends BaseCommand {
 		await subscriber.subscribe('n8n.commands');
 		await subscriber.subscribe('n8n.worker-response');
 
-		this.logger.withScope('scaling').debug('Pubsub setup completed');
+		this.logger.scoped(['scaling', 'pubsub']).debug('Pubsub setup completed');
 
 		if (!orchestrationService.isMultiMainSetupEnabled) return;
 

packages/cli/src/commands/worker.ts

@@ -69,7 +69,7 @@ export class Worker extends BaseCommand {
 
 		super(argv, cmdConfig);
 
-		this.logger = Container.get(Logger).withScope('scaling');
+		this.logger = Container.get(Logger).scoped('scaling');
 	}
 
 	async init() {
@@ -151,7 +151,7 @@ export class Worker extends BaseCommand {
 		Container.get(PubSubHandler).init();
 		await Container.get(Subscriber).subscribe('n8n.commands');
 
-		this.logger.withScope('scaling').debug('Pubsub setup ready');
+		this.logger.scoped(['scaling', 'pubsub']).debug('Pubsub setup completed');
 	}
 
 	async setConcurrency() {

packages/cli/src/concurrency/concurrency-control.service.ts

@@ -33,7 +33,7 @@ export class ConcurrencyControlService {
 		private readonly telemetry: Telemetry,
 		private readonly eventService: EventService,
 	) {
-		this.logger = this.logger.withScope('executions');
+		this.logger = this.logger.scoped('concurrency');
 
 		this.productionLimit = config.getEnv('executions.concurrency.productionLimit');
 

packages/cli/src/license.ts

@@ -40,7 +40,7 @@ export class License {
 		private readonly licenseMetricsService: LicenseMetricsService,
 		private readonly globalConfig: GlobalConfig,
 	) {
-		this.logger = this.logger.withScope('license');
+		this.logger = this.logger.scoped('license');
 	}
 
 	/**

packages/cli/src/logging/logger.service.ts

@@ -58,9 +58,11 @@ export class Logger {
 		this.internalLogger = internalLogger;
 	}
 
-	withScope(scope: LogScope) {
+	/** Create a logger that injects the given scopes into its log metadata. */
+	scoped(scopes: LogScope | LogScope[]) {
+		scopes = Array.isArray(scopes) ? scopes : [scopes];
 		const scopedLogger = new Logger(this.globalConfig, this.instanceSettings);
-		const childLogger = this.internalLogger.child({ scope });
+		const childLogger = this.internalLogger.child({ scopes });
 
 		scopedLogger.setInternalLogger(childLogger);
 
@@ -106,11 +108,14 @@ export class Logger {
 
 	private scopeFilter() {
 		return winston.format((info: TransformableInfo & { metadata: LogMetadata }) => {
-			const shouldIncludeScope = info.metadata.scope && this.scopes.has(info.metadata.scope);
+			if (!this.isScopingEnabled) return info;
 
-			if (this.isScopingEnabled && !shouldIncludeScope) return false;
+			const { scopes } = info.metadata;
 
-			return info;
+			const shouldIncludeScope =
+				scopes && scopes?.length > 0 && scopes.some((s) => this.scopes.has(s));
+
+			return shouldIncludeScope ? info : false;
 		})();
 	}
 

packages/cli/src/logging/types.ts

@@ -6,7 +6,7 @@ export type LogLevel = (typeof LOG_LEVELS)[number];
 
 export type LogMetadata = {
 	[key: string]: unknown;
-	scope?: LogScope;
+	scopes?: LogScope[];
 	file?: string;
 	function?: string;
 };

packages/cli/src/scaling/job-processor.ts

@@ -40,7 +40,7 @@ export class JobProcessor {
 		private readonly nodeTypes: NodeTypes,
 		private readonly instanceSettings: InstanceSettings,
 	) {
-		this.logger = this.logger.withScope('scaling');
+		this.logger = this.logger.scoped('scaling');
 	}
 
 	async processJob(job: Job): Promise<JobResult> {

packages/cli/src/scaling/multi-main-setup.ee.ts

@@ -36,7 +36,7 @@ export class MultiMainSetup extends TypedEmitter<MultiMainEvents> {
 		private readonly globalConfig: GlobalConfig,
 	) {
 		super();
-		this.logger = this.logger.withScope('scaling');
+		this.logger = this.logger.scoped(['scaling', 'multi-main-setup']);
 	}
 
 	private leaderKey: string;

packages/cli/src/scaling/pubsub/publisher.service.ts

@@ -26,7 +26,7 @@ export class Publisher {
 		// @TODO: Once this class is only ever initialized in scaling mode, throw in the next line instead.
 		if (config.getEnv('executions.mode') !== 'queue') return;
 
-		this.logger = this.logger.withScope('scaling');
+		this.logger = this.logger.scoped(['scaling', 'pubsub']);
 
 		this.client = this.redisClientService.createClient({ type: 'publisher(n8n)' });
 	}

packages/cli/src/scaling/pubsub/subscriber.service.ts

@@ -27,7 +27,7 @@ export class Subscriber {
 		// @TODO: Once this class is only ever initialized in scaling mode, throw in the next line instead.
 		if (config.getEnv('executions.mode') !== 'queue') return;
 
-		this.logger = this.logger.withScope('scaling');
+		this.logger = this.logger.scoped(['scaling', 'pubsub']);
 
 		this.client = this.redisClientService.createClient({ type: 'subscriber(n8n)' });
 

packages/cli/src/scaling/scaling.service.ts

@@ -51,7 +51,7 @@ export class ScalingService {
 		private readonly orchestrationService: OrchestrationService,
 		private readonly eventService: EventService,
 	) {
-		this.logger = this.logger.withScope('scaling');
+		this.logger = this.logger.scoped('scaling');
 	}
 
 	// #region Lifecycle

packages/cli/src/scaling/worker-server.ts

@@ -58,7 +58,7 @@ export class WorkerServer {
 	) {
 		assert(this.instanceSettings.instanceType === 'worker');
 
-		this.logger = this.logger.withScope('scaling');
+		this.logger = this.logger.scoped('scaling');
 
 		this.app = express();
 

packages/cli/src/services/redis-client.service.ts

@@ -37,6 +37,9 @@ export class RedisClientService extends TypedEmitter<RedisEventMap> {
 		private readonly globalConfig: GlobalConfig,
 	) {
 		super();
+
+		this.logger = this.logger.scoped(['redis', 'scaling']);
+
 		this.registerListeners();
 	}
 
@@ -99,9 +102,11 @@ export class RedisClientService extends TypedEmitter<RedisEventMap> {
 		options.host = host;
 		options.port = port;
 
-		this.logger.debug('[Redis] Initializing regular client', { type, host, port });
+		const client = new ioRedis(options);
 
-		return new ioRedis(options);
+		this.logger.debug(`Started Redis client ${type}`, { type, host, port });
+
+		return client;
 	}
 
 	private createClusterClient({
@@ -115,12 +120,14 @@ export class RedisClientService extends TypedEmitter<RedisEventMap> {
 
 		const clusterNodes = this.clusterNodes();
 
-		this.logger.debug('[Redis] Initializing cluster client', { type, clusterNodes });
+		const clusterClient = new ioRedis.Cluster(clusterNodes, {
-
-		return new ioRedis.Cluster(clusterNodes, {
 			redisOptions: options,
 			clusterRetryStrategy: this.retryStrategy(),
 		});
+
+		this.logger.debug(`Started Redis cluster client ${type}`, { type, clusterNodes });
+
+		return clusterClient;
 	}
 
 	private getOptions({ extraOptions }: { extraOptions?: RedisOptions }) {

packages/cli/src/wait-tracker.ts

@@ -31,7 +31,7 @@ export class WaitTracker {
 		private readonly orchestrationService: OrchestrationService,
 		private readonly instanceSettings: InstanceSettings,
 	) {
-		this.logger = this.logger.withScope('executions');
+		this.logger = this.logger.scoped('waiting-executions');
 	}
 
 	has(executionId: string) {

packages/cli/test/shared/mocking.ts

@@ -25,5 +25,4 @@ export const mockEntityManager = (entityClass: Class) => {
 	return entityManager;
 };
 
-export const mockLogger = () =>
+export const mockLogger = () => mock<Logger>({ scoped: jest.fn().mockReturnValue(mock<Logger>()) });
-	mock<Logger>({ withScope: jest.fn().mockReturnValue(mock<Logger>()) });