refactor(core): Load and validate all config at startup (no-changelog) (#5283)

packages/cli/src/CredentialsOverwrites.ts

@@ -1,18 +1,15 @@
+import config from '@/config';
 import type { ICredentialDataDecryptedObject, ICredentialTypes } from 'n8n-workflow';
 import { deepCopy, LoggerProxy as Logger, jsonParse } from 'n8n-workflow';
 import type { ICredentialsOverwrite } from '@/Interfaces';
-import * as GenericHelpers from '@/GenericHelpers';
 
 class CredentialsOverwritesClass {
 	private overwriteData: ICredentialsOverwrite = {};
 
 	private resolvedTypes: string[] = [];
 
-	constructor(private credentialTypes: ICredentialTypes) {}
-
-	async init() {
-		const data = (await GenericHelpers.getConfigValue('credentials.overwrite.data')) as string;
-
+	constructor(private credentialTypes: ICredentialTypes) {
+		const data = config.getEnv('credentials.overwrite.data');
 		const overwriteData = jsonParse<ICredentialsOverwrite>(data, {
 			errorMessage: 'The credentials-overwrite is not valid JSON.',
 		});

packages/cli/src/Db.ts

@@ -14,7 +14,6 @@ import type {
 import { DataSource as Connection } from 'typeorm';
 import type { TlsOptions } from 'tls';
 import type { DatabaseType, IDatabaseCollections } from '@/Interfaces';
-import * as GenericHelpers from '@/GenericHelpers';
 
 import config from '@/config';
 
@@ -44,17 +43,13 @@ export function linkRepository<Entity extends ObjectLiteral>(
 	return connection.getRepository(entityClass);
 }
 
-export async function getConnectionOptions(dbType: DatabaseType): Promise<ConnectionOptions> {
+export function getConnectionOptions(dbType: DatabaseType): ConnectionOptions {
 	switch (dbType) {
 		case 'postgresdb':
-			const sslCa = (await GenericHelpers.getConfigValue('database.postgresdb.ssl.ca')) as string;
-			const sslCert = (await GenericHelpers.getConfigValue(
-				'database.postgresdb.ssl.cert',
-			)) as string;
-			const sslKey = (await GenericHelpers.getConfigValue('database.postgresdb.ssl.key')) as string;
-			const sslRejectUnauthorized = (await GenericHelpers.getConfigValue(
-				'database.postgresdb.ssl.rejectUnauthorized',
-			)) as boolean;
+			const sslCa = config.getEnv('database.postgresdb.ssl.ca');
+			const sslCert = config.getEnv('database.postgresdb.ssl.cert');
+			const sslKey = config.getEnv('database.postgresdb.ssl.key');
+			const sslRejectUnauthorized = config.getEnv('database.postgresdb.ssl.rejectUnauthorized');
 
 			let ssl: TlsOptions | undefined;
 			if (sslCa !== '' || sslCert !== '' || sslKey !== '' || !sslRejectUnauthorized) {
@@ -68,7 +63,7 @@ export async function getConnectionOptions(dbType: DatabaseType): Promise<Connec
 
 			return {
 				...getPostgresConnectionOptions(),
-				...(await getOptionOverrides('postgresdb')),
+				...getOptionOverrides('postgresdb'),
 				ssl,
 			};
 
@@ -76,7 +71,7 @@ export async function getConnectionOptions(dbType: DatabaseType): Promise<Connec
 		case 'mysqldb':
 			return {
 				...(dbType === 'mysqldb' ? getMysqlConnectionOptions() : getMariaDBConnectionOptions()),
-				...(await getOptionOverrides('mysqldb')),
+				...getOptionOverrides('mysqldb'),
 				timezone: 'Z', // set UTC as default
 			};
 
@@ -93,17 +88,13 @@ export async function init(
 ): Promise<IDatabaseCollections> {
 	if (isInitialized) return collections;
 
-	const dbType = (await GenericHelpers.getConfigValue('database.type')) as DatabaseType;
-	const connectionOptions = testConnectionOptions ?? (await getConnectionOptions(dbType));
+	const dbType = config.getEnv('database.type');
+	const connectionOptions = testConnectionOptions ?? getConnectionOptions(dbType);
 
-	let loggingOption: LoggerOptions = (await GenericHelpers.getConfigValue(
-		'database.logging.enabled',
-	)) as boolean;
+	let loggingOption: LoggerOptions = config.getEnv('database.logging.enabled');
 
 	if (loggingOption) {
-		const optionsString = (
-			(await GenericHelpers.getConfigValue('database.logging.options')) as string
-		).replace(/\s+/g, '');
+		const optionsString = config.getEnv('database.logging.options').replace(/\s+/g, '');
 
 		if (optionsString === 'all') {
 			loggingOption = optionsString;
@@ -112,9 +103,7 @@ export async function init(
 		}
 	}
 
-	const maxQueryExecutionTime = (await GenericHelpers.getConfigValue(
-		'database.logging.maxQueryExecutionTime',
-	)) as string;
+	const maxQueryExecutionTime = config.getEnv('database.logging.maxQueryExecutionTime');
 
 	Object.assign(connectionOptions, {
 		entities: Object.values(entities),

packages/cli/src/GenericHelpers.ts

@@ -5,22 +5,19 @@
 /* eslint-disable no-underscore-dangle */
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 import type express from 'express';
-import { readFile as fsReadFile } from 'fs/promises';
 import type {
 	ExecutionError,
-	IDataObject,
 	INode,
 	IRunExecutionData,
 	Workflow,
 	WorkflowExecuteMode,
 } from 'n8n-workflow';
 import { validate } from 'class-validator';
+import { Like } from 'typeorm';
 import config from '@/config';
 import * as Db from '@/Db';
 import type { ICredentialsDb, IExecutionDb, IExecutionFlattedDb, IWorkflowDb } from '@/Interfaces';
 import * as ResponseHelper from '@/ResponseHelper';
-// eslint-disable-next-line import/order
-import { Like } from 'typeorm';
 import type { WorkflowEntity } from '@db/entities/WorkflowEntity';
 import type { CredentialsEntity } from '@db/entities/CredentialsEntity';
 import type { TagEntity } from '@db/entities/TagEntity';
@@ -28,7 +25,6 @@ import type { User } from '@db/entities/User';
 
 /**
  * Returns the base URL n8n is reachable from
- *
  */
 export function getBaseUrl(): string {
 	const protocol = config.getEnv('protocol');
@@ -44,73 +40,11 @@ export function getBaseUrl(): string {
 
 /**
  * Returns the session id if one is set
- *
  */
 export function getSessionId(req: express.Request): string | undefined {
 	return req.headers.sessionid as string | undefined;
 }
 
-/**
- * Extracts configuration schema for key
- */
-function extractSchemaForKey(configKey: string, configSchema: IDataObject): IDataObject {
-	const configKeyParts = configKey.split('.');
-
-	// eslint-disable-next-line no-restricted-syntax
-	for (const key of configKeyParts) {
-		if (configSchema[key] === undefined) {
-			throw new Error(`Key "${key}" of ConfigKey "${configKey}" does not exist`);
-			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		} else if ((configSchema[key]! as IDataObject)._cvtProperties === undefined) {
-			configSchema = configSchema[key] as IDataObject;
-		} else {
-			configSchema = (configSchema[key] as IDataObject)._cvtProperties as IDataObject;
-		}
-	}
-	return configSchema;
-}
-
-/**
- * Gets value from config with support for "_FILE" environment variables
- *
- * @param {string} configKey The key of the config data to get
- */
-export async function getConfigValue(
-	configKey: string,
-): Promise<string | boolean | number | undefined> {
-	// Get the environment variable
-	const configSchema = config.getSchema();
-	// @ts-ignore
-	const currentSchema = extractSchemaForKey(configKey, configSchema._cvtProperties as IDataObject);
-	// Check if environment variable is defined for config key
-	if (currentSchema.env === undefined) {
-		// No environment variable defined, so return value from config
-		// @ts-ignore
-		return config.getEnv(configKey);
-	}
-
-	// Check if special file environment variable exists
-	const fileEnvironmentVariable = process.env[`${currentSchema.env}_FILE`];
-	if (fileEnvironmentVariable === undefined) {
-		// Does not exist, so return value from config
-		// @ts-ignore
-		return config.getEnv(configKey);
-	}
-
-	let data;
-	try {
-		data = await fsReadFile(fileEnvironmentVariable, 'utf8');
-	} catch (error) {
-		if (error.code === 'ENOENT') {
-			throw new Error(`The file "${fileEnvironmentVariable}" could not be found.`);
-		}
-
-		throw error;
-	}
-
-	return data;
-}
-
 /**
  * Generate a unique name for a workflow or credentials entity.
  *

packages/cli/src/Server.ts

@@ -409,23 +409,17 @@ class Server extends AbstractServer {
 		// Check for basic auth credentials if activated
 		const basicAuthActive = config.getEnv('security.basicAuth.active');
 		if (basicAuthActive) {
-			const basicAuthUser = (await GenericHelpers.getConfigValue(
-				'security.basicAuth.user',
-			)) as string;
+			const basicAuthUser = config.getEnv('security.basicAuth.user');
 			if (basicAuthUser === '') {
 				throw new Error('Basic auth is activated but no user got defined. Please set one!');
 			}
 
-			const basicAuthPassword = (await GenericHelpers.getConfigValue(
-				'security.basicAuth.password',
-			)) as string;
+			const basicAuthPassword = config.getEnv('security.basicAuth.password');
 			if (basicAuthPassword === '') {
 				throw new Error('Basic auth is activated but no password got defined. Please set one!');
 			}
 
-			const basicAuthHashEnabled = (await GenericHelpers.getConfigValue(
-				'security.basicAuth.hash',
-			)) as boolean;
+			const basicAuthHashEnabled = config.getEnv('security.basicAuth.hash') as boolean;
 
 			let validPassword: null | string = null;
 
@@ -483,31 +477,19 @@ class Server extends AbstractServer {
 		// Check for and validate JWT if configured
 		const jwtAuthActive = config.getEnv('security.jwtAuth.active');
 		if (jwtAuthActive) {
-			const jwtAuthHeader = (await GenericHelpers.getConfigValue(
-				'security.jwtAuth.jwtHeader',
-			)) as string;
+			const jwtAuthHeader = config.getEnv('security.jwtAuth.jwtHeader');
 			if (jwtAuthHeader === '') {
 				throw new Error('JWT auth is activated but no request header was defined. Please set one!');
 			}
-			const jwksUri = (await GenericHelpers.getConfigValue('security.jwtAuth.jwksUri')) as string;
+			const jwksUri = config.getEnv('security.jwtAuth.jwksUri');
 			if (jwksUri === '') {
 				throw new Error('JWT auth is activated but no JWK Set URI was defined. Please set one!');
 			}
-			const jwtHeaderValuePrefix = (await GenericHelpers.getConfigValue(
-				'security.jwtAuth.jwtHeaderValuePrefix',
-			)) as string;
-			const jwtIssuer = (await GenericHelpers.getConfigValue(
-				'security.jwtAuth.jwtIssuer',
-			)) as string;
-			const jwtNamespace = (await GenericHelpers.getConfigValue(
-				'security.jwtAuth.jwtNamespace',
-			)) as string;
-			const jwtAllowedTenantKey = (await GenericHelpers.getConfigValue(
-				'security.jwtAuth.jwtAllowedTenantKey',
-			)) as string;
-			const jwtAllowedTenant = (await GenericHelpers.getConfigValue(
-				'security.jwtAuth.jwtAllowedTenant',
-			)) as string;
+			const jwtHeaderValuePrefix = config.getEnv('security.jwtAuth.jwtHeaderValuePrefix');
+			const jwtIssuer = config.getEnv('security.jwtAuth.jwtIssuer');
+			const jwtNamespace = config.getEnv('security.jwtAuth.jwtNamespace');
+			const jwtAllowedTenantKey = config.getEnv('security.jwtAuth.jwtAllowedTenantKey');
+			const jwtAllowedTenant = config.getEnv('security.jwtAuth.jwtAllowedTenant');
 
 			// eslint-disable-next-line no-inner-declarations
 			function isTenantAllowed(decodedToken: object): boolean {
@@ -1456,7 +1438,7 @@ export async function start(): Promise<void> {
 	const binaryDataConfig = config.getEnv('binaryDataManager');
 	const diagnosticInfo: IDiagnosticInfo = {
 		basicAuthActive: config.getEnv('security.basicAuth.active'),
-		databaseType: (await GenericHelpers.getConfigValue('database.type')) as DatabaseType,
+		databaseType: config.getEnv('database.type'),
 		disableProductionWebhooksOnMainProcess: config.getEnv(
 			'endpoints.disableProductionWebhooksOnMainProcess',
 		),

packages/cli/src/UserManagement/email/UserManagementMailer.ts

@@ -2,7 +2,6 @@ import { existsSync } from 'fs';
 import { readFile } from 'fs/promises';
 import Handlebars from 'handlebars';
 import { join as pathJoin } from 'path';
-import * as GenericHelpers from '@/GenericHelpers';
 import config from '@/config';
 import type {
 	InviteEmailData,
@@ -23,9 +22,7 @@ async function getTemplate(
 ): Promise<Template> {
 	let template = templates[templateName];
 	if (!template) {
-		const templateOverride = (await GenericHelpers.getConfigValue(
-			`userManagement.emails.templates.${templateName}`,
-		)) as string;
+		const templateOverride = config.getEnv(`userManagement.emails.templates.${templateName}`);
 
 		let markup;
 		if (templateOverride && existsSync(templateOverride)) {

packages/cli/src/WaitTracker.ts

@@ -14,12 +14,11 @@ import type { FindManyOptions, ObjectLiteral } from 'typeorm';
 import { LessThanOrEqual } from 'typeorm';
 import { DateUtils } from 'typeorm/util/DateUtils';
 
+import config from '@/config';
 import * as Db from '@/Db';
 import * as ResponseHelper from '@/ResponseHelper';
-import * as GenericHelpers from '@/GenericHelpers';
 import * as ActiveExecutions from '@/ActiveExecutions';
 import type {
-	DatabaseType,
 	IExecutionFlattedDb,
 	IExecutionsStopData,
 	IWorkflowExecutionDataProcess,
@@ -63,7 +62,8 @@ export class WaitTrackerClass {
 				waitTill: 'ASC',
 			},
 		};
-		const dbType = (await GenericHelpers.getConfigValue('database.type')) as DatabaseType;
+
+		const dbType = config.getEnv('database.type');
 		if (dbType === 'sqlite') {
 			// This is needed because of issue in TypeORM <> SQLite:
 			// https://github.com/typeorm/typeorm/issues/2286

packages/cli/src/WorkflowRunnerProcess.ts

@@ -108,8 +108,7 @@ class WorkflowRunnerProcess {
 		const credentialTypes = CredentialTypes(loadNodesAndCredentials);
 
 		// Load the credentials overwrites if any exist
-		const credentialsOverwrites = CredentialsOverwrites(credentialTypes);
-		await credentialsOverwrites.init();
+		CredentialsOverwrites(credentialTypes);
 
 		// Load all external hooks
 		const externalHooks = ExternalHooks();

packages/cli/src/commands/db/revert.ts

@@ -28,7 +28,7 @@ export class DbRevertMigrationCommand extends Command {
 		try {
 			const dbType = config.getEnv('database.type');
 			const connectionOptions: ConnectionOptions = {
-				...(await getConnectionOptions(dbType)),
+				...getConnectionOptions(dbType),
 				subscribers: [],
 				synchronize: false,
 				migrationsRun: false,

packages/cli/src/commands/execute.ts

@@ -130,7 +130,7 @@ export class Execute extends Command {
 		const credentialTypes = CredentialTypes(loadNodesAndCredentials);
 
 		// Load the credentials overwrites if any exist
-		await CredentialsOverwrites(credentialTypes).init();
+		CredentialsOverwrites(credentialTypes);
 
 		// Load all external hooks
 		const externalHooks = ExternalHooks();

packages/cli/src/commands/executeBatch.ts

@@ -318,7 +318,7 @@ export class ExecuteBatch extends Command {
 		const credentialTypes = CredentialTypes(loadNodesAndCredentials);
 
 		// Load the credentials overwrites if any exist
-		await CredentialsOverwrites(credentialTypes).init();
+		CredentialsOverwrites(credentialTypes);
 
 		// Load all external hooks
 		const externalHooks = ExternalHooks();

packages/cli/src/commands/start.ts

@@ -28,7 +28,6 @@ import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
 import { NodeTypes } from '@/NodeTypes';
 import { InternalHooksManager } from '@/InternalHooksManager';
 import * as Server from '@/Server';
-import type { DatabaseType } from '@/Interfaces';
 import * as TestWebhooks from '@/TestWebhooks';
 import { WaitTracker } from '@/WaitTracker';
 
@@ -279,7 +278,7 @@ export class Start extends Command {
 			const credentialTypes = CredentialTypes(loadNodesAndCredentials);
 
 			// Load the credentials overwrites if any exist
-			await CredentialsOverwrites(credentialTypes).init();
+			CredentialsOverwrites(credentialTypes);
 
 			await loadNodesAndCredentials.generateTypesForFrontend();
 
@@ -341,8 +340,7 @@ export class Start extends Command {
 				);
 			}
 
-			const dbType = (await GenericHelpers.getConfigValue('database.type')) as DatabaseType;
-
+			const dbType = config.getEnv('database.type');
 			if (dbType === 'sqlite') {
 				const shouldRunVacuum = config.getEnv('database.sqlite.executeVacuumOnStartup');
 				if (shouldRunVacuum) {

packages/cli/src/commands/webhook.ts

@@ -128,7 +128,7 @@ export class Webhook extends Command {
 			const credentialTypes = CredentialTypes(loadNodesAndCredentials);
 
 			// Load the credentials overwrites if any exist
-			await CredentialsOverwrites(credentialTypes).init();
+			CredentialsOverwrites(credentialTypes);
 
 			// Load all external hooks
 			const externalHooks = ExternalHooks();

packages/cli/src/commands/worker.ts

@@ -285,7 +285,7 @@ export class Worker extends Command {
 				const credentialTypes = CredentialTypes(loadNodesAndCredentials);
 
 				// Load the credentials overwrites if any exist
-				await CredentialsOverwrites(credentialTypes).init();
+				CredentialsOverwrites(credentialTypes);
 
 				// Load all external hooks
 				const externalHooks = ExternalHooks();

packages/cli/src/config/index.ts

@@ -1,11 +1,9 @@
-/* eslint-disable @typescript-eslint/naming-convention */
-/* eslint-disable @typescript-eslint/unbound-method */
-/* eslint-disable no-console */
 import convict from 'convict';
 import dotenv from 'dotenv';
 import { tmpdir } from 'os';
-import { mkdtempSync } from 'fs';
+import { mkdtempSync, readFileSync } from 'fs';
 import { join } from 'path';
+import { LoggerProxy as Logger } from 'n8n-workflow';
 import { schema } from './schema';
 import { inTest, inE2ETests } from '@/constants';
 
@@ -19,8 +17,7 @@ if (inE2ETests) {
 		EXTERNAL_FRONTEND_HOOKS_URLS: '',
 		N8N_PERSONALIZATION_ENABLED: 'false',
 	};
-}
-if (inTest) {
+} else if (inTest) {
 	process.env.N8N_PUBLIC_API_DISABLED = 'true';
 	process.env.N8N_PUBLIC_API_SWAGGERUI_DISABLED = 'true';
 } else {
@@ -33,20 +30,46 @@ if (inE2ETests) {
 	config.set('enterprise.features.sharing', true);
 }
 
+// eslint-disable-next-line @typescript-eslint/unbound-method
 config.getEnv = config.get;
 
-if (!inE2ETests) {
+// Load overwrites when not in tests
+if (!inE2ETests && !inTest) {
 	// Overwrite default configuration with settings which got defined in
 	// optional configuration files
 	const { N8N_CONFIG_FILES } = process.env;
 	if (N8N_CONFIG_FILES !== undefined) {
 		const configFiles = N8N_CONFIG_FILES.split(',');
-		if (!inTest) {
-			console.log(`\nLoading configuration overwrites from:\n - ${configFiles.join('\n - ')}\n`);
-		}
-
+		Logger.debug('Loading config overwrites', configFiles);
 		config.loadFile(configFiles);
 	}
+
+	// Overwrite config from files defined in "_FILE" environment variables
+	const overwrites = Object.entries(process.env).reduce<Record<string, string>>(
+		(acc, [envName, fileName]) => {
+			if (envName.endsWith('_FILE') && fileName) {
+				const key = envName.replace(/_FILE$/, '');
+				// @ts-ignore
+				if (key in config._env) {
+					let value: string;
+					try {
+						value = readFileSync(fileName, 'utf8').trim();
+					} catch (error) {
+						// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+						if (error.code === 'ENOENT') {
+							throw new Error(`The file "${fileName}" could not be found.`);
+						}
+						throw error;
+					}
+					Logger.debug('Loading config overwrite', { fileName });
+					acc[key] = value;
+				}
+			}
+			return acc;
+		},
+		{},
+	);
+	config.load(overwrites);
 }
 
 config.validate({

packages/cli/src/databases/config.ts

@@ -10,7 +10,6 @@ import { postgresMigrations } from './migrations/postgresdb';
 import { sqliteMigrations } from './migrations/sqlite';
 import type { DatabaseType } from '@/Interfaces';
 import config from '@/config';
-import { getConfigValue } from '@/GenericHelpers';
 
 const entitiesDir = path.resolve(__dirname, 'entities');
 
@@ -43,12 +42,12 @@ const getDBConnectionOptions = (dbType: DatabaseType) => {
 	};
 };
 
-export const getOptionOverrides = async (dbType: 'postgresdb' | 'mysqldb') => ({
-	database: (await getConfigValue(`database.${dbType}.database`)) as string,
-	host: (await getConfigValue(`database.${dbType}.host`)) as string,
-	port: (await getConfigValue(`database.${dbType}.port`)) as number,
-	username: (await getConfigValue(`database.${dbType}.user`)) as string,
-	password: (await getConfigValue(`database.${dbType}.password`)) as string,
+export const getOptionOverrides = (dbType: 'postgresdb' | 'mysqldb') => ({
+	database: config.getEnv(`database.${dbType}.database`),
+	host: config.getEnv(`database.${dbType}.host`),
+	port: config.getEnv(`database.${dbType}.port`),
+	username: config.getEnv(`database.${dbType}.user`),
+	password: config.getEnv(`database.${dbType}.password`),
 });
 
 export const getSqliteConnectionOptions = (): SqliteConnectionOptions => ({

packages/cli/src/executions/executions.service.ts

@@ -12,7 +12,6 @@ import config from '@/config';
 import type { User } from '@db/entities/User';
 import type { ExecutionEntity } from '@db/entities/ExecutionEntity';
 import type {
-	DatabaseType,
 	IExecutionFlattedResponse,
 	IExecutionResponse,
 	IExecutionsListResponse,
@@ -70,7 +69,7 @@ export class ExecutionsService {
 		countFilter: IDataObject,
 		user: User,
 	): Promise<{ count: number; estimated: boolean }> {
-		const dbType = (await GenericHelpers.getConfigValue('database.type')) as DatabaseType;
+		const dbType = config.getEnv('database.type');
 		const filteredFields = Object.keys(countFilter).filter((field) => field !== 'id');
 
 		// For databases other than Postgres, do a regular count