refactor(core): Reduce code duplication in DB config (no-changelog) (#8679)

packages/cli/package.json

@@ -37,8 +37,7 @@
     "test:sqlite": "N8N_LOG_LEVEL=silent DB_TYPE=sqlite jest",
     "test:postgres": "N8N_LOG_LEVEL=silent DB_TYPE=postgresdb DB_POSTGRESDB_SCHEMA=alt_schema DB_TABLE_PREFIX=test_ jest --no-coverage",
     "test:mysql": "N8N_LOG_LEVEL=silent DB_TYPE=mysqldb DB_TABLE_PREFIX=test_ jest --no-coverage",
-    "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\"",
+    "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\""
-    "typeorm": "node ../../node_modules/typeorm/cli.js"
   },
   "bin": {
     "n8n": "./bin/n8n"

packages/cli/src/Db.ts

@@ -1,27 +1,14 @@
 /* eslint-disable @typescript-eslint/restrict-template-expressions */
 import { Container } from 'typedi';
-import type {
+import type { EntityManager } from '@n8n/typeorm';
-	DataSourceOptions as ConnectionOptions,
-	EntityManager,
-	LoggerOptions,
-} from '@n8n/typeorm';
 import { DataSource as Connection } from '@n8n/typeorm';
-import type { TlsOptions } from 'tls';
+import { ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';
-import { ApplicationError, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';
 
 import config from '@/config';
-
-import { entities } from '@db/entities';
-import {
-	getMariaDBConnectionOptions,
-	getMysqlConnectionOptions,
-	getOptionOverrides,
-	getPostgresConnectionOptions,
-	getSqliteConnectionOptions,
-} from '@db/config';
 import { inTest } from '@/constants';
 import { wrapMigration } from '@db/utils/migrationHelpers';
-import type { DatabaseType, Migration } from '@db/types';
+import type { Migration } from '@db/types';
+import { getConnectionOptions } from '@db/config';
 
 let connection: Connection;
 
@@ -61,46 +48,6 @@ export async function transaction<T>(fn: (entityManager: EntityManager) => Promi
 	return await connection.transaction(fn);
 }
 
-export function getConnectionOptions(dbType: DatabaseType): ConnectionOptions {
-	switch (dbType) {
-		case 'postgresdb':
-			const sslCa = config.getEnv('database.postgresdb.ssl.ca');
-			const sslCert = config.getEnv('database.postgresdb.ssl.cert');
-			const sslKey = config.getEnv('database.postgresdb.ssl.key');
-			const sslRejectUnauthorized = config.getEnv('database.postgresdb.ssl.rejectUnauthorized');
-
-			let ssl: TlsOptions | boolean = config.getEnv('database.postgresdb.ssl.enabled');
-			if (sslCa !== '' || sslCert !== '' || sslKey !== '' || !sslRejectUnauthorized) {
-				ssl = {
-					ca: sslCa || undefined,
-					cert: sslCert || undefined,
-					key: sslKey || undefined,
-					rejectUnauthorized: sslRejectUnauthorized,
-				};
-			}
-
-			return {
-				...getPostgresConnectionOptions(),
-				...getOptionOverrides('postgresdb'),
-				ssl,
-			};
-
-		case 'mariadb':
-		case 'mysqldb':
-			return {
-				...(dbType === 'mysqldb' ? getMysqlConnectionOptions() : getMariaDBConnectionOptions()),
-				...getOptionOverrides('mysqldb'),
-				timezone: 'Z', // set UTC as default
-			};
-
-		case 'sqlite':
-			return getSqliteConnectionOptions();
-
-		default:
-			throw new ApplicationError('Database type currently not supported', { extra: { dbType } });
-	}
-}
-
 export async function setSchema(conn: Connection) {
 	const schema = config.getEnv('database.postgresdb.schema');
 	const searchPath = ['public'];
@@ -111,33 +58,11 @@ export async function setSchema(conn: Connection) {
 	await conn.query(`SET search_path TO ${searchPath.join(',')};`);
 }
 
-export async function init(testConnectionOptions?: ConnectionOptions): Promise<void> {
+export async function init(): Promise<void> {
 	if (connectionState.connected) return;
 
 	const dbType = config.getEnv('database.type');
-	const connectionOptions = testConnectionOptions ?? getConnectionOptions(dbType);
+	const connectionOptions = getConnectionOptions();
-
-	let loggingOption: LoggerOptions = config.getEnv('database.logging.enabled');
-
-	if (loggingOption) {
-		const optionsString = config.getEnv('database.logging.options').replace(/\s+/g, '');
-
-		if (optionsString === 'all') {
-			loggingOption = optionsString;
-		} else {
-			loggingOption = optionsString.split(',') as LoggerOptions;
-		}
-	}
-
-	const maxQueryExecutionTime = config.getEnv('database.logging.maxQueryExecutionTime');
-
-	Object.assign(connectionOptions, {
-		entities: Object.values(entities),
-		synchronize: false,
-		logging: loggingOption,
-		maxQueryExecutionTime,
-		migrationsRun: false,
-	});
 
 	connection = new Connection(connectionOptions);
 	Container.set(Connection, connection);

packages/cli/src/commands/db/revert.ts

@@ -3,7 +3,8 @@ import type { DataSourceOptions as ConnectionOptions } from '@n8n/typeorm';
 import { DataSource as Connection } from '@n8n/typeorm';
 import { Container } from 'typedi';
 import { Logger } from '@/Logger';
-import { getConnectionOptions, setSchema } from '@/Db';
+import { setSchema } from '@/Db';
+import { getConnectionOptions } from '@db/config';
 import type { Migration } from '@db/types';
 import { wrapMigration } from '@db/utils/migrationHelpers';
 import config from '@/config';
@@ -28,7 +29,7 @@ export class DbRevertMigrationCommand extends Command {
 	async run() {
 		const dbType = config.getEnv('database.type');
 		const connectionOptions: ConnectionOptions = {
-			...getConnectionOptions(dbType),
+			...getConnectionOptions(),
 			subscribers: [],
 			synchronize: false,
 			migrationsRun: false,

packages/cli/src/databases/config.ts

@@ -1,45 +1,41 @@
 import path from 'path';
 import { Container } from 'typedi';
+import type { TlsOptions } from 'tls';
+import type { DataSourceOptions, LoggerOptions } from '@n8n/typeorm';
 import type { SqliteConnectionOptions } from '@n8n/typeorm/driver/sqlite/SqliteConnectionOptions';
 import type { PostgresConnectionOptions } from '@n8n/typeorm/driver/postgres/PostgresConnectionOptions';
 import type { MysqlConnectionOptions } from '@n8n/typeorm/driver/mysql/MysqlConnectionOptions';
 import { InstanceSettings } from 'n8n-core';
+import { ApplicationError } from 'n8n-workflow';
 
+import config from '@/config';
 import { entities } from './entities';
 import { mysqlMigrations } from './migrations/mysqldb';
 import { postgresMigrations } from './migrations/postgresdb';
 import { sqliteMigrations } from './migrations/sqlite';
-import type { DatabaseType } from '@db/types';
-import config from '@/config';
 
-const entitiesDir = path.resolve(__dirname, 'entities');
+const getCommonOptions = () => {
-
-const getDBConnectionOptions = (dbType: DatabaseType) => {
 	const entityPrefix = config.getEnv('database.tablePrefix');
-	const migrationsDir = path.resolve(__dirname, 'migrations', dbType);
+	const maxQueryExecutionTime = config.getEnv('database.logging.maxQueryExecutionTime');
-	const configDBType = dbType === 'mariadb' ? 'mysqldb' : dbType;
+
-	const connectionDetails =
+	let loggingOption: LoggerOptions = config.getEnv('database.logging.enabled');
-		configDBType === 'sqlite'
+	if (loggingOption) {
-			? {
+		const optionsString = config.getEnv('database.logging.options').replace(/\s+/g, '');
-					database: path.resolve(
+
-						Container.get(InstanceSettings).n8nFolder,
+		if (optionsString === 'all') {
-						config.getEnv('database.sqlite.database'),
+			loggingOption = optionsString;
-					),
+		} else {
-					enableWAL: config.getEnv('database.sqlite.enableWAL'),
+			loggingOption = optionsString.split(',') as LoggerOptions;
+		}
 	}
-			: {
-					database: config.getEnv(`database.${configDBType}.database`),
-					username: config.getEnv(`database.${configDBType}.user`),
-					password: config.getEnv(`database.${configDBType}.password`),
-					host: config.getEnv(`database.${configDBType}.host`),
-					port: config.getEnv(`database.${configDBType}.port`),
-			  };
 	return {
 		entityPrefix,
 		entities: Object.values(entities),
 		migrationsTableName: `${entityPrefix}migrations`,
-		cli: { entitiesDir, migrationsDir },
+		migrationsRun: false,
-		...connectionDetails,
+		synchronize: false,
+		maxQueryExecutionTime,
+		logging: loggingOption,
 	};
 };
 
@@ -51,28 +47,63 @@ export const getOptionOverrides = (dbType: 'postgresdb' | 'mysqldb') => ({
 	password: config.getEnv(`database.${dbType}.password`),
 });
 
-export const getSqliteConnectionOptions = (): SqliteConnectionOptions => ({
+const getSqliteConnectionOptions = (): SqliteConnectionOptions => ({
 	type: 'sqlite',
-	...getDBConnectionOptions('sqlite'),
+	...getCommonOptions(),
+	database: path.resolve(
+		Container.get(InstanceSettings).n8nFolder,
+		config.getEnv('database.sqlite.database'),
+	),
+	enableWAL: config.getEnv('database.sqlite.enableWAL'),
 	migrations: sqliteMigrations,
 });
 
-export const getPostgresConnectionOptions = (): PostgresConnectionOptions => ({
+const getPostgresConnectionOptions = (): PostgresConnectionOptions => {
+	const sslCa = config.getEnv('database.postgresdb.ssl.ca');
+	const sslCert = config.getEnv('database.postgresdb.ssl.cert');
+	const sslKey = config.getEnv('database.postgresdb.ssl.key');
+	const sslRejectUnauthorized = config.getEnv('database.postgresdb.ssl.rejectUnauthorized');
+
+	let ssl: TlsOptions | boolean = config.getEnv('database.postgresdb.ssl.enabled');
+	if (sslCa !== '' || sslCert !== '' || sslKey !== '' || !sslRejectUnauthorized) {
+		ssl = {
+			ca: sslCa || undefined,
+			cert: sslCert || undefined,
+			key: sslKey || undefined,
+			rejectUnauthorized: sslRejectUnauthorized,
+		};
+	}
+
+	return {
 		type: 'postgres',
-	...getDBConnectionOptions('postgresdb'),
+		...getCommonOptions(),
+		...getOptionOverrides('postgresdb'),
 		schema: config.getEnv('database.postgresdb.schema'),
 		poolSize: config.getEnv('database.postgresdb.poolSize'),
 		migrations: postgresMigrations,
+		ssl,
+	};
+};
+
+const getMysqlConnectionOptions = (dbType: 'mariadb' | 'mysqldb'): MysqlConnectionOptions => ({
+	type: dbType === 'mysqldb' ? 'mysql' : 'mariadb',
+	...getCommonOptions(),
+	...getOptionOverrides('mysqldb'),
+	migrations: mysqlMigrations,
+	timezone: 'Z', // set UTC as default
 });
 
-export const getMysqlConnectionOptions = (): MysqlConnectionOptions => ({
+export function getConnectionOptions(): DataSourceOptions {
-	type: 'mysql',
+	const dbType = config.getEnv('database.type');
-	...getDBConnectionOptions('mysqldb'),
+	switch (dbType) {
-	migrations: mysqlMigrations,
+		case 'sqlite':
-});
+			return getSqliteConnectionOptions();
-
+		case 'postgresdb':
-export const getMariaDBConnectionOptions = (): MysqlConnectionOptions => ({
+			return getPostgresConnectionOptions();
-	type: 'mariadb',
+		case 'mariadb':
-	...getDBConnectionOptions('mysqldb'),
+		case 'mysqldb':
-	migrations: mysqlMigrations,
+			return getMysqlConnectionOptions(dbType);
-});
+		default:
+			throw new ApplicationError('Database type currently not supported', { extra: { dbType } });
+	}
+}

packages/cli/src/databases/ormconfig.ts

@@ -1,13 +0,0 @@
-import {
-	getMariaDBConnectionOptions,
-	getMysqlConnectionOptions,
-	getPostgresConnectionOptions,
-	getSqliteConnectionOptions,
-} from './config';
-
-export default [
-	getSqliteConnectionOptions(),
-	getPostgresConnectionOptions(),
-	getMysqlConnectionOptions(),
-	getMariaDBConnectionOptions(),
-];

packages/cli/test/integration/shared/constants.ts

@@ -33,8 +33,3 @@ export const COMMUNITY_NODE_VERSION = {
 	CURRENT: 1,
 	UPDATED: 2,
 };
-
-/**
- * Timeout (in milliseconds) to account for DB being slow to initialize.
- */
-export const DB_INITIALIZATION_TIMEOUT = 30_000;

packages/cli/test/integration/shared/testDb.ts

@@ -1,82 +1,40 @@
-import type { DataSourceOptions as ConnectionOptions, Repository } from '@n8n/typeorm';
+import type { DataSourceOptions, Repository } from '@n8n/typeorm';
 import { DataSource as Connection } from '@n8n/typeorm';
 import { Container } from 'typedi';
 import type { Class } from 'n8n-core';
 
 import config from '@/config';
 import * as Db from '@/Db';
-import { entities } from '@db/entities';
+import { getOptionOverrides } from '@db/config';
-import { mysqlMigrations } from '@db/migrations/mysqldb';
-import { postgresMigrations } from '@db/migrations/postgresdb';
-import { sqliteMigrations } from '@db/migrations/sqlite';
 
-import { DB_INITIALIZATION_TIMEOUT } from './constants';
 import { randomString } from './random';
-import type { PostgresSchemaSection } from './types';
-
-export type TestDBType = 'postgres' | 'mysql';
 
 export const testDbPrefix = 'n8n_test_';
 
-export function getPostgresSchemaSection(
-	schema = config.getSchema(),
-): PostgresSchemaSection | null {
-	for (const [key, value] of Object.entries(schema)) {
-		if (key === 'postgresdb') {
-			return value._cvtProperties;
-		}
-	}
-	return null;
-}
-
 /**
  * Initialize one test DB per suite run, with bootstrap connection if needed.
  */
 export async function init() {
-	jest.setTimeout(DB_INITIALIZATION_TIMEOUT);
 	const dbType = config.getEnv('database.type');
 	const testDbName = `${testDbPrefix}${randomString(6, 10)}_${Date.now()}`;
 
-	if (dbType === 'sqlite') {
+	if (dbType === 'postgresdb') {
-		// no bootstrap connection required
+		const bootstrapPostgres = await new Connection(
-		await Db.init(getSqliteOptions({ name: testDbName }));
+			getBootstrapDBOptions('postgresdb'),
-	} else if (dbType === 'postgresdb') {
+		).initialize();
-		let bootstrapPostgres;
-		const pgOptions = getBootstrapDBOptions('postgres');
-
-		try {
-			bootstrapPostgres = await new Connection(pgOptions).initialize();
-		} catch (error) {
-			const pgConfig = getPostgresSchemaSection();
-
-			if (!pgConfig) throw new Error("Failed to find config schema section for 'postgresdb'");
-
-			const message = [
-				"ERROR: Failed to connect to Postgres default DB 'postgres'",
-				'Please review your Postgres connection options:',
-				`host: ${pgOptions.host} | port: ${pgOptions.port} | schema: ${pgOptions.schema} | username: ${pgOptions.username} | password: ${pgOptions.password}`,
-				'Fix by setting correct values via environment variables:',
-				`${pgConfig.host.env} | ${pgConfig.port.env} | ${pgConfig.schema.env} | ${pgConfig.user.env} | ${pgConfig.password.env}`,
-				'Otherwise, make sure your Postgres server is running.',
-			].join('\n');
-
-			console.error(message);
-
-			process.exit(1);
-		}
-
 		await bootstrapPostgres.query(`CREATE DATABASE ${testDbName}`);
 		await bootstrapPostgres.destroy();
 
-		await Db.init(getDBOptions('postgres', testDbName));
+		config.set('database.postgresdb.database', testDbName);
 	} else if (dbType === 'mysqldb' || dbType === 'mariadb') {
-		const bootstrapMysql = await new Connection(getBootstrapDBOptions('mysql')).initialize();
+		const bootstrapMysql = await new Connection(getBootstrapDBOptions('mysqldb')).initialize();
 		await bootstrapMysql.query(`CREATE DATABASE ${testDbName} DEFAULT CHARACTER SET utf8mb4`);
 		await bootstrapMysql.destroy();
 
-		await Db.init(getDBOptions('mysql', testDbName));
+		config.set('database.mysqldb.database', testDbName);
 	}
 
+	await Db.init();
 	await Db.migrate();
 }
 
@@ -124,57 +82,16 @@ export async function truncate(names: Array<(typeof repositories)[number]>) {
 	}
 }
 
-// ----------------------------------
-//        connection options
-// ----------------------------------
-
-/**
- * Generate options for an in-memory sqlite database connection,
- * one per test suite run.
- */
-const getSqliteOptions = ({ name }: { name: string }): ConnectionOptions => {
-	return {
-		name,
-		type: 'sqlite',
-		database: ':memory:',
-		entityPrefix: config.getEnv('database.tablePrefix'),
-		dropSchema: true,
-		migrations: sqliteMigrations,
-		migrationsTableName: 'migrations',
-		migrationsRun: false,
-		enableWAL: config.getEnv('database.sqlite.enableWAL'),
-	};
-};
-
-const baseOptions = (type: TestDBType) => ({
-	host: config.getEnv(`database.${type}db.host`),
-	port: config.getEnv(`database.${type}db.port`),
-	username: config.getEnv(`database.${type}db.user`),
-	password: config.getEnv(`database.${type}db.password`),
-	entityPrefix: config.getEnv('database.tablePrefix'),
-	schema: type === 'postgres' ? config.getEnv('database.postgresdb.schema') : undefined,
-});
-
 /**
  * Generate options for a bootstrap DB connection, to create and drop test databases.
  */
-export const getBootstrapDBOptions = (type: TestDBType) => ({
+export const getBootstrapDBOptions = (dbType: 'postgresdb' | 'mysqldb'): DataSourceOptions => {
+	const type = dbType === 'postgresdb' ? 'postgres' : 'mysql';
+	return {
 		type,
-	name: type,
+		...getOptionOverrides(dbType),
 		database: type,
-	...baseOptions(type),
+		entityPrefix: config.getEnv('database.tablePrefix'),
-});
+		schema: dbType === 'postgresdb' ? config.getEnv('database.postgresdb.schema') : undefined,
-
+	};
-const getDBOptions = (type: TestDBType, name: string) => ({
+};
-	type,
-	name,
-	database: name,
-	...baseOptions(type),
-	dropSchema: true,
-	migrations: type === 'postgres' ? postgresMigrations : mysqlMigrations,
-	migrationsRun: false,
-	migrationsTableName: 'migrations',
-	entities: Object.values(entities),
-	synchronize: false,
-	logging: false,
-});

packages/cli/test/integration/shared/types.ts

@@ -61,7 +61,3 @@ export type SaveCredentialFunction = (
 	credentialPayload: CredentialPayload,
 	{ user }: { user: User },
 ) => Promise<CredentialsEntity & ICredentialsDb>;
-
-export type PostgresSchemaSection = {
-	[K in 'host' | 'port' | 'schema' | 'user' | 'password']: { env: string };
-};

packages/cli/test/teardown.ts

@@ -4,14 +4,14 @@ import config from '@/config';
 import { getBootstrapDBOptions, testDbPrefix } from './integration/shared/testDb';
 
 export default async () => {
-	const dbType = config.getEnv('database.type').replace(/db$/, '');
+	const dbType = config.getEnv('database.type');
-	if (dbType !== 'postgres' && dbType !== 'mysql') return;
+	if (dbType !== 'postgresdb' && dbType !== 'mysqldb') return;
 
 	const connection = new Connection(getBootstrapDBOptions(dbType));
 	await connection.initialize();
 
 	const query =
-		dbType === 'postgres' ? 'SELECT datname as "Database" FROM pg_database' : 'SHOW DATABASES';
+		dbType === 'postgresdb' ? 'SELECT datname as "Database" FROM pg_database' : 'SHOW DATABASES';
 	const results: Array<{ Database: string }> = await connection.query(query);
 	const databases = results
 		.filter(({ Database: dbName }) => dbName.startsWith(testDbPrefix))