mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2024-11-09 22:24:05 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor(core): Reduce code duplication in DB config (no-changelog) (#8679)

Browse source
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™ 2024-02-20 14:28:53 +01:00 committed by GitHub
parent 0e36aeb421
commit b6c8a0c413
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 108 additions and 257 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
packages/cli/package.json
View file

@ -37,8 +37,7 @@
"test:sqlite": "N8N_LOG_LEVEL=silent DB_TYPE=sqlite jest",
"test:postgres": "N8N_LOG_LEVEL=silent DB_TYPE=postgresdb DB_POSTGRESDB_SCHEMA=alt_schema DB_TABLE_PREFIX=test_ jest --no-coverage",
"test:mysql": "N8N_LOG_LEVEL=silent DB_TYPE=mysqldb DB_TABLE_PREFIX=test_ jest --no-coverage",
"watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\"",
"typeorm": "node ../../node_modules/typeorm/cli.js"
"watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\""
},
"bin": {
"n8n": "./bin/n8n"

87
packages/cli/src/Db.ts
View file

@ -1,27 +1,14 @@
/* eslint-disable @typescript-eslint/restrict-template-expressions */
import { Container } from 'typedi';
import type {
DataSourceOptions as ConnectionOptions,
EntityManager,
LoggerOptions,
} from '@n8n/typeorm';
import type { EntityManager } from '@n8n/typeorm';
import { DataSource as Connection } from '@n8n/typeorm';
import type { TlsOptions } from 'tls';
import { ApplicationError, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';
import { ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';
import config from '@/config';
import { entities } from '@db/entities';
import {
getMariaDBConnectionOptions,
getMysqlConnectionOptions,
getOptionOverrides,
getPostgresConnectionOptions,
getSqliteConnectionOptions,
} from '@db/config';
import { inTest } from '@/constants';
import { wrapMigration } from '@db/utils/migrationHelpers';
import type { DatabaseType, Migration } from '@db/types';
import type { Migration } from '@db/types';
import { getConnectionOptions } from '@db/config';
let connection: Connection;
@ -61,46 +48,6 @@ export async function transaction<T>(fn: (entityManager: EntityManager) => Promi
return await connection.transaction(fn);
}
export function getConnectionOptions(dbType: DatabaseType): ConnectionOptions {
switch (dbType) {
case 'postgresdb':
const sslCa = config.getEnv('database.postgresdb.ssl.ca');
const sslCert = config.getEnv('database.postgresdb.ssl.cert');
const sslKey = config.getEnv('database.postgresdb.ssl.key');
const sslRejectUnauthorized = config.getEnv('database.postgresdb.ssl.rejectUnauthorized');
let ssl: TlsOptions | boolean = config.getEnv('database.postgresdb.ssl.enabled');
if (sslCa !== '' || sslCert !== '' || sslKey !== '' || !sslRejectUnauthorized) {
ssl = {
ca: sslCa || undefined,
cert: sslCert || undefined,
key: sslKey || undefined,
rejectUnauthorized: sslRejectUnauthorized,
};
}
return {
...getPostgresConnectionOptions(),
...getOptionOverrides('postgresdb'),
ssl,
};
case 'mariadb':
case 'mysqldb':
return {
...(dbType === 'mysqldb' ? getMysqlConnectionOptions() : getMariaDBConnectionOptions()),
...getOptionOverrides('mysqldb'),
timezone: 'Z', // set UTC as default
};
case 'sqlite':
return getSqliteConnectionOptions();
default:
throw new ApplicationError('Database type currently not supported', { extra: { dbType } });
}
}
export async function setSchema(conn: Connection) {
const schema = config.getEnv('database.postgresdb.schema');
const searchPath = ['public'];
@ -111,33 +58,11 @@ export async function setSchema(conn: Connection) {
await conn.query(`SET search_path TO ${searchPath.join(',')};`);
}
export async function init(testConnectionOptions?: ConnectionOptions): Promise<void> {
export async function init(): Promise<void> {
if (connectionState.connected) return;
const dbType = config.getEnv('database.type');
const connectionOptions = testConnectionOptions ?? getConnectionOptions(dbType);
let loggingOption: LoggerOptions = config.getEnv('database.logging.enabled');
if (loggingOption) {
const optionsString = config.getEnv('database.logging.options').replace(/\s+/g, '');
if (optionsString === 'all') {
loggingOption = optionsString;
} else {
loggingOption = optionsString.split(',') as LoggerOptions;
}
}
const maxQueryExecutionTime = config.getEnv('database.logging.maxQueryExecutionTime');
Object.assign(connectionOptions, {
entities: Object.values(entities),
synchronize: false,
logging: loggingOption,
maxQueryExecutionTime,
migrationsRun: false,
});
const connectionOptions = getConnectionOptions();
connection = new Connection(connectionOptions);
Container.set(Connection, connection);

5
packages/cli/src/commands/db/revert.ts
View file

@ -3,7 +3,8 @@ import type { DataSourceOptions as ConnectionOptions } from '@n8n/typeorm';
import { DataSource as Connection } from '@n8n/typeorm';
import { Container } from 'typedi';
import { Logger } from '@/Logger';
import { getConnectionOptions, setSchema } from '@/Db';
import { setSchema } from '@/Db';
import { getConnectionOptions } from '@db/config';
import type { Migration } from '@db/types';
import { wrapMigration } from '@db/utils/migrationHelpers';
import config from '@/config';
@ -28,7 +29,7 @@ export class DbRevertMigrationCommand extends Command {
async run() {
const dbType = config.getEnv('database.type');
const connectionOptions: ConnectionOptions = {
...getConnectionOptions(dbType),
...getConnectionOptions(),
subscribers: [],
synchronize: false,
migrationsRun: false,

119
packages/cli/src/databases/config.ts
View file

@ -1,45 +1,41 @@
import path from 'path';
import { Container } from 'typedi';
import type { TlsOptions } from 'tls';
import type { DataSourceOptions, LoggerOptions } from '@n8n/typeorm';
import type { SqliteConnectionOptions } from '@n8n/typeorm/driver/sqlite/SqliteConnectionOptions';
import type { PostgresConnectionOptions } from '@n8n/typeorm/driver/postgres/PostgresConnectionOptions';
import type { MysqlConnectionOptions } from '@n8n/typeorm/driver/mysql/MysqlConnectionOptions';
import { InstanceSettings } from 'n8n-core';
import { ApplicationError } from 'n8n-workflow';
import config from '@/config';
import { entities } from './entities';
import { mysqlMigrations } from './migrations/mysqldb';
import { postgresMigrations } from './migrations/postgresdb';
import { sqliteMigrations } from './migrations/sqlite';
import type { DatabaseType } from '@db/types';
import config from '@/config';
const entitiesDir = path.resolve(__dirname, 'entities');
const getDBConnectionOptions = (dbType: DatabaseType) => {
const getCommonOptions = () => {
const entityPrefix = config.getEnv('database.tablePrefix');
const migrationsDir = path.resolve(__dirname, 'migrations', dbType);
const configDBType = dbType === 'mariadb' ? 'mysqldb' : dbType;
const connectionDetails =
configDBType === 'sqlite'
? {
database: path.resolve(
Container.get(InstanceSettings).n8nFolder,
config.getEnv('database.sqlite.database'),
),
enableWAL: config.getEnv('database.sqlite.enableWAL'),
}
: {
database: config.getEnv(`database.${configDBType}.database`),
username: config.getEnv(`database.${configDBType}.user`),
password: config.getEnv(`database.${configDBType}.password`),
host: config.getEnv(`database.${configDBType}.host`),
port: config.getEnv(`database.${configDBType}.port`),
};
const maxQueryExecutionTime = config.getEnv('database.logging.maxQueryExecutionTime');
let loggingOption: LoggerOptions = config.getEnv('database.logging.enabled');
if (loggingOption) {
const optionsString = config.getEnv('database.logging.options').replace(/\s+/g, '');
if (optionsString === 'all') {
loggingOption = optionsString;
} else {
loggingOption = optionsString.split(',') as LoggerOptions;
}
}
return {
entityPrefix,
entities: Object.values(entities),
migrationsTableName: `${entityPrefix}migrations`,
cli: { entitiesDir, migrationsDir },
...connectionDetails,
migrationsRun: false,
synchronize: false,
maxQueryExecutionTime,
logging: loggingOption,
};
};
@ -51,28 +47,63 @@ export const getOptionOverrides = (dbType: 'postgresdb' | 'mysqldb') => ({
password: config.getEnv(`database.${dbType}.password`),
});
export const getSqliteConnectionOptions = (): SqliteConnectionOptions => ({
const getSqliteConnectionOptions = (): SqliteConnectionOptions => ({
type: 'sqlite',
...getDBConnectionOptions('sqlite'),
...getCommonOptions(),
database: path.resolve(
Container.get(InstanceSettings).n8nFolder,
config.getEnv('database.sqlite.database'),
),
enableWAL: config.getEnv('database.sqlite.enableWAL'),
migrations: sqliteMigrations,
});
export const getPostgresConnectionOptions = (): PostgresConnectionOptions => ({
type: 'postgres',
...getDBConnectionOptions('postgresdb'),
schema: config.getEnv('database.postgresdb.schema'),
poolSize: config.getEnv('database.postgresdb.poolSize'),
migrations: postgresMigrations,
const getPostgresConnectionOptions = (): PostgresConnectionOptions => {
const sslCa = config.getEnv('database.postgresdb.ssl.ca');
const sslCert = config.getEnv('database.postgresdb.ssl.cert');
const sslKey = config.getEnv('database.postgresdb.ssl.key');
const sslRejectUnauthorized = config.getEnv('database.postgresdb.ssl.rejectUnauthorized');
let ssl: TlsOptions | boolean = config.getEnv('database.postgresdb.ssl.enabled');
if (sslCa !== '' || sslCert !== '' || sslKey !== '' || !sslRejectUnauthorized) {
ssl = {
ca: sslCa || undefined,
cert: sslCert || undefined,
key: sslKey || undefined,
rejectUnauthorized: sslRejectUnauthorized,
};
}
return {
type: 'postgres',
...getCommonOptions(),
...getOptionOverrides('postgresdb'),
schema: config.getEnv('database.postgresdb.schema'),
poolSize: config.getEnv('database.postgresdb.poolSize'),
migrations: postgresMigrations,
ssl,
};
};
const getMysqlConnectionOptions = (dbType: 'mariadb' | 'mysqldb'): MysqlConnectionOptions => ({
type: dbType === 'mysqldb' ? 'mysql' : 'mariadb',
...getCommonOptions(),
...getOptionOverrides('mysqldb'),
migrations: mysqlMigrations,
timezone: 'Z', // set UTC as default
});
export const getMysqlConnectionOptions = (): MysqlConnectionOptions => ({
type: 'mysql',
...getDBConnectionOptions('mysqldb'),
migrations: mysqlMigrations,
});
export const getMariaDBConnectionOptions = (): MysqlConnectionOptions => ({
type: 'mariadb',
...getDBConnectionOptions('mysqldb'),
migrations: mysqlMigrations,
});
export function getConnectionOptions(): DataSourceOptions {
const dbType = config.getEnv('database.type');
switch (dbType) {
case 'sqlite':
return getSqliteConnectionOptions();
case 'postgresdb':
return getPostgresConnectionOptions();
case 'mariadb':
case 'mysqldb':
return getMysqlConnectionOptions(dbType);
default:
throw new ApplicationError('Database type currently not supported', { extra: { dbType } });
}
}

13
packages/cli/src/databases/ormconfig.ts
View file

@ -1,13 +0,0 @@
import {
getMariaDBConnectionOptions,
getMysqlConnectionOptions,
getPostgresConnectionOptions,
getSqliteConnectionOptions,
} from './config';
export default [
getSqliteConnectionOptions(),
getPostgresConnectionOptions(),
getMysqlConnectionOptions(),
getMariaDBConnectionOptions(),
];

5
packages/cli/test/integration/shared/constants.ts
View file

@ -33,8 +33,3 @@ export const COMMUNITY_NODE_VERSION = {
CURRENT: 1,
UPDATED: 2,
};
/**
* Timeout (in milliseconds) to account for DB being slow to initialize.
*/
export const DB_INITIALIZATION_TIMEOUT = 30_000;

123
packages/cli/test/integration/shared/testDb.ts
View file

@ -1,82 +1,40 @@
import type { DataSourceOptions as ConnectionOptions, Repository } from '@n8n/typeorm';
import type { DataSourceOptions, Repository } from '@n8n/typeorm';
import { DataSource as Connection } from '@n8n/typeorm';
import { Container } from 'typedi';
import type { Class } from 'n8n-core';
import config from '@/config';
import * as Db from '@/Db';
import { entities } from '@db/entities';
import { mysqlMigrations } from '@db/migrations/mysqldb';
import { postgresMigrations } from '@db/migrations/postgresdb';
import { sqliteMigrations } from '@db/migrations/sqlite';
import { getOptionOverrides } from '@db/config';
import { DB_INITIALIZATION_TIMEOUT } from './constants';
import { randomString } from './random';
import type { PostgresSchemaSection } from './types';
export type TestDBType = 'postgres' | 'mysql';
export const testDbPrefix = 'n8n_test_';
export function getPostgresSchemaSection(
schema = config.getSchema(),
): PostgresSchemaSection | null {
for (const [key, value] of Object.entries(schema)) {
if (key === 'postgresdb') {
return value._cvtProperties;
}
}
return null;
}
/**
* Initialize one test DB per suite run, with bootstrap connection if needed.
*/
export async function init() {
jest.setTimeout(DB_INITIALIZATION_TIMEOUT);
const dbType = config.getEnv('database.type');
const testDbName = `${testDbPrefix}${randomString(6, 10)}_${Date.now()}`;
if (dbType === 'sqlite') {
// no bootstrap connection required
await Db.init(getSqliteOptions({ name: testDbName }));
} else if (dbType === 'postgresdb') {
let bootstrapPostgres;
const pgOptions = getBootstrapDBOptions('postgres');
try {
bootstrapPostgres = await new Connection(pgOptions).initialize();
} catch (error) {
const pgConfig = getPostgresSchemaSection();
if (!pgConfig) throw new Error("Failed to find config schema section for 'postgresdb'");
const message = [
"ERROR: Failed to connect to Postgres default DB 'postgres'",
'Please review your Postgres connection options:',
`host: ${pgOptions.host} | port: ${pgOptions.port} | schema: ${pgOptions.schema} | username: ${pgOptions.username} | password: ${pgOptions.password}`,
'Fix by setting correct values via environment variables:',
`${pgConfig.host.env} | ${pgConfig.port.env} | ${pgConfig.schema.env} | ${pgConfig.user.env} | ${pgConfig.password.env}`,
'Otherwise, make sure your Postgres server is running.',
].join('\n');
console.error(message);
process.exit(1);
}
if (dbType === 'postgresdb') {
const bootstrapPostgres = await new Connection(
getBootstrapDBOptions('postgresdb'),
).initialize();
await bootstrapPostgres.query(`CREATE DATABASE ${testDbName}`);
await bootstrapPostgres.destroy();
await Db.init(getDBOptions('postgres', testDbName));
config.set('database.postgresdb.database', testDbName);
} else if (dbType === 'mysqldb' || dbType === 'mariadb') {
const bootstrapMysql = await new Connection(getBootstrapDBOptions('mysql')).initialize();
const bootstrapMysql = await new Connection(getBootstrapDBOptions('mysqldb')).initialize();
await bootstrapMysql.query(`CREATE DATABASE ${testDbName} DEFAULT CHARACTER SET utf8mb4`);
await bootstrapMysql.destroy();
await Db.init(getDBOptions('mysql', testDbName));
config.set('database.mysqldb.database', testDbName);
}
await Db.init();
await Db.migrate();
}
@ -124,57 +82,16 @@ export async function truncate(names: Array<(typeof repositories)[number]>) {
}
}
// ----------------------------------
// connection options
// ----------------------------------
/**
* Generate options for an in-memory sqlite database connection,
* one per test suite run.
*/
const getSqliteOptions = ({ name }: { name: string }): ConnectionOptions => {
return {
name,
type: 'sqlite',
database: ':memory:',
entityPrefix: config.getEnv('database.tablePrefix'),
dropSchema: true,
migrations: sqliteMigrations,
migrationsTableName: 'migrations',
migrationsRun: false,
enableWAL: config.getEnv('database.sqlite.enableWAL'),
};
};
const baseOptions = (type: TestDBType) => ({
host: config.getEnv(`database.${type}db.host`),
port: config.getEnv(`database.${type}db.port`),
username: config.getEnv(`database.${type}db.user`),
password: config.getEnv(`database.${type}db.password`),
entityPrefix: config.getEnv('database.tablePrefix'),
schema: type === 'postgres' ? config.getEnv('database.postgresdb.schema') : undefined,
});
/**
* Generate options for a bootstrap DB connection, to create and drop test databases.
*/
export const getBootstrapDBOptions = (type: TestDBType) => ({
type,
name: type,
database: type,
...baseOptions(type),
});
const getDBOptions = (type: TestDBType, name: string) => ({
type,
name,
database: name,
...baseOptions(type),
dropSchema: true,
migrations: type === 'postgres' ? postgresMigrations : mysqlMigrations,
migrationsRun: false,
migrationsTableName: 'migrations',
entities: Object.values(entities),
synchronize: false,
logging: false,
});
export const getBootstrapDBOptions = (dbType: 'postgresdb' | 'mysqldb'): DataSourceOptions => {
const type = dbType === 'postgresdb' ? 'postgres' : 'mysql';
return {
type,
...getOptionOverrides(dbType),
database: type,
entityPrefix: config.getEnv('database.tablePrefix'),
schema: dbType === 'postgresdb' ? config.getEnv('database.postgresdb.schema') : undefined,
};
};

4
packages/cli/test/integration/shared/types.ts
View file

@ -61,7 +61,3 @@ export type SaveCredentialFunction = (
credentialPayload: CredentialPayload,
{ user }: { user: User },
) => Promise<CredentialsEntity & ICredentialsDb>;
export type PostgresSchemaSection = {
[K in 'host' | 'port' | 'schema' | 'user' | 'password']: { env: string };
};

6
packages/cli/test/teardown.ts
View file

@ -4,14 +4,14 @@ import config from '@/config';
import { getBootstrapDBOptions, testDbPrefix } from './integration/shared/testDb';
export default async () => {
const dbType = config.getEnv('database.type').replace(/db$/, '');
if (dbType !== 'postgres' && dbType !== 'mysql') return;
const dbType = config.getEnv('database.type');
if (dbType !== 'postgresdb' && dbType !== 'mysqldb') return;
const connection = new Connection(getBootstrapDBOptions(dbType));
await connection.initialize();
const query =
dbType === 'postgres' ? 'SELECT datname as "Database" FROM pg_database' : 'SHOW DATABASES';
dbType === 'postgresdb' ? 'SELECT datname as "Database" FROM pg_database' : 'SHOW DATABASES';
const results: Array<{ Database: string }> = await connection.query(query);
const databases = results
.filter(({ Database: dbName }) => dbName.startsWith(testDbPrefix))