feat(core): Create a dsl for writing db agnostic migrations (#6853)

packages/cli/package.json

@@ -159,6 +159,7 @@
     "open": "^7.0.0",
     "openapi-types": "^10.0.0",
     "p-cancelable": "^2.0.0",
+    "p-lazy": "^3.1.0",
     "passport": "^0.6.0",
     "passport-cookie": "^1.0.9",
     "passport-jwt": "^4.0.0",

packages/cli/src/databases/dsl/Column.ts

@@ -0,0 +1,123 @@
+import type { Driver, TableColumnOptions } from 'typeorm';
+
+export class Column {
+	private type: 'int' | 'boolean' | 'varchar' | 'text' | 'json' | 'timestamp' | 'uuid';
+
+	private isGenerated = false;
+
+	private isNullable = true;
+
+	private isPrimary = false;
+
+	private length: number | 'auto';
+
+	private defaultValue: unknown;
+
+	constructor(private name: string) {}
+
+	get bool() {
+		this.type = 'boolean';
+		return this;
+	}
+
+	get int() {
+		this.type = 'int';
+		return this;
+	}
+
+	varchar(length?: number) {
+		this.type = 'varchar';
+		this.length = length ?? 'auto';
+		return this;
+	}
+
+	get text() {
+		this.type = 'text';
+		return this;
+	}
+
+	get json() {
+		this.type = 'json';
+		return this;
+	}
+
+	timestamp(msPrecision?: number) {
+		this.type = 'timestamp';
+		this.length = msPrecision ?? 'auto';
+		return this;
+	}
+
+	get uuid() {
+		this.type = 'uuid';
+		return this;
+	}
+
+	get primary() {
+		this.isPrimary = true;
+		return this;
+	}
+
+	get notNull() {
+		this.isNullable = false;
+		return this;
+	}
+
+	default(value: unknown) {
+		this.defaultValue = value;
+		return this;
+	}
+
+	get autoGenerate() {
+		this.isGenerated = true;
+		return this;
+	}
+
+	toOptions(driver: Driver): TableColumnOptions {
+		const { name, type, isNullable, isPrimary, isGenerated, length } = this;
+		const isMysql = 'mysql' in driver;
+		const isPostgres = 'postgres' in driver;
+		const isSqlite = 'sqlite' in driver;
+
+		const options: TableColumnOptions = {
+			name,
+			isNullable,
+			isPrimary,
+			type,
+		};
+
+		if (options.type === 'int' && isSqlite) {
+			options.type = 'integer';
+		} else if (type === 'boolean' && isMysql) {
+			options.type = 'tinyint(1)';
+		} else if (type === 'timestamp') {
+			options.type = isPostgres ? 'timestamptz' : 'datetime';
+		} else if (type === 'json' && isSqlite) {
+			options.type = 'text';
+		}
+
+		if ((type === 'varchar' || type === 'timestamp') && length !== 'auto') {
+			options.type = `${options.type}(${length})`;
+		}
+
+		if (isGenerated) {
+			options.isGenerated = true;
+			options.generationStrategy = type === 'uuid' ? 'uuid' : 'increment';
+		}
+
+		if (isPrimary || isGenerated) {
+			options.isNullable = false;
+		}
+
+		if (this.defaultValue !== undefined) {
+			if (type === 'timestamp' && this.defaultValue === 'NOW()') {
+				options.default = isSqlite
+					? "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')"
+					: 'CURRENT_TIMESTAMP(3)';
+			} else {
+				options.default = this.defaultValue;
+			}
+		}
+
+		return options;
+	}
+}

packages/cli/src/databases/dsl/Indices.ts

@@ -0,0 +1,45 @@
+import { QueryRunner, TableIndex } from 'typeorm';
+import LazyPromise from 'p-lazy';
+
+abstract class IndexOperation extends LazyPromise<void> {
+	abstract execute(queryRunner: QueryRunner): Promise<void>;
+
+	constructor(
+		protected name: string,
+		protected tableName: string,
+		protected prefix: string,
+		queryRunner: QueryRunner,
+	) {
+		super((resolve) => {
+			void this.execute(queryRunner).then(resolve);
+		});
+	}
+}
+
+export class CreateIndex extends IndexOperation {
+	constructor(
+		name: string,
+		tableName: string,
+		protected columnNames: string[],
+		protected isUnique: boolean,
+		prefix: string,
+		queryRunner: QueryRunner,
+	) {
+		super(name, tableName, prefix, queryRunner);
+	}
+
+	async execute(queryRunner: QueryRunner) {
+		const { tableName, name, columnNames, prefix, isUnique } = this;
+		return queryRunner.createIndex(
+			`${prefix}${tableName}`,
+			new TableIndex({ name: `IDX_${prefix}${name}`, columnNames, isUnique }),
+		);
+	}
+}
+
+export class DropIndex extends IndexOperation {
+	async execute(queryRunner: QueryRunner) {
+		const { tableName, name, prefix } = this;
+		return queryRunner.dropIndex(`${prefix}${tableName}`, `IDX_${prefix}${name}`);
+	}
+}

packages/cli/src/databases/dsl/Table.ts

@@ -0,0 +1,82 @@
+import type { TableForeignKeyOptions, TableIndexOptions } from 'typeorm';
+import { Table, QueryRunner } from 'typeorm';
+import LazyPromise from 'p-lazy';
+import { Column } from './Column';
+
+abstract class TableOperation<R = void> extends LazyPromise<R> {
+	abstract execute(queryRunner: QueryRunner): Promise<R>;
+
+	constructor(
+		protected tableName: string,
+		protected prefix: string,
+		queryRunner: QueryRunner,
+	) {
+		super((resolve) => {
+			void this.execute(queryRunner).then(resolve);
+		});
+	}
+}
+
+export class CreateTable extends TableOperation {
+	private columns: Column[] = [];
+
+	private indices = new Set<TableIndexOptions>();
+
+	private foreignKeys = new Set<TableForeignKeyOptions>();
+
+	withColumns(...columns: Column[]) {
+		this.columns.push(...columns);
+		return this;
+	}
+
+	get withTimestamps() {
+		this.columns.push(
+			new Column('createdAt').timestamp(3).notNull.default('NOW()'),
+			new Column('updatedAt').timestamp(3).notNull.default('NOW()'),
+		);
+		return this;
+	}
+
+	withIndexOn(columnName: string | string[], isUnique = false) {
+		const columnNames = Array.isArray(columnName) ? columnName : [columnName];
+		this.indices.add({ columnNames, isUnique });
+		return this;
+	}
+
+	withForeignKey(
+		columnName: string,
+		ref: { tableName: string; columnName: string; onDelete?: 'CASCADE'; onUpdate?: 'CASCADE' },
+	) {
+		const foreignKey: TableForeignKeyOptions = {
+			columnNames: [columnName],
+			referencedTableName: `${this.prefix}${ref.tableName}`,
+			referencedColumnNames: [ref.columnName],
+		};
+		if (ref.onDelete) foreignKey.onDelete = ref.onDelete;
+		if (ref.onUpdate) foreignKey.onUpdate = ref.onUpdate;
+		this.foreignKeys.add(foreignKey);
+		return this;
+	}
+
+	async execute(queryRunner: QueryRunner) {
+		const { driver } = queryRunner.connection;
+		const { columns, tableName: name, prefix, indices, foreignKeys } = this;
+		return queryRunner.createTable(
+			new Table({
+				name: `${prefix}${name}`,
+				columns: columns.map((c) => c.toOptions(driver)),
+				...(indices.size ? { indices: [...indices] } : {}),
+				...(foreignKeys.size ? { foreignKeys: [...foreignKeys] } : {}),
+				...('mysql' in driver ? { engine: 'InnoDB' } : {}),
+			}),
+			true,
+		);
+	}
+}
+
+export class DropTable extends TableOperation {
+	async execute(queryRunner: QueryRunner) {
+		const { tableName: name, prefix } = this;
+		return queryRunner.dropTable(`${prefix}${name}`, true);
+	}
+}

packages/cli/src/databases/dsl/index.ts

@@ -0,0 +1,17 @@
+import type { QueryRunner } from 'typeorm';
+import { Column } from './Column';
+import { CreateTable, DropTable } from './Table';
+import { CreateIndex, DropIndex } from './Indices';
+
+export const createSchemaBuilder = (tablePrefix: string, queryRunner: QueryRunner) => ({
+	column: (name: string) => new Column(name),
+	/* eslint-disable @typescript-eslint/promise-function-async */
+	// NOTE: Do not add `async` to these functions, as that messes up the lazy-evaluation of LazyPromise
+	createTable: (name: string) => new CreateTable(name, tablePrefix, queryRunner),
+	dropTable: (name: string) => new DropTable(name, tablePrefix, queryRunner),
+	createIndex: (name: string, tableName: string, columnNames: string[], isUnique = false) =>
+		new CreateIndex(name, tableName, columnNames, isUnique, tablePrefix, queryRunner),
+	dropIndex: (name: string, tableName: string) =>
+		new DropIndex(name, tableName, tablePrefix, queryRunner),
+	/* eslint-enable */
+});

packages/cli/src/databases/types.ts

@@ -1,6 +1,7 @@
-import type { Logger } from '@/Logger';
 import type { INodeTypes } from 'n8n-workflow';
 import type { QueryRunner, ObjectLiteral } from 'typeorm';
+import type { Logger } from '@/Logger';
+import type { createSchemaBuilder } from './dsl';
 
 export type DatabaseType = 'mariadb' | 'postgresdb' | 'mysqldb' | 'sqlite';
 
@@ -13,6 +14,7 @@ export interface MigrationContext {
 	dbName: string;
 	migrationName: string;
 	nodeTypes: INodeTypes;
+	schemaBuilder: ReturnType<typeof createSchemaBuilder>;
 	loadSurveyFromDisk(): string | null;
 	parseJson<T>(data: string | T): T;
 	escape: {

packages/cli/src/databases/utils/migrationHelpers.ts

@@ -3,12 +3,13 @@ import { readFileSync, rmSync } from 'fs';
 import { UserSettings } from 'n8n-core';
 import type { ObjectLiteral } from 'typeorm';
 import type { QueryRunner } from 'typeorm/query-runner/QueryRunner';
+import { jsonParse } from 'n8n-workflow';
 import config from '@/config';
 import { inTest } from '@/constants';
 import type { BaseMigration, Migration, MigrationContext, MigrationFn } from '@db/types';
+import { createSchemaBuilder } from '@db/dsl';
 import { getLogger } from '@/Logger';
 import { NodeTypes } from '@/NodeTypes';
-import { jsonParse } from 'n8n-workflow';
 
 const logger = getLogger();
 
@@ -99,6 +100,7 @@ const createContext = (queryRunner: QueryRunner, migration: Migration): Migratio
 	dbName,
 	migrationName: migration.name,
 	queryRunner,
+	schemaBuilder: createSchemaBuilder(tablePrefix, queryRunner),
 	nodeTypes: Container.get(NodeTypes),
 	loadSurveyFromDisk,
 	parseJson,

pnpm-lock.yaml

@@ -367,6 +367,9 @@ importers:
       p-cancelable:
         specifier: ^2.0.0
         version: 2.1.1
+      p-lazy:
+        specifier: ^3.1.0
+        version: 3.1.0
       passport:
         specifier: ^0.6.0
         version: 0.6.0
@@ -16292,6 +16295,11 @@ packages:
     resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==}
     engines: {node: '>=4'}
 
+  /p-lazy@3.1.0:
+    resolution: {integrity: sha512-sCJn0Cdahs6G6SX9+DUihVFUhrzDEduzE5xeViVBGtoqy5dBWko7W8T6Kk6TjR2uevRXJO7CShfWrqdH5s3w3g==}
+    engines: {node: '>=8'}
+    dev: false
+
   /p-limit@2.3.0:
     resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==}
     engines: {node: '>=6'}