From f8a7fb38ccf433e79b1916a159f203d44a94f2d2 Mon Sep 17 00:00:00 2001 From: Danny Martini Date: Tue, 25 Feb 2025 11:35:52 +0100 Subject: [PATCH] feat(core): Add tables for reporting dashboard (no-changelog) (#13336) --- packages/cli/src/databases/dsl/column.ts | 80 +++++++++++-- .../1739549398681-CreateAnalyticsTables.ts | 106 ++++++++++++++++++ .../src/databases/migrations/mysqldb/index.ts | 2 + .../databases/migrations/postgresdb/index.ts | 2 + .../src/databases/migrations/sqlite/index.ts | 2 + 5 files changed, 185 insertions(+), 7 deletions(-) create mode 100644 packages/cli/src/databases/migrations/common/1739549398681-CreateAnalyticsTables.ts diff --git a/packages/cli/src/databases/dsl/column.ts b/packages/cli/src/databases/dsl/column.ts index 95ff31c200..83f2f8fe58 100644 --- a/packages/cli/src/databases/dsl/column.ts +++ b/packages/cli/src/databases/dsl/column.ts @@ -1,10 +1,20 @@ import type { Driver, TableColumnOptions } from '@n8n/typeorm'; export class Column { - private type: 'int' | 'boolean' | 'varchar' | 'text' | 'json' | 'timestamp' | 'uuid'; + private type: + | 'int' + | 'boolean' + | 'varchar' + | 'text' + | 'json' + | 'timestamptz' + | 'timestamp' + | 'uuid'; private isGenerated = false; + private isGenerated2 = false; + private isNullable = true; private isPrimary = false; @@ -15,6 +25,8 @@ export class Column { private primaryKeyConstraintName: string | undefined; + private commentValue: string | undefined; + constructor(private name: string) {} get bool() { @@ -43,7 +55,22 @@ export class Column { return this; } + /** + * @deprecated use `timestampTimezone` instead + **/ timestamp(msPrecision = 3) { + this.type = 'timestamptz'; + this.length = msPrecision ?? 'auto'; + return this; + } + + timestampTimezone(msPrecision = 3) { + this.type = 'timestamptz'; + this.length = msPrecision ?? 'auto'; + return this; + } + + timestampNoTimezone(msPrecision = 3) { this.type = 'timestamp'; this.length = msPrecision ?? 'auto'; return this; @@ -75,15 +102,40 @@ export class Column { return this; } + /** + * @deprecated, use autoGenerate2 instead + **/ get autoGenerate() { this.isGenerated = true; return this; } + /** + * Prefers `identity` over `increment` (which turns to `serial` for pg) + * See https://wiki.postgresql.org/wiki/Don%27t_Do_This#Don.27t_use_serial + **/ + get autoGenerate2() { + this.isGenerated2 = true; + return this; + } + + comment(comment: string) { + this.commentValue = comment; + return this; + } + // eslint-disable-next-line complexity toOptions(driver: Driver): TableColumnOptions { - const { name, type, isNullable, isPrimary, isGenerated, length, primaryKeyConstraintName } = - this; + const { + name, + type, + isNullable, + isPrimary, + isGenerated, + isGenerated2, + length, + primaryKeyConstraintName, + } = this; const isMysql = 'mysql' in driver; const isPostgres = 'postgres' in driver; const isSqlite = 'sqlite' in driver; @@ -100,8 +152,10 @@ export class Column { options.type = 'integer'; } else if (type === 'boolean' && isMysql) { options.type = 'tinyint(1)'; - } else if (type === 'timestamp') { + } else if (type === 'timestamptz') { options.type = isPostgres ? 'timestamptz' : 'datetime'; + } else if (type === 'timestamp') { + options.type = isPostgres ? 'timestamp' : 'datetime'; } else if (type === 'json' && isSqlite) { options.type = 'text'; } else if (type === 'uuid') { @@ -111,7 +165,10 @@ export class Column { if (isSqlite) options.type = 'varchar'; } - if ((type === 'varchar' || type === 'timestamp') && length !== 'auto') { + if ( + (type === 'varchar' || type === 'timestamptz' || type === 'timestamp') && + length !== 'auto' + ) { options.type = `${options.type}(${length})`; } @@ -120,12 +177,17 @@ export class Column { options.generationStrategy = type === 'uuid' ? 'uuid' : 'increment'; } - if (isPrimary || isGenerated) { + if (isGenerated2) { + options.isGenerated = true; + options.generationStrategy = type === 'uuid' ? 'uuid' : 'identity'; + } + + if (isPrimary || isGenerated || isGenerated2) { options.isNullable = false; } if (this.defaultValue !== undefined) { - if (type === 'timestamp' && this.defaultValue === 'NOW()') { + if ((type === 'timestamptz' || type === 'timestamp') && this.defaultValue === 'NOW()') { options.default = isSqlite ? "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')" : 'CURRENT_TIMESTAMP(3)'; @@ -134,6 +196,10 @@ export class Column { } } + if (this.commentValue) { + options.comment = this.commentValue; + } + return options; } } diff --git a/packages/cli/src/databases/migrations/common/1739549398681-CreateAnalyticsTables.ts b/packages/cli/src/databases/migrations/common/1739549398681-CreateAnalyticsTables.ts new file mode 100644 index 0000000000..ee1bdb4ccf --- /dev/null +++ b/packages/cli/src/databases/migrations/common/1739549398681-CreateAnalyticsTables.ts @@ -0,0 +1,106 @@ +import type { MigrationContext, ReversibleMigration } from '@/databases/types'; + +const names = { + // table names + t: { + analyticsMetadata: 'analytics_metadata', + analyticsRaw: 'analytics_raw', + analyticsByPeriod: 'analytics_by_period', + workflowEntity: 'workflow_entity', + project: 'project', + }, + // column names by table + c: { + analyticsMetadata: { + metaId: 'metaId', + projectId: 'projectId', + workflowId: 'workflowId', + }, + analyticsRaw: { + metaId: 'metaId', + }, + analyticsByPeriod: { + metaId: 'metaId', + type: 'type', + periodUnit: 'periodUnit', + periodStart: 'periodStart', + }, + project: { + id: 'id', + }, + workflowEntity: { + id: 'id', + }, + }, +}; + +export class CreateAnalyticsTables1739549398681 implements ReversibleMigration { + async up({ schemaBuilder: { createTable, column } }: MigrationContext) { + await createTable(names.t.analyticsMetadata) + .withColumns( + column(names.c.analyticsMetadata.metaId).int.primary.autoGenerate2, + column(names.c.analyticsMetadata.workflowId).varchar(16), + column(names.c.analyticsMetadata.projectId).varchar(36), + column('workflowName').varchar(128).notNull, + column('projectName').varchar(255).notNull, + ) + .withForeignKey(names.c.analyticsMetadata.workflowId, { + tableName: names.t.workflowEntity, + columnName: names.c.workflowEntity.id, + onDelete: 'SET NULL', + }) + .withForeignKey(names.c.analyticsMetadata.projectId, { + tableName: names.t.project, + columnName: names.c.project.id, + onDelete: 'SET NULL', + }); + + const typeComment = '0: time_saved_minutes, 1: runtime_milliseconds, 2: success, 3: failure'; + + await createTable(names.t.analyticsRaw) + .withColumns( + column('id').int.primary.autoGenerate2, + column(names.c.analyticsRaw.metaId).int.notNull, + column('type').int.notNull.comment(typeComment), + column('value').int.notNull, + column('timestamp').timestampNoTimezone(0).default('CURRENT_TIMESTAMP').notNull, + ) + .withForeignKey(names.c.analyticsRaw.metaId, { + tableName: names.t.analyticsMetadata, + columnName: names.c.analyticsMetadata.metaId, + onDelete: 'CASCADE', + }); + + await createTable(names.t.analyticsByPeriod) + .withColumns( + column('id').int.primary.autoGenerate2, + column(names.c.analyticsByPeriod.metaId).int.notNull, + column(names.c.analyticsByPeriod.type).int.notNull.comment(typeComment), + column('value').int.notNull, + column(names.c.analyticsByPeriod.periodUnit).int.notNull.comment( + '0: hour, 1: day, 2: week', + ), + column(names.c.analyticsByPeriod.periodStart).timestampNoTimezone(0), + ) + .withForeignKey(names.c.analyticsByPeriod.metaId, { + tableName: names.t.analyticsMetadata, + columnName: names.c.analyticsMetadata.metaId, + onDelete: 'CASCADE', + }) + .withIndexOn( + [ + names.c.analyticsByPeriod.periodStart, + names.c.analyticsByPeriod.type, + names.c.analyticsByPeriod.periodUnit, + names.c.analyticsByPeriod.metaId, + ], + true, + ); + } + + async down({ schemaBuilder: { dropTable } }: MigrationContext) { + await dropTable(names.t.analyticsMetadata); + await dropTable(names.t.analyticsRaw); + await dropTable(names.t.analyticsByPeriod); + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index e16c387167..5e727487dd 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -81,6 +81,7 @@ import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/17344 import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable'; import { AddErrorColumnsToTestRuns1737715421462 } from '../common/1737715421462-AddErrorColumnsToTestRuns'; import { CreateFolderTable1738709609940 } from '../common/1738709609940-CreateFolderTable'; +import { CreateAnalyticsTables1739549398681 } from '../common/1739549398681-CreateAnalyticsTables'; export const mysqlMigrations: Migration[] = [ InitialMigration1588157391238, @@ -164,4 +165,5 @@ export const mysqlMigrations: Migration[] = [ AddErrorColumnsToTestRuns1737715421462, CreateFolderTable1738709609940, FixTestDefinitionPrimaryKey1739873751194, + CreateAnalyticsTables1739549398681, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index cb667aabed..0d5d7d72bd 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -80,6 +80,7 @@ import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-A import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable'; import { AddErrorColumnsToTestRuns1737715421462 } from '../common/1737715421462-AddErrorColumnsToTestRuns'; import { CreateFolderTable1738709609940 } from '../common/1738709609940-CreateFolderTable'; +import { CreateAnalyticsTables1739549398681 } from '../common/1739549398681-CreateAnalyticsTables'; export const postgresMigrations: Migration[] = [ InitialMigration1587669153312, @@ -162,4 +163,5 @@ export const postgresMigrations: Migration[] = [ CreateTestCaseExecutionTable1736947513045, AddErrorColumnsToTestRuns1737715421462, CreateFolderTable1738709609940, + CreateAnalyticsTables1739549398681, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index 5e26e91075..73e1bc180c 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -77,6 +77,7 @@ import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/17344 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun'; import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable'; import { AddErrorColumnsToTestRuns1737715421462 } from '../common/1737715421462-AddErrorColumnsToTestRuns'; +import { CreateAnalyticsTables1739549398681 } from '../common/1739549398681-CreateAnalyticsTables'; const sqliteMigrations: Migration[] = [ InitialMigration1588102412422, @@ -156,6 +157,7 @@ const sqliteMigrations: Migration[] = [ CreateTestCaseExecutionTable1736947513045, AddErrorColumnsToTestRuns1737715421462, CreateFolderTable1738709609940, + CreateAnalyticsTables1739549398681, ]; export { sqliteMigrations };