diff --git a/packages/cli/src/databases/dsl/Indices.ts b/packages/cli/src/databases/dsl/Indices.ts index a5ec95e101..040ab04105 100644 --- a/packages/cli/src/databases/dsl/Indices.ts +++ b/packages/cli/src/databases/dsl/Indices.ts @@ -1,5 +1,5 @@ import type { QueryRunner } from '@n8n/typeorm'; -import { TableIndex } from '@n8n/typeorm'; +import { TableIndex, TypeORMError } from '@n8n/typeorm'; import LazyPromise from 'p-lazy'; abstract class IndexOperation extends LazyPromise { @@ -48,10 +48,29 @@ export class CreateIndex extends IndexOperation { } export class DropIndex extends IndexOperation { + constructor( + tableName: string, + columnNames: string[], + tablePrefix: string, + queryRunner: QueryRunner, + customIndexName?: string, + protected skipIfMissing = false, + ) { + super(tableName, columnNames, tablePrefix, queryRunner, customIndexName); + } + async execute(queryRunner: QueryRunner) { - return await queryRunner.dropIndex( - this.fullTableName, - this.customIndexName ?? this.fullIndexName, - ); + return await queryRunner + .dropIndex(this.fullTableName, this.customIndexName ?? this.fullIndexName) + .catch((error) => { + if ( + error instanceof TypeORMError && + error.message.includes('not found') && + this.skipIfMissing + ) { + return; + } + throw error; + }); } } diff --git a/packages/cli/src/databases/dsl/index.ts b/packages/cli/src/databases/dsl/index.ts index bb5ef859ec..a51e61bd17 100644 --- a/packages/cli/src/databases/dsl/index.ts +++ b/packages/cli/src/databases/dsl/index.ts @@ -32,8 +32,14 @@ export const createSchemaBuilder = (tablePrefix: string, queryRunner: QueryRunne customIndexName?: string, ) => new CreateIndex(tableName, columnNames, isUnique, tablePrefix, queryRunner, customIndexName), - dropIndex: (tableName: string, columnNames: string[], customIndexName?: string) => - new DropIndex(tableName, columnNames, tablePrefix, queryRunner, customIndexName), + dropIndex: ( + tableName: string, + columnNames: string[], + { customIndexName, skipIfMissing }: { customIndexName?: string; skipIfMissing?: boolean } = { + skipIfMissing: false, + }, + ) => + new DropIndex(tableName, columnNames, tablePrefix, queryRunner, customIndexName, skipIfMissing), addForeignKey: ( tableName: string, diff --git a/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts b/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts new file mode 100644 index 0000000000..dcf558f202 --- /dev/null +++ b/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts @@ -0,0 +1,118 @@ +import type { MigrationContext, ReversibleMigration } from '@/databases/types'; + +/** + * Add new indices: + * + * - `workflowId, startedAt` for `ExecutionRepository.findManyByRangeQuery` (default query) and for `ExecutionRepository.findManyByRangeQuery` (filter query) + * - `waitTill, status, deletedAt` for `ExecutionRepository.getWaitingExecutions` + * - `stoppedAt, status, deletedAt` for `ExecutionRepository.softDeletePrunableExecutions` + * + * Remove unused indices in sqlite: + * + * - `stoppedAt` (duplicate with different casing) + * - `waitTill` + * - `status, workflowId` + * + * Remove unused indices in MySQL: + * + * - `status` + * + * Remove unused indices in all DBs: + * + * - `waitTill, id` + * - `workflowId, id` + * + * Remove incomplete index in all DBs: + * + * - `stopped_at` (replaced with composite index) + * + * Keep index as is: + * + * - `deletedAt` for query at `ExecutionRepository.hardDeleteSoftDeletedExecutions` + */ +export class RefactorExecutionIndices1723796243146 implements ReversibleMigration { + async up({ schemaBuilder, isPostgres, isSqlite, isMysql, runQuery, escape }: MigrationContext) { + if (isSqlite || isPostgres) { + const executionEntity = escape.tableName('execution_entity'); + + const workflowId = escape.columnName('workflowId'); + const startedAt = escape.columnName('startedAt'); + const waitTill = escape.columnName('waitTill'); + const status = escape.columnName('status'); + const deletedAt = escape.columnName('deletedAt'); + const stoppedAt = escape.columnName('stoppedAt'); + + await runQuery(` + CREATE INDEX idx_execution_entity_workflow_id_started_at + ON ${executionEntity} (${workflowId}, ${startedAt}) + WHERE ${startedAt} IS NOT NULL AND ${deletedAt} IS NULL; + `); + + await runQuery(` + CREATE INDEX idx_execution_entity_wait_till_status_deleted_at + ON ${executionEntity} (${waitTill}, ${status}, ${deletedAt}) + WHERE ${waitTill} IS NOT NULL AND ${deletedAt} IS NULL; + `); + + await runQuery(` + CREATE INDEX idx_execution_entity_stopped_at_status_deleted_at + ON ${executionEntity} (${stoppedAt}, ${status}, ${deletedAt}) + WHERE ${stoppedAt} IS NOT NULL AND ${deletedAt} IS NULL; + `); + } else if (isMysql) { + await schemaBuilder.createIndex('execution_entity', ['workflowId', 'startedAt']); + await schemaBuilder.createIndex('execution_entity', ['waitTill', 'status', 'deletedAt']); + await schemaBuilder.createIndex('execution_entity', ['stoppedAt', 'status', 'deletedAt']); + } + + if (isSqlite) { + await schemaBuilder.dropIndex('execution_entity', ['waitTill'], { + customIndexName: 'idx_execution_entity_wait_till', + skipIfMissing: true, + }); + + await schemaBuilder.dropIndex('execution_entity', ['status', 'workflowId'], { + customIndexName: 'IDX_8b6f3f9ae234f137d707b98f3bf43584', + skipIfMissing: true, + }); + } + + if (isMysql) { + await schemaBuilder.dropIndex('execution_entity', ['status'], { + customIndexName: 'IDX_8b6f3f9ae234f137d707b98f3bf43584', + skipIfMissing: true, + }); + } + + // all DBs + + await schemaBuilder.dropIndex( + 'execution_entity', + ['stoppedAt'], + isSqlite ? { customIndexName: 'idx_execution_entity_stopped_at', skipIfMissing: true } : {}, + ); + await schemaBuilder.dropIndex('execution_entity', ['waitTill', 'id'], { + customIndexName: isPostgres + ? 'IDX_85b981df7b444f905f8bf50747' + : 'IDX_b94b45ce2c73ce46c54f20b5f9', + skipIfMissing: true, + }); + await schemaBuilder.dropIndex('execution_entity', ['workflowId', 'id'], { + customIndexName: + isPostgres || isMysql + ? 'idx_execution_entity_workflow_id_id' + : 'IDX_81fc04c8a17de15835713505e4', + skipIfMissing: true, + }); + } + + async down({ schemaBuilder }: MigrationContext) { + await schemaBuilder.dropIndex('execution_entity', ['workflowId', 'startedAt']); + await schemaBuilder.dropIndex('execution_entity', ['waitTill', 'status']); + await schemaBuilder.dropIndex('execution_entity', ['stoppedAt', 'deletedAt', 'status']); + + await schemaBuilder.createIndex('execution_entity', ['waitTill', 'id']); + await schemaBuilder.createIndex('execution_entity', ['stoppedAt']); + await schemaBuilder.createIndex('execution_entity', ['workflowId', 'id']); + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index b4900eb59d..ed08c6b2f2 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -58,6 +58,7 @@ import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-Move import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNodesAccess'; import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; +import { RefactorExecutionIndices1723796243146 } from '../common/1723796243146-RefactorExecutionIndices'; import { AddConstraintToExecutionMetadata1720101653148 } from '../common/1720101653148-AddConstraintToExecutionMetadata'; import { CreateInvalidAuthTokenTable1723627610222 } from '../common/1723627610222-CreateInvalidAuthTokenTable'; @@ -123,4 +124,5 @@ export const mysqlMigrations: Migration[] = [ AddActivatedAtUserSetting1717498465931, AddConstraintToExecutionMetadata1720101653148, CreateInvalidAuthTokenTable1723627610222, + RefactorExecutionIndices1723796243146, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index 85bd58f371..3a37f36936 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -57,6 +57,7 @@ import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-Move import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNodesAccess'; import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; +import { RefactorExecutionIndices1723796243146 } from '../common/1723796243146-RefactorExecutionIndices'; import { AddConstraintToExecutionMetadata1720101653148 } from '../common/1720101653148-AddConstraintToExecutionMetadata'; import { FixExecutionMetadataSequence1721377157740 } from './1721377157740-FixExecutionMetadataSequence'; import { CreateInvalidAuthTokenTable1723627610222 } from '../common/1723627610222-CreateInvalidAuthTokenTable'; @@ -123,4 +124,5 @@ export const postgresMigrations: Migration[] = [ AddConstraintToExecutionMetadata1720101653148, FixExecutionMetadataSequence1721377157740, CreateInvalidAuthTokenTable1723627610222, + RefactorExecutionIndices1723796243146, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index 974c743d0f..e55a36c1c1 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -55,6 +55,7 @@ import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-Move import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNodesAccess'; import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; +import { RefactorExecutionIndices1723796243146 } from '../common/1723796243146-RefactorExecutionIndices'; import { AddConstraintToExecutionMetadata1720101653148 } from '../common/1720101653148-AddConstraintToExecutionMetadata'; import { CreateInvalidAuthTokenTable1723627610222 } from '../common/1723627610222-CreateInvalidAuthTokenTable'; @@ -117,6 +118,7 @@ const sqliteMigrations: Migration[] = [ AddActivatedAtUserSetting1717498465931, AddConstraintToExecutionMetadata1720101653148, CreateInvalidAuthTokenTable1723627610222, + RefactorExecutionIndices1723796243146, ]; export { sqliteMigrations }; diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 364619fab6..dbcbfc994e 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -784,8 +784,8 @@ export class ExecutionRepository extends Repository { if (firstId) qb.andWhere('execution.id > :firstId', { firstId }); if (lastId) qb.andWhere('execution.id < :lastId', { lastId }); - if (query.order?.stoppedAt === 'DESC') { - qb.orderBy({ 'execution.stoppedAt': 'DESC' }); + if (query.order?.startedAt === 'DESC') { + qb.orderBy({ 'execution.startedAt': 'DESC' }); } else if (query.order?.top) { qb.orderBy(`(CASE WHEN execution.status = '${query.order.top}' THEN 0 ELSE 1 END)`); } else { diff --git a/packages/cli/src/databases/types.ts b/packages/cli/src/databases/types.ts index f4c3db0452..96a26e4712 100644 --- a/packages/cli/src/databases/types.ts +++ b/packages/cli/src/databases/types.ts @@ -11,6 +11,8 @@ export interface MigrationContext { tablePrefix: string; dbType: DatabaseType; isMysql: boolean; + isSqlite: boolean; + isPostgres: boolean; dbName: string; migrationName: string; nodeTypes: INodeTypes; diff --git a/packages/cli/src/databases/utils/migrationHelpers.ts b/packages/cli/src/databases/utils/migrationHelpers.ts index 0f116590ee..ed100f9c47 100644 --- a/packages/cli/src/databases/utils/migrationHelpers.ts +++ b/packages/cli/src/databases/utils/migrationHelpers.ts @@ -93,6 +93,8 @@ function parseJson(data: string | T): T { const globalConfig = Container.get(GlobalConfig); const dbType = globalConfig.database.type; const isMysql = ['mariadb', 'mysqldb'].includes(dbType); +const isSqlite = dbType === 'sqlite'; +const isPostgres = dbType === 'postgresdb'; const dbName = globalConfig.database[dbType === 'mariadb' ? 'mysqldb' : dbType].database; const tablePrefix = globalConfig.database.tablePrefix; @@ -101,6 +103,8 @@ const createContext = (queryRunner: QueryRunner, migration: Migration): Migratio tablePrefix, dbType, isMysql, + isSqlite, + isPostgres, dbName, migrationName: migration.name, queryRunner, diff --git a/packages/cli/src/executions/execution.service.ts b/packages/cli/src/executions/execution.service.ts index 61a1dc0db1..a4ae0ffbae 100644 --- a/packages/cli/src/executions/execution.service.ts +++ b/packages/cli/src/executions/execution.service.ts @@ -361,7 +361,7 @@ export class ExecutionService { /** * Return: * - * - the latest summaries of current and completed executions that satisfy a query, + * - the summaries of latest current and completed executions that satisfy a query, * - the total count of all completed executions that satisfy the query, and * - whether the total of completed executions is an estimate. * @@ -382,7 +382,7 @@ export class ExecutionService { this.findRangeWithCount({ ...query, status: completedStatuses, - order: { stoppedAt: 'DESC' }, + order: { startedAt: 'DESC' }, }), ]); diff --git a/packages/cli/src/executions/execution.types.ts b/packages/cli/src/executions/execution.types.ts index 15c27261fc..10a2ec7285 100644 --- a/packages/cli/src/executions/execution.types.ts +++ b/packages/cli/src/executions/execution.types.ts @@ -86,7 +86,7 @@ export namespace ExecutionSummaries { type OrderFields = { order?: { top?: ExecutionStatus; - stoppedAt?: 'DESC'; + startedAt?: 'DESC'; }; };