diff --git a/packages/cli/src/databases/entities/index.ts b/packages/cli/src/databases/entities/index.ts index e6a1dedb3f..1393b6a305 100644 --- a/packages/cli/src/databases/entities/index.ts +++ b/packages/cli/src/databases/entities/index.ts @@ -20,6 +20,7 @@ import { Settings } from './settings'; import { SharedCredentials } from './shared-credentials'; import { SharedWorkflow } from './shared-workflow'; import { TagEntity } from './tag-entity'; +import { TestCaseExecution } from './test-case-execution.ee'; import { TestDefinition } from './test-definition.ee'; import { TestMetric } from './test-metric.ee'; import { TestRun } from './test-run.ee'; @@ -64,4 +65,5 @@ export const entities = { TestDefinition, TestMetric, TestRun, + TestCaseExecution, }; diff --git a/packages/cli/src/databases/entities/test-case-execution.ee.ts b/packages/cli/src/databases/entities/test-case-execution.ee.ts new file mode 100644 index 0000000000..dc15ae63b5 --- /dev/null +++ b/packages/cli/src/databases/entities/test-case-execution.ee.ts @@ -0,0 +1,68 @@ +import { Column, Entity, ManyToOne, OneToOne } from '@n8n/typeorm'; + +import { + datetimeColumnType, + jsonColumnType, + WithStringId, +} from '@/databases/entities/abstract-entity'; +import type { ExecutionEntity } from '@/databases/entities/execution-entity'; +import { TestRun } from '@/databases/entities/test-run.ee'; + +export type TestCaseRunMetrics = Record; + +/** + * This entity represents the linking between the test runs and individual executions. + * It stores status, links to past, new and evaluation executions, and metrics produced by individual evaluation wf executions + * Entries in this table are meant to outlive the execution entities, which might be pruned over time. + * This allows us to keep track of the details of test runs' status and metrics even after the executions are deleted. + */ +@Entity({ name: 'test_case_execution' }) +export class TestCaseExecution extends WithStringId { + @ManyToOne('TestRun') + testRun: TestRun; + + @ManyToOne('ExecutionEntity', { + onDelete: 'SET NULL', + nullable: true, + }) + pastExecution: ExecutionEntity | null; + + @Column({ type: 'varchar', nullable: true }) + pastExecutionId: string | null; + + @OneToOne('ExecutionEntity', { + onDelete: 'SET NULL', + nullable: true, + }) + execution: ExecutionEntity | null; + + @Column({ type: 'varchar', nullable: true }) + executionId: string | null; + + @OneToOne('ExecutionEntity', { + onDelete: 'SET NULL', + nullable: true, + }) + evaluationExecution: ExecutionEntity | null; + + @Column({ type: 'varchar', nullable: true }) + evaluationExecutionId: string | null; + + @Column() + status: 'new' | 'running' | 'evaluation_running' | 'success' | 'error' | 'cancelled'; + + @Column({ type: datetimeColumnType, nullable: true }) + runAt: Date | null; + + @Column({ type: datetimeColumnType, nullable: true }) + completedAt: Date | null; + + @Column('varchar', { nullable: true }) + errorCode: string | null; + + @Column(jsonColumnType, { nullable: true }) + errorDetails: Record; + + @Column(jsonColumnType, { nullable: true }) + metrics: TestCaseRunMetrics; +} diff --git a/packages/cli/src/databases/migrations/common/1736947513045-CreateTestCaseExecutionTable.ts b/packages/cli/src/databases/migrations/common/1736947513045-CreateTestCaseExecutionTable.ts new file mode 100644 index 0000000000..f7e840cb49 --- /dev/null +++ b/packages/cli/src/databases/migrations/common/1736947513045-CreateTestCaseExecutionTable.ts @@ -0,0 +1,47 @@ +import type { MigrationContext, ReversibleMigration } from '@/databases/types'; + +const testCaseExecutionTableName = 'test_case_execution'; + +export class CreateTestCaseExecutionTable1736947513045 implements ReversibleMigration { + async up({ schemaBuilder: { createTable, column } }: MigrationContext) { + await createTable(testCaseExecutionTableName) + .withColumns( + column('id').varchar(36).primary.notNull, + column('testRunId').varchar(36).notNull, + column('pastExecutionId').int, // Might be null if execution was deleted after the test run + column('executionId').int, // Execution of the workflow under test. Might be null if execution was deleted after the test run + column('evaluationExecutionId').int, // Execution of the evaluation workflow. Might be null if execution was deleted after the test run, or if the test run was cancelled + column('status').varchar().notNull, + column('runAt').timestamp(), + column('completedAt').timestamp(), + column('errorCode').varchar(), + column('errorDetails').json, + column('metrics').json, + ) + .withIndexOn('testRunId') + .withForeignKey('testRunId', { + tableName: 'test_run', + columnName: 'id', + onDelete: 'CASCADE', + }) + .withForeignKey('pastExecutionId', { + tableName: 'execution_entity', + columnName: 'id', + onDelete: 'SET NULL', + }) + .withForeignKey('executionId', { + tableName: 'execution_entity', + columnName: 'id', + onDelete: 'SET NULL', + }) + .withForeignKey('evaluationExecutionId', { + tableName: 'execution_entity', + columnName: 'id', + onDelete: 'SET NULL', + }).withTimestamps; + } + + async down({ schemaBuilder: { dropTable } }: MigrationContext) { + await dropTable(testCaseExecutionTableName); + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index b76409c0c1..32b1dd9751 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -77,6 +77,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable'; import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun'; +import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable'; export const mysqlMigrations: Migration[] = [ InitialMigration1588157391238, @@ -156,4 +157,5 @@ export const mysqlMigrations: Migration[] = [ AddManagedColumnToCredentialsTable1734479635324, AddProjectIcons1729607673469, AddStatsColumnsToTestRun1736172058779, + CreateTestCaseExecutionTable1736947513045, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index 7cf90bde5b..c5547271b7 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -77,6 +77,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable'; import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun'; +import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable'; export const postgresMigrations: Migration[] = [ InitialMigration1587669153312, @@ -156,4 +157,5 @@ export const postgresMigrations: Migration[] = [ AddManagedColumnToCredentialsTable1734479635324, AddProjectIcons1729607673469, AddStatsColumnsToTestRun1736172058779, + CreateTestCaseExecutionTable1736947513045, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index 363a6e47c3..b8b8e26d3d 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -74,6 +74,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable'; import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun'; +import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable'; const sqliteMigrations: Migration[] = [ InitialMigration1588102412422, @@ -150,6 +151,7 @@ const sqliteMigrations: Migration[] = [ AddManagedColumnToCredentialsTable1734479635324, AddProjectIcons1729607673469, AddStatsColumnsToTestRun1736172058779, + CreateTestCaseExecutionTable1736947513045, ]; export { sqliteMigrations }; diff --git a/packages/cli/src/databases/repositories/test-case-execution.repository.ee.ts b/packages/cli/src/databases/repositories/test-case-execution.repository.ee.ts new file mode 100644 index 0000000000..c9798c8941 --- /dev/null +++ b/packages/cli/src/databases/repositories/test-case-execution.repository.ee.ts @@ -0,0 +1,99 @@ +import { Service } from '@n8n/di'; +import type { EntityManager } from '@n8n/typeorm'; +import { DataSource, In, Not, Repository } from '@n8n/typeorm'; +import type { DeepPartial } from '@n8n/typeorm/common/DeepPartial'; + +import { TestCaseExecution } from '@/databases/entities/test-case-execution.ee'; + +@Service() +export class TestCaseExecutionRepository extends Repository { + constructor(dataSource: DataSource) { + super(TestCaseExecution, dataSource.manager); + } + + async createBatch(testRunId: string, pastExecutionIds: string[]) { + const mappings = this.create( + pastExecutionIds.map>((id) => ({ + testRun: { + id: testRunId, + }, + pastExecution: { + id, + }, + status: 'new', + })), + ); + + return await this.save(mappings); + } + + async markAsRunning(testRunId: string, pastExecutionId: string, executionId: string) { + return await this.update( + { testRun: { id: testRunId }, pastExecutionId }, + { + status: 'running', + executionId, + runAt: new Date(), + }, + ); + } + + async markAsEvaluationRunning( + testRunId: string, + pastExecutionId: string, + evaluationExecutionId: string, + ) { + return await this.update( + { testRun: { id: testRunId }, pastExecutionId }, + { + status: 'evaluation_running', + evaluationExecutionId, + }, + ); + } + + async markAsCompleted( + testRunId: string, + pastExecutionId: string, + metrics: Record, + trx?: EntityManager, + ) { + trx = trx ?? this.manager; + + return await trx.update( + TestCaseExecution, + { testRun: { id: testRunId }, pastExecutionId }, + { + status: 'success', + completedAt: new Date(), + metrics, + }, + ); + } + + async markAllPendingAsCancelled(testRunId: string, trx?: EntityManager) { + trx = trx ?? this.manager; + + return await trx.update( + TestCaseExecution, + { testRun: { id: testRunId }, status: Not(In(['success', 'error', 'cancelled'])) }, + { + status: 'cancelled', + completedAt: new Date(), + }, + ); + } + + async markAsFailed(testRunId: string, pastExecutionId: string, trx?: EntityManager) { + trx = trx ?? this.manager; + + return await trx.update( + TestCaseExecution, + { testRun: { id: testRunId }, pastExecutionId }, + { + status: 'error', + completedAt: new Date(), + }, + ); + } +} diff --git a/packages/cli/src/databases/repositories/test-run.repository.ee.ts b/packages/cli/src/databases/repositories/test-run.repository.ee.ts index 728f0bc464..1f2039acee 100644 --- a/packages/cli/src/databases/repositories/test-run.repository.ee.ts +++ b/packages/cli/src/databases/repositories/test-run.repository.ee.ts @@ -1,5 +1,5 @@ import { Service } from '@n8n/di'; -import type { FindManyOptions } from '@n8n/typeorm'; +import type { EntityManager, FindManyOptions } from '@n8n/typeorm'; import { DataSource, Repository } from '@n8n/typeorm'; import type { AggregatedTestRunMetrics } from '@/databases/entities/test-run.ee'; @@ -35,16 +35,19 @@ export class TestRunRepository extends Repository { return await this.update(id, { status: 'completed', completedAt: new Date(), metrics }); } - async markAsCancelled(id: string) { - return await this.update(id, { status: 'cancelled' }); + async markAsCancelled(id: string, trx?: EntityManager) { + trx = trx ?? this.manager; + return await trx.update(TestRun, id, { status: 'cancelled' }); } - async incrementPassed(id: string) { - return await this.increment({ id }, 'passedCases', 1); + async incrementPassed(id: string, trx?: EntityManager) { + trx = trx ?? this.manager; + return await trx.increment(TestRun, { id }, 'passedCases', 1); } - async incrementFailed(id: string) { - return await this.increment({ id }, 'failedCases', 1); + async incrementFailed(id: string, trx?: EntityManager) { + trx = trx ?? this.manager; + return await trx.increment(TestRun, { id }, 'failedCases', 1); } async getMany(testDefinitionId: string, options: ListQuery.Options) { diff --git a/packages/cli/src/evaluation.ee/test-definitions.types.ee.ts b/packages/cli/src/evaluation.ee/test-definitions.types.ee.ts index 9b0507f248..36d9715acc 100644 --- a/packages/cli/src/evaluation.ee/test-definitions.types.ee.ts +++ b/packages/cli/src/evaluation.ee/test-definitions.types.ee.ts @@ -94,4 +94,6 @@ export declare namespace TestRunsRequest { type Delete = AuthenticatedRequest; type Cancel = AuthenticatedRequest; + + type GetCases = AuthenticatedRequest; } diff --git a/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts b/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts index f82b09a8e1..7cfe7b3705 100644 --- a/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts +++ b/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts @@ -13,6 +13,7 @@ import type { TestMetric } from '@/databases/entities/test-metric.ee'; import type { TestRun } from '@/databases/entities/test-run.ee'; import type { User } from '@/databases/entities/user'; import type { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import type { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee'; import type { TestMetricRepository } from '@/databases/repositories/test-metric.repository.ee'; import type { TestRunRepository } from '@/databases/repositories/test-run.repository.ee'; import type { WorkflowRepository } from '@/databases/repositories/workflow.repository'; @@ -25,6 +26,10 @@ import { mockNodeTypesData } from '@test-integration/utils/node-types-data'; import { TestRunnerService } from '../test-runner.service.ee'; +jest.mock('@/db', () => ({ + transaction: (cb: any) => cb(), +})); + const wfUnderTestJson = JSON.parse( readFileSync(path.join(__dirname, './mock-data/workflow.under-test.json'), { encoding: 'utf-8' }), ); @@ -147,6 +152,7 @@ describe('TestRunnerService', () => { const activeExecutions = mock(); const testRunRepository = mock(); const testMetricRepository = mock(); + const testCaseExecutionRepository = mock(); const mockNodeTypes = mockInstance(NodeTypes); mockInstance(LoadNodesAndCredentials, { @@ -190,6 +196,7 @@ describe('TestRunnerService', () => { executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, errorReporter, @@ -207,6 +214,7 @@ describe('TestRunnerService', () => { executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, errorReporter, @@ -247,6 +255,7 @@ describe('TestRunnerService', () => { executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, errorReporter, @@ -350,6 +359,7 @@ describe('TestRunnerService', () => { executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, errorReporter, @@ -410,6 +420,7 @@ describe('TestRunnerService', () => { executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, errorReporter, @@ -466,6 +477,7 @@ describe('TestRunnerService', () => { executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, errorReporter, @@ -526,6 +538,7 @@ describe('TestRunnerService', () => { executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, errorReporter, @@ -602,6 +615,7 @@ describe('TestRunnerService', () => { executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, errorReporter, @@ -629,6 +643,7 @@ describe('TestRunnerService', () => { executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, errorReporter, @@ -661,9 +676,10 @@ describe('TestRunnerService', () => { executionRepository, activeExecutions, testRunRepository, + testCaseExecutionRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ diff --git a/packages/cli/src/evaluation.ee/test-runner/evaluation-metrics.ee.ts b/packages/cli/src/evaluation.ee/test-runner/evaluation-metrics.ee.ts index ab5c921f8c..cd320bd8da 100644 --- a/packages/cli/src/evaluation.ee/test-runner/evaluation-metrics.ee.ts +++ b/packages/cli/src/evaluation.ee/test-runner/evaluation-metrics.ee.ts @@ -9,12 +9,17 @@ export class EvaluationMetrics { } } - addResults(result: IDataObject) { + addResults(result: IDataObject): Record { + const addedMetrics: Record = {}; + for (const [metricName, metricValue] of Object.entries(result)) { if (typeof metricValue === 'number' && this.metricNames.has(metricName)) { + addedMetrics[metricName] = metricValue; this.rawMetricsByName.get(metricName)!.push(metricValue); } } + + return addedMetrics; } getAggregatedMetrics() { diff --git a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts index f2928f0b91..3f8dfacea3 100644 --- a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts +++ b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts @@ -19,9 +19,11 @@ import type { TestRun } from '@/databases/entities/test-run.ee'; import type { User } from '@/databases/entities/user'; import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee'; import { TestMetricRepository } from '@/databases/repositories/test-metric.repository.ee'; import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import * as Db from '@/db'; import { NodeTypes } from '@/node-types'; import { Telemetry } from '@/telemetry'; import { getRunData } from '@/workflow-execute-additional-data'; @@ -30,6 +32,15 @@ import { WorkflowRunner } from '@/workflow-runner'; import { EvaluationMetrics } from './evaluation-metrics.ee'; import { createPinData, getPastExecutionTriggerNode } from './utils.ee'; +interface TestRunMetadata { + testRunId: string; + userId: string; +} + +interface TestCaseRunMetadata extends TestRunMetadata { + pastExecutionId: string; +} + /** * This service orchestrates the running of test cases. * It uses the test definitions to find @@ -50,6 +61,7 @@ export class TestRunnerService { private readonly executionRepository: ExecutionRepository, private readonly activeExecutions: ActiveExecutions, private readonly testRunRepository: TestRunRepository, + private readonly testCaseExecutionRepository: TestCaseExecutionRepository, private readonly testMetricRepository: TestMetricRepository, private readonly nodeTypes: NodeTypes, private readonly errorReporter: ErrorReporter, @@ -105,7 +117,7 @@ export class TestRunnerService { pastExecutionData: IRunExecutionData, pastExecutionWorkflowData: IWorkflowBase, mockedNodes: MockedNodeItem[], - userId: string, + metadata: TestCaseRunMetadata, abortSignal: AbortSignal, ): Promise { // Do not run if the test run is cancelled @@ -128,7 +140,7 @@ export class TestRunnerService { runData: {}, pinData, workflowData: { ...workflow, pinData }, - userId, + userId: metadata.userId, partialExecutionVersion: '1', }; @@ -141,6 +153,13 @@ export class TestRunnerService { this.activeExecutions.stopExecution(executionId); }); + // Update status of the test run execution mapping + await this.testCaseExecutionRepository.markAsRunning( + metadata.testRunId, + metadata.pastExecutionId, + executionId, + ); + // Wait for the execution to finish const executePromise = this.activeExecutions.getPostExecutePromise(executionId); @@ -155,7 +174,7 @@ export class TestRunnerService { expectedData: IRunData, actualData: IRunData, abortSignal: AbortSignal, - testRunId?: string, + metadata: TestCaseRunMetadata, ) { // Do not run if the test run is cancelled if (abortSignal.aborted) { @@ -173,13 +192,6 @@ export class TestRunnerService { // Prepare the data to run the evaluation workflow const data = await getRunData(evaluationWorkflow, [evaluationInputData]); - // FIXME: This is a hack to add the testRunId to the evaluation workflow execution data - // So that we can fetch all execution runs for a test run - if (testRunId && data.executionData) { - data.executionData.resultData.metadata = { - testRunId, - }; - } data.executionMode = 'evaluation'; // Trigger the evaluation workflow @@ -191,6 +203,13 @@ export class TestRunnerService { this.activeExecutions.stopExecution(executionId); }); + // Update status of the test run execution mapping + await this.testCaseExecutionRepository.markAsEvaluationRunning( + metadata.testRunId, + metadata.pastExecutionId, + executionId, + ); + // Wait for the execution to finish const executePromise = this.activeExecutions.getPostExecutePromise(executionId); @@ -248,9 +267,18 @@ export class TestRunnerService { const abortController = new AbortController(); this.abortControllers.set(testRun.id, abortController); + // 0.2 Initialize metadata + // This will be passed to the test case executions + const testRunMetadata = { + testRunId: testRun.id, + userId: user.id, + }; + const abortSignal = abortController.signal; try { + /// // 1. Make test cases from previous executions + /// // Select executions with the annotation tag and workflow ID of the test. // Fetch only ids to reduce the data transfer. @@ -266,13 +294,22 @@ export class TestRunnerService { this.logger.debug('Found past executions', { count: pastExecutions.length }); + // Add all past executions mappings to the test run. + // This will be used to track the status of each test case and keep the connection between test run and all related executions (past, current, and evaluation). + await this.testCaseExecutionRepository.createBatch( + testRun.id, + pastExecutions.map((e) => e.id), + ); + // Get the metrics to collect from the evaluation workflow const testMetricNames = await this.getTestMetricNames(test.id); // 2. Run over all the test cases const pastExecutionIds = pastExecutions.map((e) => e.id); + // Update test run status await this.testRunRepository.markAsRunning(testRun.id, pastExecutions.length); + this.telemetry.track('User runs test', { user_id: user.id, test_id: test.id, @@ -282,9 +319,13 @@ export class TestRunnerService { evaluation_workflow_id: test.evaluationWorkflowId, }); - // Object to collect the results of the evaluation workflow executions + // Initialize object to collect the results of the evaluation workflow executions const metrics = new EvaluationMetrics(testMetricNames); + /// + // 2. Run over all the test cases + /// + for (const pastExecutionId of pastExecutionIds) { if (abortSignal.aborted) { this.logger.debug('Test run was cancelled', { @@ -306,13 +347,18 @@ export class TestRunnerService { const executionData = parse(pastExecution.executionData.data) as IRunExecutionData; + const testCaseMetadata = { + ...testRunMetadata, + pastExecutionId, + }; + // Run the test case and wait for it to finish const testCaseExecution = await this.runTestCase( workflow, executionData, pastExecution.executionData.workflowData, test.mockedNodes, - user.id, + testCaseMetadata, abortSignal, ); @@ -321,10 +367,18 @@ export class TestRunnerService { // In case of a permission check issue, the test case execution will be undefined. // Skip them, increment the failed count and continue with the next test case if (!testCaseExecution) { - await this.testRunRepository.incrementFailed(testRun.id); + await Db.transaction(async (trx) => { + await this.testRunRepository.incrementFailed(testRun.id, trx); + await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx); + }); continue; } + // Update status of the test case execution mapping entry in case of an error + if (testCaseExecution.data.resultData.error) { + await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId); + } + // Collect the results of the test case execution const testCaseRunData = testCaseExecution.data.resultData.runData; @@ -337,23 +391,37 @@ export class TestRunnerService { originalRunData, testCaseRunData, abortSignal, - testRun.id, + testCaseMetadata, ); assert(evalExecution); this.logger.debug('Evaluation execution finished', { pastExecutionId }); // Extract the output of the last node executed in the evaluation workflow - metrics.addResults(this.extractEvaluationResult(evalExecution)); + const addedMetrics = metrics.addResults(this.extractEvaluationResult(evalExecution)); if (evalExecution.data.resultData.error) { - await this.testRunRepository.incrementFailed(testRun.id); + await Db.transaction(async (trx) => { + await this.testRunRepository.incrementFailed(testRun.id, trx); + await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx); + }); } else { - await this.testRunRepository.incrementPassed(testRun.id); + await Db.transaction(async (trx) => { + await this.testRunRepository.incrementPassed(testRun.id, trx); + await this.testCaseExecutionRepository.markAsCompleted( + testRun.id, + pastExecutionId, + addedMetrics, + trx, + ); + }); } } catch (e) { // In case of an unexpected error, increment the failed count and continue with the next test case - await this.testRunRepository.incrementFailed(testRun.id); + await Db.transaction(async (trx) => { + await this.testRunRepository.incrementFailed(testRun.id, trx); + await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx); + }); this.errorReporter.error(e); } @@ -361,7 +429,10 @@ export class TestRunnerService { // Mark the test run as completed or cancelled if (abortSignal.aborted) { - await this.testRunRepository.markAsCancelled(testRun.id); + await Db.transaction(async (trx) => { + await this.testRunRepository.markAsCancelled(testRun.id, trx); + await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx); + }); } else { const aggregatedMetrics = metrics.getAggregatedMetrics(); await this.testRunRepository.markAsCompleted(testRun.id, aggregatedMetrics); @@ -375,7 +446,10 @@ export class TestRunnerService { stoppedOn: e.extra?.executionId, }); - await this.testRunRepository.markAsCancelled(testRun.id); + await Db.transaction(async (trx) => { + await this.testRunRepository.markAsCancelled(testRun.id, trx); + await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx); + }); } else { throw e; } @@ -402,8 +476,11 @@ export class TestRunnerService { abortController.abort(); this.abortControllers.delete(testRunId); } else { - // If there is no abort controller - just mark the test run as cancelled - await this.testRunRepository.markAsCancelled(testRunId); + // If there is no abort controller - just mark the test run and all its' pending test case executions as cancelled + await Db.transaction(async (trx) => { + await this.testRunRepository.markAsCancelled(testRunId, trx); + await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRunId, trx); + }); } } } diff --git a/packages/cli/src/evaluation.ee/test-runs.controller.ee.ts b/packages/cli/src/evaluation.ee/test-runs.controller.ee.ts index 7e95cb4dce..61853e6e42 100644 --- a/packages/cli/src/evaluation.ee/test-runs.controller.ee.ts +++ b/packages/cli/src/evaluation.ee/test-runs.controller.ee.ts @@ -1,6 +1,7 @@ import express from 'express'; import { InstanceSettings } from 'n8n-core'; +import { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee'; import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee'; import { Delete, Get, Post, RestController } from '@/decorators'; import { ConflictError } from '@/errors/response-errors/conflict.error'; @@ -18,6 +19,7 @@ export class TestRunsController { constructor( private readonly testDefinitionService: TestDefinitionService, private readonly testRunRepository: TestRunRepository, + private readonly testCaseExecutionRepository: TestCaseExecutionRepository, private readonly testRunnerService: TestRunnerService, private readonly instanceSettings: InstanceSettings, ) {} @@ -76,6 +78,16 @@ export class TestRunsController { return await this.getTestRun(req); } + @Get('/:testDefinitionId/runs/:id/cases') + async getTestCases(req: TestRunsRequest.GetCases) { + await this.getTestDefinition(req); + await this.getTestRun(req); + + return await this.testCaseExecutionRepository.find({ + where: { testRun: { id: req.params.id } }, + }); + } + @Delete('/:testDefinitionId/runs/:id') async delete(req: TestRunsRequest.Delete) { const { id: testRunId } = req.params; diff --git a/packages/editor-ui/src/api/testDefinition.ee.ts b/packages/editor-ui/src/api/testDefinition.ee.ts index 1cc39e3bd5..52e344d608 100644 --- a/packages/editor-ui/src/api/testDefinition.ee.ts +++ b/packages/editor-ui/src/api/testDefinition.ee.ts @@ -61,6 +61,21 @@ interface DeleteTestRunParams { runId: string; } +export interface TestCaseExecutionRecord { + id: string; + testRunId: string; + executionId: string; + pastExecutionId: string; + evaluationExecutionId: string; + status: 'running' | 'completed' | 'error'; + createdAt: string; + updatedAt: string; + runAt: string; + metrics?: Record; + errorCode?: string; + errorDetails?: Record; +} + const endpoint = '/evaluation/test-definitions'; const getMetricsEndpoint = (testDefinitionId: string, metricId?: string) => `${endpoint}/${testDefinitionId}/metrics${metricId ? `/${metricId}` : ''}`; @@ -244,3 +259,19 @@ export const deleteTestRun = async (context: IRestApiContext, params: DeleteTest getRunsEndpoint(params.testDefinitionId, params.runId), ); }; + +const getRunExecutionsEndpoint = (testDefinitionId: string, runId: string) => + `${endpoint}/${testDefinitionId}/runs/${runId}/cases`; + +// Get all test cases of a test run +export const getTestCaseExecutions = async ( + context: IRestApiContext, + testDefinitionId: string, + runId: string, +) => { + return await makeRestApiRequest( + context, + 'GET', + getRunExecutionsEndpoint(testDefinitionId, runId), + ); +}; diff --git a/packages/editor-ui/src/components/TestDefinition/shared/TableCell.vue b/packages/editor-ui/src/components/TestDefinition/shared/TableCell.vue index 24913d2f95..70a20e27ec 100644 --- a/packages/editor-ui/src/components/TestDefinition/shared/TableCell.vue +++ b/packages/editor-ui/src/components/TestDefinition/shared/TableCell.vue @@ -23,8 +23,9 @@ function hasStatus(row: unknown): row is WithStatus { } const statusThemeMap: Record = { - new: 'info', + new: 'default', running: 'warning', + evaluation_running: 'warning', completed: 'success', error: 'danger', success: 'success', @@ -34,6 +35,7 @@ const statusThemeMap: Record = { const statusLabelMap: Record = { new: locale.baseText('testDefinition.listRuns.status.new'), running: locale.baseText('testDefinition.listRuns.status.running'), + evaluation_running: locale.baseText('testDefinition.listRuns.status.evaluating'), completed: locale.baseText('testDefinition.listRuns.status.completed'), error: locale.baseText('testDefinition.listRuns.status.error'), success: locale.baseText('testDefinition.listRuns.status.success'), @@ -53,11 +55,11 @@ const getCellContent = (column: TestTableColumn, row: T) => {