feat(core): Keep track of test case executions during test run (no-changelog) (#12787)

packages/cli/src/databases/entities/index.ts

@@ -20,6 +20,7 @@ import { Settings } from './settings';
 import { SharedCredentials } from './shared-credentials';
 import { SharedWorkflow } from './shared-workflow';
 import { TagEntity } from './tag-entity';
+import { TestCaseExecution } from './test-case-execution.ee';
 import { TestDefinition } from './test-definition.ee';
 import { TestMetric } from './test-metric.ee';
 import { TestRun } from './test-run.ee';
@@ -64,4 +65,5 @@ export const entities = {
 	TestDefinition,
 	TestMetric,
 	TestRun,
+	TestCaseExecution,
 };

packages/cli/src/databases/entities/test-case-execution.ee.ts

@@ -0,0 +1,68 @@
+import { Column, Entity, ManyToOne, OneToOne } from '@n8n/typeorm';
+
+import {
+	datetimeColumnType,
+	jsonColumnType,
+	WithStringId,
+} from '@/databases/entities/abstract-entity';
+import type { ExecutionEntity } from '@/databases/entities/execution-entity';
+import { TestRun } from '@/databases/entities/test-run.ee';
+
+export type TestCaseRunMetrics = Record<string, number | boolean>;
+
+/**
+ * This entity represents the linking between the test runs and individual executions.
+ * It stores status, links to past, new and evaluation executions, and metrics produced by individual evaluation wf executions
+ * Entries in this table are meant to outlive the execution entities, which might be pruned over time.
+ * This allows us to keep track of the details of test runs' status and metrics even after the executions are deleted.
+ */
+@Entity({ name: 'test_case_execution' })
+export class TestCaseExecution extends WithStringId {
+	@ManyToOne('TestRun')
+	testRun: TestRun;
+
+	@ManyToOne('ExecutionEntity', {
+		onDelete: 'SET NULL',
+		nullable: true,
+	})
+	pastExecution: ExecutionEntity | null;
+
+	@Column({ type: 'varchar', nullable: true })
+	pastExecutionId: string | null;
+
+	@OneToOne('ExecutionEntity', {
+		onDelete: 'SET NULL',
+		nullable: true,
+	})
+	execution: ExecutionEntity | null;
+
+	@Column({ type: 'varchar', nullable: true })
+	executionId: string | null;
+
+	@OneToOne('ExecutionEntity', {
+		onDelete: 'SET NULL',
+		nullable: true,
+	})
+	evaluationExecution: ExecutionEntity | null;
+
+	@Column({ type: 'varchar', nullable: true })
+	evaluationExecutionId: string | null;
+
+	@Column()
+	status: 'new' | 'running' | 'evaluation_running' | 'success' | 'error' | 'cancelled';
+
+	@Column({ type: datetimeColumnType, nullable: true })
+	runAt: Date | null;
+
+	@Column({ type: datetimeColumnType, nullable: true })
+	completedAt: Date | null;
+
+	@Column('varchar', { nullable: true })
+	errorCode: string | null;
+
+	@Column(jsonColumnType, { nullable: true })
+	errorDetails: Record<string, unknown>;
+
+	@Column(jsonColumnType, { nullable: true })
+	metrics: TestCaseRunMetrics;
+}

packages/cli/src/databases/migrations/common/1736947513045-CreateTestCaseExecutionTable.ts

@@ -0,0 +1,47 @@
+import type { MigrationContext, ReversibleMigration } from '@/databases/types';
+
+const testCaseExecutionTableName = 'test_case_execution';
+
+export class CreateTestCaseExecutionTable1736947513045 implements ReversibleMigration {
+	async up({ schemaBuilder: { createTable, column } }: MigrationContext) {
+		await createTable(testCaseExecutionTableName)
+			.withColumns(
+				column('id').varchar(36).primary.notNull,
+				column('testRunId').varchar(36).notNull,
+				column('pastExecutionId').int, // Might be null if execution was deleted after the test run
+				column('executionId').int, // Execution of the workflow under test. Might be null if execution was deleted after the test run
+				column('evaluationExecutionId').int, // Execution of the evaluation workflow. Might be null if execution was deleted after the test run, or if the test run was cancelled
+				column('status').varchar().notNull,
+				column('runAt').timestamp(),
+				column('completedAt').timestamp(),
+				column('errorCode').varchar(),
+				column('errorDetails').json,
+				column('metrics').json,
+			)
+			.withIndexOn('testRunId')
+			.withForeignKey('testRunId', {
+				tableName: 'test_run',
+				columnName: 'id',
+				onDelete: 'CASCADE',
+			})
+			.withForeignKey('pastExecutionId', {
+				tableName: 'execution_entity',
+				columnName: 'id',
+				onDelete: 'SET NULL',
+			})
+			.withForeignKey('executionId', {
+				tableName: 'execution_entity',
+				columnName: 'id',
+				onDelete: 'SET NULL',
+			})
+			.withForeignKey('evaluationExecutionId', {
+				tableName: 'execution_entity',
+				columnName: 'id',
+				onDelete: 'SET NULL',
+			}).withTimestamps;
+	}
+
+	async down({ schemaBuilder: { dropTable } }: MigrationContext) {
+		await dropTable(testCaseExecutionTableName);
+	}
+}

packages/cli/src/databases/migrations/mysqldb/index.ts

@@ -77,6 +77,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
+import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
 
 export const mysqlMigrations: Migration[] = [
 	InitialMigration1588157391238,
@@ -156,4 +157,5 @@ export const mysqlMigrations: Migration[] = [
 	AddManagedColumnToCredentialsTable1734479635324,
 	AddProjectIcons1729607673469,
 	AddStatsColumnsToTestRun1736172058779,
+	CreateTestCaseExecutionTable1736947513045,
 ];

packages/cli/src/databases/migrations/postgresdb/index.ts

@@ -77,6 +77,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
+import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
 
 export const postgresMigrations: Migration[] = [
 	InitialMigration1587669153312,
@@ -156,4 +157,5 @@ export const postgresMigrations: Migration[] = [
 	AddManagedColumnToCredentialsTable1734479635324,
 	AddProjectIcons1729607673469,
 	AddStatsColumnsToTestRun1736172058779,
+	CreateTestCaseExecutionTable1736947513045,
 ];

packages/cli/src/databases/migrations/sqlite/index.ts

@@ -74,6 +74,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
+import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
 
 const sqliteMigrations: Migration[] = [
 	InitialMigration1588102412422,
@@ -150,6 +151,7 @@ const sqliteMigrations: Migration[] = [
 	AddManagedColumnToCredentialsTable1734479635324,
 	AddProjectIcons1729607673469,
 	AddStatsColumnsToTestRun1736172058779,
+	CreateTestCaseExecutionTable1736947513045,
 ];
 
 export { sqliteMigrations };

packages/cli/src/databases/repositories/test-case-execution.repository.ee.ts

@@ -0,0 +1,99 @@
+import { Service } from '@n8n/di';
+import type { EntityManager } from '@n8n/typeorm';
+import { DataSource, In, Not, Repository } from '@n8n/typeorm';
+import type { DeepPartial } from '@n8n/typeorm/common/DeepPartial';
+
+import { TestCaseExecution } from '@/databases/entities/test-case-execution.ee';
+
+@Service()
+export class TestCaseExecutionRepository extends Repository<TestCaseExecution> {
+	constructor(dataSource: DataSource) {
+		super(TestCaseExecution, dataSource.manager);
+	}
+
+	async createBatch(testRunId: string, pastExecutionIds: string[]) {
+		const mappings = this.create(
+			pastExecutionIds.map<DeepPartial<TestCaseExecution>>((id) => ({
+				testRun: {
+					id: testRunId,
+				},
+				pastExecution: {
+					id,
+				},
+				status: 'new',
+			})),
+		);
+
+		return await this.save(mappings);
+	}
+
+	async markAsRunning(testRunId: string, pastExecutionId: string, executionId: string) {
+		return await this.update(
+			{ testRun: { id: testRunId }, pastExecutionId },
+			{
+				status: 'running',
+				executionId,
+				runAt: new Date(),
+			},
+		);
+	}
+
+	async markAsEvaluationRunning(
+		testRunId: string,
+		pastExecutionId: string,
+		evaluationExecutionId: string,
+	) {
+		return await this.update(
+			{ testRun: { id: testRunId }, pastExecutionId },
+			{
+				status: 'evaluation_running',
+				evaluationExecutionId,
+			},
+		);
+	}
+
+	async markAsCompleted(
+		testRunId: string,
+		pastExecutionId: string,
+		metrics: Record<string, number>,
+		trx?: EntityManager,
+	) {
+		trx = trx ?? this.manager;
+
+		return await trx.update(
+			TestCaseExecution,
+			{ testRun: { id: testRunId }, pastExecutionId },
+			{
+				status: 'success',
+				completedAt: new Date(),
+				metrics,
+			},
+		);
+	}
+
+	async markAllPendingAsCancelled(testRunId: string, trx?: EntityManager) {
+		trx = trx ?? this.manager;
+
+		return await trx.update(
+			TestCaseExecution,
+			{ testRun: { id: testRunId }, status: Not(In(['success', 'error', 'cancelled'])) },
+			{
+				status: 'cancelled',
+				completedAt: new Date(),
+			},
+		);
+	}
+
+	async markAsFailed(testRunId: string, pastExecutionId: string, trx?: EntityManager) {
+		trx = trx ?? this.manager;
+
+		return await trx.update(
+			TestCaseExecution,
+			{ testRun: { id: testRunId }, pastExecutionId },
+			{
+				status: 'error',
+				completedAt: new Date(),
+			},
+		);
+	}
+}

packages/cli/src/databases/repositories/test-run.repository.ee.ts

@@ -1,5 +1,5 @@
 import { Service } from '@n8n/di';
-import type { FindManyOptions } from '@n8n/typeorm';
+import type { EntityManager, FindManyOptions } from '@n8n/typeorm';
 import { DataSource, Repository } from '@n8n/typeorm';
 
 import type { AggregatedTestRunMetrics } from '@/databases/entities/test-run.ee';
@@ -35,16 +35,19 @@ export class TestRunRepository extends Repository<TestRun> {
 		return await this.update(id, { status: 'completed', completedAt: new Date(), metrics });
 	}
 
-	async markAsCancelled(id: string) {
+	async markAsCancelled(id: string, trx?: EntityManager) {
-		return await this.update(id, { status: 'cancelled' });
+		trx = trx ?? this.manager;
+		return await trx.update(TestRun, id, { status: 'cancelled' });
 	}
 
-	async incrementPassed(id: string) {
+	async incrementPassed(id: string, trx?: EntityManager) {
-		return await this.increment({ id }, 'passedCases', 1);
+		trx = trx ?? this.manager;
+		return await trx.increment(TestRun, { id }, 'passedCases', 1);
 	}
 
-	async incrementFailed(id: string) {
+	async incrementFailed(id: string, trx?: EntityManager) {
-		return await this.increment({ id }, 'failedCases', 1);
+		trx = trx ?? this.manager;
+		return await trx.increment(TestRun, { id }, 'failedCases', 1);
 	}
 
 	async getMany(testDefinitionId: string, options: ListQuery.Options) {

packages/cli/src/evaluation.ee/test-definitions.types.ee.ts

@@ -94,4 +94,6 @@ export declare namespace TestRunsRequest {
 	type Delete = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
 
 	type Cancel = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
+
+	type GetCases = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
 }

packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts

@@ -13,6 +13,7 @@ import type { TestMetric } from '@/databases/entities/test-metric.ee';
 import type { TestRun } from '@/databases/entities/test-run.ee';
 import type { User } from '@/databases/entities/user';
 import type { ExecutionRepository } from '@/databases/repositories/execution.repository';
+import type { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee';
 import type { TestMetricRepository } from '@/databases/repositories/test-metric.repository.ee';
 import type { TestRunRepository } from '@/databases/repositories/test-run.repository.ee';
 import type { WorkflowRepository } from '@/databases/repositories/workflow.repository';
@@ -25,6 +26,10 @@ import { mockNodeTypesData } from '@test-integration/utils/node-types-data';
 
 import { TestRunnerService } from '../test-runner.service.ee';
 
+jest.mock('@/db', () => ({
+	transaction: (cb: any) => cb(),
+}));
+
 const wfUnderTestJson = JSON.parse(
 	readFileSync(path.join(__dirname, './mock-data/workflow.under-test.json'), { encoding: 'utf-8' }),
 );
@@ -147,6 +152,7 @@ describe('TestRunnerService', () => {
 	const activeExecutions = mock<ActiveExecutions>();
 	const testRunRepository = mock<TestRunRepository>();
 	const testMetricRepository = mock<TestMetricRepository>();
+	const testCaseExecutionRepository = mock<TestCaseExecutionRepository>();
 
 	const mockNodeTypes = mockInstance(NodeTypes);
 	mockInstance(LoadNodesAndCredentials, {
@@ -190,6 +196,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -207,6 +214,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -247,6 +255,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -350,6 +359,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -410,6 +420,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -466,6 +477,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -526,6 +538,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -602,6 +615,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -629,6 +643,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -661,9 +676,10 @@ describe('TestRunnerService', () => {
 				executionRepository,
 				activeExecutions,
 				testRunRepository,
+				testCaseExecutionRepository,
 				testMetricRepository,
 				mockNodeTypes,
-				mock<ErrorReporter>(),
+				errorReporter,
 			);
 
 			workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({

packages/cli/src/evaluation.ee/test-runner/evaluation-metrics.ee.ts

@@ -9,12 +9,17 @@ export class EvaluationMetrics {
 		}
 	}
 
-	addResults(result: IDataObject) {
+	addResults(result: IDataObject): Record<string, number> {
+		const addedMetrics: Record<string, number> = {};
+
 		for (const [metricName, metricValue] of Object.entries(result)) {
 			if (typeof metricValue === 'number' && this.metricNames.has(metricName)) {
+				addedMetrics[metricName] = metricValue;
 				this.rawMetricsByName.get(metricName)!.push(metricValue);
 			}
 		}
+
+		return addedMetrics;
 	}
 
 	getAggregatedMetrics() {

packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts

@@ -19,9 +19,11 @@ import type { TestRun } from '@/databases/entities/test-run.ee';
 import type { User } from '@/databases/entities/user';
 import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
 import { ExecutionRepository } from '@/databases/repositories/execution.repository';
+import { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee';
 import { TestMetricRepository } from '@/databases/repositories/test-metric.repository.ee';
 import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee';
 import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
+import * as Db from '@/db';
 import { NodeTypes } from '@/node-types';
 import { Telemetry } from '@/telemetry';
 import { getRunData } from '@/workflow-execute-additional-data';
@@ -30,6 +32,15 @@ import { WorkflowRunner } from '@/workflow-runner';
 import { EvaluationMetrics } from './evaluation-metrics.ee';
 import { createPinData, getPastExecutionTriggerNode } from './utils.ee';
 
+interface TestRunMetadata {
+	testRunId: string;
+	userId: string;
+}
+
+interface TestCaseRunMetadata extends TestRunMetadata {
+	pastExecutionId: string;
+}
+
 /**
  * This service orchestrates the running of test cases.
  * It uses the test definitions to find
@@ -50,6 +61,7 @@ export class TestRunnerService {
 		private readonly executionRepository: ExecutionRepository,
 		private readonly activeExecutions: ActiveExecutions,
 		private readonly testRunRepository: TestRunRepository,
+		private readonly testCaseExecutionRepository: TestCaseExecutionRepository,
 		private readonly testMetricRepository: TestMetricRepository,
 		private readonly nodeTypes: NodeTypes,
 		private readonly errorReporter: ErrorReporter,
@@ -105,7 +117,7 @@ export class TestRunnerService {
 		pastExecutionData: IRunExecutionData,
 		pastExecutionWorkflowData: IWorkflowBase,
 		mockedNodes: MockedNodeItem[],
-		userId: string,
+		metadata: TestCaseRunMetadata,
 		abortSignal: AbortSignal,
 	): Promise<IRun | undefined> {
 		// Do not run if the test run is cancelled
@@ -128,7 +140,7 @@ export class TestRunnerService {
 			runData: {},
 			pinData,
 			workflowData: { ...workflow, pinData },
-			userId,
+			userId: metadata.userId,
 			partialExecutionVersion: '1',
 		};
 
@@ -141,6 +153,13 @@ export class TestRunnerService {
 			this.activeExecutions.stopExecution(executionId);
 		});
 
+		// Update status of the test run execution mapping
+		await this.testCaseExecutionRepository.markAsRunning(
+			metadata.testRunId,
+			metadata.pastExecutionId,
+			executionId,
+		);
+
 		// Wait for the execution to finish
 		const executePromise = this.activeExecutions.getPostExecutePromise(executionId);
 
@@ -155,7 +174,7 @@ export class TestRunnerService {
 		expectedData: IRunData,
 		actualData: IRunData,
 		abortSignal: AbortSignal,
-		testRunId?: string,
+		metadata: TestCaseRunMetadata,
 	) {
 		// Do not run if the test run is cancelled
 		if (abortSignal.aborted) {
@@ -173,13 +192,6 @@ export class TestRunnerService {
 
 		// Prepare the data to run the evaluation workflow
 		const data = await getRunData(evaluationWorkflow, [evaluationInputData]);
-		// FIXME: This is a hack to add the testRunId to the evaluation workflow execution data
-		// So that we can fetch all execution runs for a test run
-		if (testRunId && data.executionData) {
-			data.executionData.resultData.metadata = {
-				testRunId,
-			};
-		}
 		data.executionMode = 'evaluation';
 
 		// Trigger the evaluation workflow
@@ -191,6 +203,13 @@ export class TestRunnerService {
 			this.activeExecutions.stopExecution(executionId);
 		});
 
+		// Update status of the test run execution mapping
+		await this.testCaseExecutionRepository.markAsEvaluationRunning(
+			metadata.testRunId,
+			metadata.pastExecutionId,
+			executionId,
+		);
+
 		// Wait for the execution to finish
 		const executePromise = this.activeExecutions.getPostExecutePromise(executionId);
 
@@ -248,9 +267,18 @@ export class TestRunnerService {
 		const abortController = new AbortController();
 		this.abortControllers.set(testRun.id, abortController);
 
+		// 0.2 Initialize metadata
+		// This will be passed to the test case executions
+		const testRunMetadata = {
+			testRunId: testRun.id,
+			userId: user.id,
+		};
+
 		const abortSignal = abortController.signal;
 		try {
+			///
 			// 1. Make test cases from previous executions
+			///
 
 			// Select executions with the annotation tag and workflow ID of the test.
 			// Fetch only ids to reduce the data transfer.
@@ -266,13 +294,22 @@ export class TestRunnerService {
 
 			this.logger.debug('Found past executions', { count: pastExecutions.length });
 
+			// Add all past executions mappings to the test run.
+			// This will be used to track the status of each test case and keep the connection between test run and all related executions (past, current, and evaluation).
+			await this.testCaseExecutionRepository.createBatch(
+				testRun.id,
+				pastExecutions.map((e) => e.id),
+			);
+
 			// Get the metrics to collect from the evaluation workflow
 			const testMetricNames = await this.getTestMetricNames(test.id);
 
 			// 2. Run over all the test cases
 			const pastExecutionIds = pastExecutions.map((e) => e.id);
 
+			// Update test run status
 			await this.testRunRepository.markAsRunning(testRun.id, pastExecutions.length);
+
 			this.telemetry.track('User runs test', {
 				user_id: user.id,
 				test_id: test.id,
@@ -282,9 +319,13 @@ export class TestRunnerService {
 				evaluation_workflow_id: test.evaluationWorkflowId,
 			});
 
-			// Object to collect the results of the evaluation workflow executions
+			// Initialize object to collect the results of the evaluation workflow executions
 			const metrics = new EvaluationMetrics(testMetricNames);
 
+			///
+			// 2. Run over all the test cases
+			///
+
 			for (const pastExecutionId of pastExecutionIds) {
 				if (abortSignal.aborted) {
 					this.logger.debug('Test run was cancelled', {
@@ -306,13 +347,18 @@ export class TestRunnerService {
 
 					const executionData = parse(pastExecution.executionData.data) as IRunExecutionData;
 
+					const testCaseMetadata = {
+						...testRunMetadata,
+						pastExecutionId,
+					};
+
 					// Run the test case and wait for it to finish
 					const testCaseExecution = await this.runTestCase(
 						workflow,
 						executionData,
 						pastExecution.executionData.workflowData,
 						test.mockedNodes,
-						user.id,
+						testCaseMetadata,
 						abortSignal,
 					);
 
@@ -321,10 +367,18 @@ export class TestRunnerService {
 					// In case of a permission check issue, the test case execution will be undefined.
 					// Skip them, increment the failed count and continue with the next test case
 					if (!testCaseExecution) {
-						await this.testRunRepository.incrementFailed(testRun.id);
+						await Db.transaction(async (trx) => {
+							await this.testRunRepository.incrementFailed(testRun.id, trx);
+							await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx);
+						});
 						continue;
 					}
 
+					// Update status of the test case execution mapping entry in case of an error
+					if (testCaseExecution.data.resultData.error) {
+						await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId);
+					}
+
 					// Collect the results of the test case execution
 					const testCaseRunData = testCaseExecution.data.resultData.runData;
 
@@ -337,23 +391,37 @@ export class TestRunnerService {
 						originalRunData,
 						testCaseRunData,
 						abortSignal,
-						testRun.id,
+						testCaseMetadata,
 					);
 					assert(evalExecution);
 
 					this.logger.debug('Evaluation execution finished', { pastExecutionId });
 
 					// Extract the output of the last node executed in the evaluation workflow
-					metrics.addResults(this.extractEvaluationResult(evalExecution));
+					const addedMetrics = metrics.addResults(this.extractEvaluationResult(evalExecution));
 
 					if (evalExecution.data.resultData.error) {
-						await this.testRunRepository.incrementFailed(testRun.id);
+						await Db.transaction(async (trx) => {
+							await this.testRunRepository.incrementFailed(testRun.id, trx);
+							await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx);
+						});
 					} else {
-						await this.testRunRepository.incrementPassed(testRun.id);
+						await Db.transaction(async (trx) => {
+							await this.testRunRepository.incrementPassed(testRun.id, trx);
+							await this.testCaseExecutionRepository.markAsCompleted(
+								testRun.id,
+								pastExecutionId,
+								addedMetrics,
+								trx,
+							);
+						});
 					}
 				} catch (e) {
 					// In case of an unexpected error, increment the failed count and continue with the next test case
-					await this.testRunRepository.incrementFailed(testRun.id);
+					await Db.transaction(async (trx) => {
+						await this.testRunRepository.incrementFailed(testRun.id, trx);
+						await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx);
+					});
 
 					this.errorReporter.error(e);
 				}
@@ -361,7 +429,10 @@ export class TestRunnerService {
 
 			// Mark the test run as completed or cancelled
 			if (abortSignal.aborted) {
-				await this.testRunRepository.markAsCancelled(testRun.id);
+				await Db.transaction(async (trx) => {
+					await this.testRunRepository.markAsCancelled(testRun.id, trx);
+					await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx);
+				});
 			} else {
 				const aggregatedMetrics = metrics.getAggregatedMetrics();
 				await this.testRunRepository.markAsCompleted(testRun.id, aggregatedMetrics);
@@ -375,7 +446,10 @@ export class TestRunnerService {
 					stoppedOn: e.extra?.executionId,
 				});
 
-				await this.testRunRepository.markAsCancelled(testRun.id);
+				await Db.transaction(async (trx) => {
+					await this.testRunRepository.markAsCancelled(testRun.id, trx);
+					await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx);
+				});
 			} else {
 				throw e;
 			}
@@ -402,8 +476,11 @@ export class TestRunnerService {
 			abortController.abort();
 			this.abortControllers.delete(testRunId);
 		} else {
-			// If there is no abort controller - just mark the test run as cancelled
+			// If there is no abort controller - just mark the test run and all its' pending test case executions as cancelled
-			await this.testRunRepository.markAsCancelled(testRunId);
+			await Db.transaction(async (trx) => {
+				await this.testRunRepository.markAsCancelled(testRunId, trx);
+				await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRunId, trx);
+			});
 		}
 	}
 }

packages/cli/src/evaluation.ee/test-runs.controller.ee.ts

@@ -1,6 +1,7 @@
 import express from 'express';
 import { InstanceSettings } from 'n8n-core';
 
+import { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee';
 import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee';
 import { Delete, Get, Post, RestController } from '@/decorators';
 import { ConflictError } from '@/errors/response-errors/conflict.error';
@@ -18,6 +19,7 @@ export class TestRunsController {
 	constructor(
 		private readonly testDefinitionService: TestDefinitionService,
 		private readonly testRunRepository: TestRunRepository,
+		private readonly testCaseExecutionRepository: TestCaseExecutionRepository,
 		private readonly testRunnerService: TestRunnerService,
 		private readonly instanceSettings: InstanceSettings,
 	) {}
@@ -76,6 +78,16 @@ export class TestRunsController {
 		return await this.getTestRun(req);
 	}
 
+	@Get('/:testDefinitionId/runs/:id/cases')
+	async getTestCases(req: TestRunsRequest.GetCases) {
+		await this.getTestDefinition(req);
+		await this.getTestRun(req);
+
+		return await this.testCaseExecutionRepository.find({
+			where: { testRun: { id: req.params.id } },
+		});
+	}
+
 	@Delete('/:testDefinitionId/runs/:id')
 	async delete(req: TestRunsRequest.Delete) {
 		const { id: testRunId } = req.params;

packages/editor-ui/src/api/testDefinition.ee.ts

@@ -61,6 +61,21 @@ interface DeleteTestRunParams {
 	runId: string;
 }
 
+export interface TestCaseExecutionRecord {
+	id: string;
+	testRunId: string;
+	executionId: string;
+	pastExecutionId: string;
+	evaluationExecutionId: string;
+	status: 'running' | 'completed' | 'error';
+	createdAt: string;
+	updatedAt: string;
+	runAt: string;
+	metrics?: Record<string, number>;
+	errorCode?: string;
+	errorDetails?: Record<string, unknown>;
+}
+
 const endpoint = '/evaluation/test-definitions';
 const getMetricsEndpoint = (testDefinitionId: string, metricId?: string) =>
 	`${endpoint}/${testDefinitionId}/metrics${metricId ? `/${metricId}` : ''}`;
@@ -244,3 +259,19 @@ export const deleteTestRun = async (context: IRestApiContext, params: DeleteTest
 		getRunsEndpoint(params.testDefinitionId, params.runId),
 	);
 };
+
+const getRunExecutionsEndpoint = (testDefinitionId: string, runId: string) =>
+	`${endpoint}/${testDefinitionId}/runs/${runId}/cases`;
+
+// Get all test cases of a test run
+export const getTestCaseExecutions = async (
+	context: IRestApiContext,
+	testDefinitionId: string,
+	runId: string,
+) => {
+	return await makeRestApiRequest<TestCaseExecutionRecord[]>(
+		context,
+		'GET',
+		getRunExecutionsEndpoint(testDefinitionId, runId),
+	);
+};

packages/editor-ui/src/components/TestDefinition/shared/TableCell.vue

@@ -23,8 +23,9 @@ function hasStatus(row: unknown): row is WithStatus {
 }
 
 const statusThemeMap: Record<string, string> = {
-	new: 'info',
+	new: 'default',
 	running: 'warning',
+	evaluation_running: 'warning',
 	completed: 'success',
 	error: 'danger',
 	success: 'success',
@@ -34,6 +35,7 @@ const statusThemeMap: Record<string, string> = {
 const statusLabelMap: Record<string, string> = {
 	new: locale.baseText('testDefinition.listRuns.status.new'),
 	running: locale.baseText('testDefinition.listRuns.status.running'),
+	evaluation_running: locale.baseText('testDefinition.listRuns.status.evaluating'),
 	completed: locale.baseText('testDefinition.listRuns.status.completed'),
 	error: locale.baseText('testDefinition.listRuns.status.error'),
 	success: locale.baseText('testDefinition.listRuns.status.success'),
@@ -53,11 +55,11 @@ const getCellContent = (column: TestTableColumn<T>, row: T) => {
 </script>
 
 <template>
-	<div v-if="column.route">
+	<div v-if="column.route?.(row)">
-		<a v-if="column.openInNewTab" :href="router.resolve(column.route(row)).href" target="_blank">
+		<a v-if="column.openInNewTab" :href="router.resolve(column.route(row)!).href" target="_blank">
 			{{ getCellContent(column, row) }}
 		</a>
-		<router-link v-else :to="column.route(row)">
+		<router-link v-else :to="column.route(row)!">
 			{{ getCellContent(column, row) }}
 		</router-link>
 	</div>

packages/editor-ui/src/components/TestDefinition/shared/TestTableBase.vue

@@ -21,7 +21,7 @@ export type TestTableColumn<TRow> = {
 	sortable?: boolean;
 	filters?: Array<{ text: string; value: string }>;
 	filterMethod?: (value: string, row: TRow) => boolean;
-	route?: (row: TRow) => RouteLocationRaw;
+	route?: (row: TRow) => RouteLocationRaw | undefined;
 	sortMethod?: (a: TRow, b: TRow) => number;
 	openInNewTab?: boolean;
 	formatter?: (row: TRow) => string;

packages/editor-ui/src/plugins/i18n/locales/en.json

@@ -2854,6 +2854,7 @@
 	"testDefinition.list.loadError": "Failed to load tests",
 	"testDefinition.listRuns.status.new": "New",
 	"testDefinition.listRuns.status.running": "Running",
+	"testDefinition.listRuns.status.evaluating": "Evaluating",
 	"testDefinition.listRuns.status.completed": "Completed",
 	"testDefinition.listRuns.status.cancelled": "Cancelled",
 	"testDefinition.listRuns.status.error": "Error",

packages/editor-ui/src/stores/testDefinition.store.ee.ts

@@ -2,7 +2,11 @@ import { defineStore } from 'pinia';
 import { computed, ref } from 'vue';
 import { useRootStore } from './root.store';
 import * as testDefinitionsApi from '@/api/testDefinition.ee';
-import type { TestDefinitionRecord, TestRunRecord } from '@/api/testDefinition.ee';
+import type {
+	TestCaseExecutionRecord,
+	TestDefinitionRecord,
+	TestRunRecord,
+} from '@/api/testDefinition.ee';
 import { usePostHog } from './posthog.store';
 import { STORES, WORKFLOW_EVALUATION_EXPERIMENT } from '@/constants';
 import { useAnnotationTagsStore } from './tags.store';
@@ -19,6 +23,7 @@ export const useTestDefinitionStore = defineStore(
 		const fetchedAll = ref(false);
 		const metricsById = ref<Record<string, testDefinitionsApi.TestMetricRecord>>({});
 		const testRunsById = ref<Record<string, TestRunRecord>>({});
+		const testCaseExecutionsById = ref<Record<string, TestCaseExecutionRecord>>({});
 		const pollingTimeouts = ref<Record<string, NodeJS.Timeout>>({});
 		const fieldsIssues = ref<Record<string, FieldIssue[]>>({});
 
@@ -167,6 +172,20 @@ export const useTestDefinitionStore = defineStore(
 			return testDefinition;
 		};
 
+		const fetchTestCaseExecutions = async (params: { testDefinitionId: string; runId: string }) => {
+			const testCaseExecutions = await testDefinitionsApi.getTestCaseExecutions(
+				rootStore.restApiContext,
+				params.testDefinitionId,
+				params.runId,
+			);
+
+			testCaseExecutions.forEach((testCaseExecution) => {
+				testCaseExecutionsById.value[testCaseExecution.id] = testCaseExecution;
+			});
+
+			return testCaseExecutions;
+		};
+
 		/**
 		 * Fetches all test definitions from the API.
 		 * @param {boolean} force - If true, fetches the definitions from the API even if they were already fetched before.
@@ -421,6 +440,7 @@ export const useTestDefinitionStore = defineStore(
 			fetchedAll,
 			testDefinitionsById,
 			testRunsById,
+			testCaseExecutionsById,
 
 			// Computed
 			allTestDefinitions,
@@ -435,6 +455,7 @@ export const useTestDefinitionStore = defineStore(
 
 			// Methods
 			fetchTestDefinition,
+			fetchTestCaseExecutions,
 			fetchAll,
 			create,
 			update,

packages/editor-ui/src/views/TestDefinition/TestDefinitionRunDetailView.vue

@@ -7,24 +7,17 @@ import { useI18n } from '@/composables/useI18n';
 import { N8nCard, N8nText } from 'n8n-design-system';
 import TestTableBase from '@/components/TestDefinition/shared/TestTableBase.vue';
 import type { TestTableColumn } from '@/components/TestDefinition/shared/TestTableBase.vue';
-import { useExecutionsStore } from '@/stores/executions.store';
-import { get } from 'lodash-es';
-import type { ExecutionSummaryWithScopes } from '@/Interface';
 import { VIEWS } from '@/constants';
 import { useToast } from '@/composables/useToast';
-
+import type { TestCaseExecutionRecord } from '@/api/testDefinition.ee';
-interface TestCase extends ExecutionSummaryWithScopes {
-	metrics: Record<string, number>;
-}
 
 const router = useRouter();
 const toast = useToast();
 const testDefinitionStore = useTestDefinitionStore();
-const executionsStore = useExecutionsStore();
 const locale = useI18n();
 
 const isLoading = ref(true);
-const testCases = ref<TestCase[]>([]);
+const testCases = ref<TestCaseExecutionRecord[]>([]);
 
 const runId = computed(() => router.currentRoute.value.params.runId as string);
 const testId = computed(() => router.currentRoute.value.params.testId as string);
@@ -36,17 +29,26 @@ const filteredTestCases = computed(() => {
 });
 
 const columns = computed(
-	(): Array<TestTableColumn<TestCase>> => [
+	(): Array<TestTableColumn<TestCaseExecutionRecord>> => [
 		{
 			prop: 'id',
 			width: 200,
 			label: locale.baseText('testDefinition.runDetail.testCase'),
 			sortable: true,
-			route: (row: TestCase) => ({
+			route: (row: TestCaseExecutionRecord) => {
-				name: VIEWS.EXECUTION_PREVIEW,
+				if (test.value?.evaluationWorkflowId && row.evaluationExecutionId) {
-				params: { name: row.workflowId, executionId: row.id },
+					return {
-			}),
+						name: VIEWS.EXECUTION_PREVIEW,
-			formatter: (row: TestCase) => `[${row.id}] ${row.workflowName}`,
+						params: {
+							name: test.value?.evaluationWorkflowId,
+							executionId: row.evaluationExecutionId,
+						},
+					};
+				}
+
+				return undefined;
+			},
+			formatter: (row: TestCaseExecutionRecord) => `${row.id}`,
 			openInNewTab: true,
 		},
 		{
@@ -58,14 +60,14 @@ const columns = computed(
 				{ text: locale.baseText('testDefinition.listRuns.status.success'), value: 'success' },
 				{ text: locale.baseText('testDefinition.listRuns.status.error'), value: 'error' },
 			],
-			filterMethod: (value: string, row: TestCase) => row.status === value,
+			filterMethod: (value: string, row: TestCaseExecutionRecord) => row.status === value,
 		},
 		...Object.keys(run.value?.metrics ?? {}).map((metric) => ({
 			prop: `metrics.${metric}`,
 			label: metric,
 			sortable: true,
 			filter: true,
-			formatter: (row: TestCase) => row.metrics[metric]?.toFixed(2) ?? '-',
+			formatter: (row: TestCaseExecutionRecord) => row.metrics?.[metric]?.toFixed(2) ?? '-',
 		})),
 	],
 );
@@ -73,57 +75,24 @@ const columns = computed(
 const metrics = computed(() => run.value?.metrics ?? {});
 
 // Temporary workaround to fetch test cases by manually getting workflow executions
-// TODO: Replace with dedicated API endpoint once available
 const fetchExecutionTestCases = async () => {
 	if (!runId.value || !testId.value) return;
 
 	isLoading.value = true;
 	try {
+		await testDefinitionStore.fetchTestDefinition(testId.value);
+
 		const testRun = await testDefinitionStore.getTestRun({
 			testDefinitionId: testId.value,
 			runId: runId.value,
 		});
-		const testDefinition = await testDefinitionStore.fetchTestDefinition(testId.value);
 
-		// Fetch workflow executions that match this test run
+		const testCaseEvaluationExecutions = await testDefinitionStore.fetchTestCaseExecutions({
-		const evaluationWorkflowExecutions = await executionsStore.fetchExecutions({
+			testDefinitionId: testId.value,
-			workflowId: testDefinition.evaluationWorkflowId ?? '',
+			runId: testRun.id,
-			metadata: [{ key: 'testRunId', value: testRun.id }],
 		});
 
-		// For each execution, fetch full details and extract metrics
+		testCases.value = testCaseEvaluationExecutions ?? [];
-		const executionsData = await Promise.all(
-			evaluationWorkflowExecutions?.results.map(async (execution) => {
-				const executionData = await executionsStore.fetchExecution(execution.id);
-				const lastExecutedNode = executionData?.data?.resultData?.lastNodeExecuted;
-				if (!lastExecutedNode) {
-					throw new Error('Last executed node is required');
-				}
-				const metricsData = get(
-					executionData,
-					[
-						'data',
-						'resultData',
-						'runData',
-						lastExecutedNode,
-						'0',
-						'data',
-						'main',
-						'0',
-						'0',
-						'json',
-					],
-					{},
-				);
-
-				return {
-					...execution,
-					metrics: metricsData,
-				};
-			}),
-		);
-
-		testCases.value = executionsData ?? [];
 	} catch (error) {
 		toast.showError(error, 'Failed to load run details');
 	} finally {

packages/editor-ui/src/views/TestDefinition/tests/TestDefinitionRunDetailView.test.ts

@@ -7,7 +7,6 @@ import TestDefinitionRunDetailView from '@/views/TestDefinition/TestDefinitionRu
 import { useRoute, useRouter } from 'vue-router';
 import { useToast } from '@/composables/useToast';
 import { useTestDefinitionStore } from '@/stores/testDefinition.store.ee';
-import { useExecutionsStore } from '@/stores/executions.store';
 import { useWorkflowsStore } from '@/stores/workflows.store';
 import { nextTick, ref } from 'vue';
 import { mockedStore, waitAllPromises } from '@/__tests__/utils';
@@ -23,8 +22,7 @@ describe('TestDefinitionRunDetailView', () => {
 
 	let showErrorMock: Mock;
 	let getTestRunMock: Mock;
-	let fetchExecutionsMock: Mock;
+	let fetchTestCaseExecutionsMock: Mock;
-	let fetchExecutionMock: Mock;
 
 	const mockTestRun: TestRunRecord = {
 		id: 'run1',
@@ -52,12 +50,10 @@ describe('TestDefinitionRunDetailView', () => {
 		name: 'Evaluation Workflow',
 	};
 
-	const mockExecutions = {
+	const mockTestCaseExecutions = [
-		results: [
+		{ id: 'exec1', status: 'success' },
-			{ id: 'exec1', status: 'success' },
+		{ id: 'exec2', status: 'error' },
-			{ id: 'exec2', status: 'error' },
+	];
-		],
-	};
 
 	beforeEach(() => {
 		setActivePinia(createPinia());
@@ -78,17 +74,7 @@ describe('TestDefinitionRunDetailView', () => {
 
 		showErrorMock = vi.fn();
 		getTestRunMock = vi.fn().mockResolvedValue(mockTestRun);
-		fetchExecutionsMock = vi.fn().mockResolvedValue(mockExecutions);
+		fetchTestCaseExecutionsMock = vi.fn().mockResolvedValue(mockTestCaseExecutions);
-		fetchExecutionMock = vi.fn().mockResolvedValue({
-			data: {
-				resultData: {
-					lastNodeExecuted: 'Node1',
-					runData: {
-						Node1: [{ data: { main: [[{ json: { accuracy: 0.95 } }]] } }],
-					},
-				},
-			},
-		});
 
 		vi.mocked(useToast).mockReturnValue({
 			showError: showErrorMock,
@@ -108,10 +94,6 @@ describe('TestDefinitionRunDetailView', () => {
 		testDefinitionStore.testDefinitionsById = { test1: mockTestDefinition };
 		testDefinitionStore.getTestRun = getTestRunMock;
 
-		const executionsStore = mockedStore(useExecutionsStore);
-		executionsStore.fetchExecutions = fetchExecutionsMock;
-		executionsStore.fetchExecution = fetchExecutionMock;
-
 		const workflowsStore = mockedStore(useWorkflowsStore);
 		workflowsStore.workflowsById = { workflow1: mockWorkflow as IWorkflowDb };
 
@@ -151,7 +133,7 @@ describe('TestDefinitionRunDetailView', () => {
 		testDefinitionStore.getTestRun = vi.fn().mockRejectedValue(new Error('Failed to load'));
 
 		renderComponent({ pinia });
-		await nextTick();
+		await waitAllPromises();
 
 		expect(showErrorMock).toHaveBeenCalledWith(expect.any(Error), 'Failed to load run details');
 	});
@@ -243,7 +225,6 @@ describe('TestDefinitionRunDetailView', () => {
 		setActivePinia(pinia);
 
 		const testDefinitionStore = mockedStore(useTestDefinitionStore);
-		const executionsStore = mockedStore(useExecutionsStore);
 
 		// Mock all required store methods
 		testDefinitionStore.testRunsById = { run1: mockTestRun };
@@ -251,17 +232,14 @@ describe('TestDefinitionRunDetailView', () => {
 		testDefinitionStore.getTestRun = getTestRunMock;
 		// Add this mock for fetchTestDefinition
 		testDefinitionStore.fetchTestDefinition = vi.fn().mockResolvedValue(mockTestDefinition);
-
+		testDefinitionStore.fetchTestCaseExecutions = fetchTestCaseExecutionsMock;
-		executionsStore.fetchExecutions = fetchExecutionsMock;
-		executionsStore.fetchExecution = fetchExecutionMock;
 
 		const { container } = renderComponent({ pinia });
-		await nextTick();
 
 		// Wait for all promises to resolve
 		await waitAllPromises();
 
 		const tableRows = container.querySelectorAll('.el-table__row');
-		expect(tableRows.length).toBe(mockExecutions.results.length);
+		expect(tableRows.length).toBe(mockTestCaseExecutions.length);
 	});
 });