feat(core): Initial TestRunner service with basic test execution (no-changelog) (#11735)

packages/cli/src/databases/repositories/execution.repository.ts

@@ -163,7 +163,13 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
 			if (!queryParams.relations) {
 				queryParams.relations = [];
 			}
-			(queryParams.relations as string[]).push('executionData', 'metadata');
+
+			if (Array.isArray(queryParams.relations)) {
+				queryParams.relations.push('executionData', 'metadata');
+			} else {
+				queryParams.relations.executionData = true;
+				queryParams.relations.metadata = true;
+			}
 		}
 
 		const executions = await this.find(queryParams);

packages/cli/src/evaluation/test-definitions.controller.ee.ts

@@ -8,6 +8,7 @@ import {
 	testDefinitionCreateRequestBodySchema,
 	testDefinitionPatchRequestBodySchema,
 } from '@/evaluation/test-definition.schema';
+import { TestRunnerService } from '@/evaluation/test-runner/test-runner.service.ee';
 import { listQueryMiddleware } from '@/middlewares';
 import { getSharedWorkflowIds } from '@/public-api/v1/handlers/workflows/workflows.service';
 
@@ -16,7 +17,10 @@ import { TestDefinitionsRequest } from './test-definitions.types.ee';
 
 @RestController('/evaluation/test-definitions')
 export class TestDefinitionsController {
-	constructor(private readonly testDefinitionService: TestDefinitionService) {}
+	constructor(
+		private readonly testDefinitionService: TestDefinitionService,
+		private readonly testRunnerService: TestRunnerService,
+	) {}
 
 	@Get('/', { middlewares: listQueryMiddleware })
 	async getMany(req: TestDefinitionsRequest.GetMany) {
@@ -125,4 +129,20 @@ export class TestDefinitionsController {
 
 		return testDefinition;
 	}
+
+	@Post('/:id/run')
+	async runTest(req: TestDefinitionsRequest.Run, res: express.Response) {
+		const { id: testDefinitionId } = req.params;
+
+		const workflowIds = await getSharedWorkflowIds(req.user, ['workflow:read']);
+
+		// Check test definition exists
+		const testDefinition = await this.testDefinitionService.findOne(testDefinitionId, workflowIds);
+		if (!testDefinition) throw new NotFoundError('Test definition not found');
+
+		// We do not await for the test run to complete
+		void this.testRunnerService.runTest(req.user, testDefinition);
+
+		res.status(202).json({ success: true });
+	}
 }

packages/cli/src/evaluation/test-definitions.types.ee.ts

@@ -30,4 +30,6 @@ export declare namespace TestDefinitionsRequest {
 	>;
 
 	type Delete = AuthenticatedRequest<RouteParams.TestId>;
+
+	type Run = AuthenticatedRequest<RouteParams.TestId>;
 }

packages/cli/src/evaluation/test-runner/__tests__/mock-data/execution-data.json

@@ -0,0 +1,171 @@
+{
+	"startData": {},
+	"resultData": {
+		"runData": {
+			"When clicking ‘Test workflow’": [
+				{
+					"hints": [],
+					"startTime": 1731079118048,
+					"executionTime": 0,
+					"source": [],
+					"executionStatus": "success",
+					"data": {
+						"main": [
+							[
+								{
+									"json": {
+										"query": "First item"
+									},
+									"pairedItem": {
+										"item": 0
+									}
+								},
+								{
+									"json": {
+										"query": "Second item"
+									},
+									"pairedItem": {
+										"item": 0
+									}
+								},
+								{
+									"json": {
+										"query": "Third item"
+									},
+									"pairedItem": {
+										"item": 0
+									}
+								}
+							]
+						]
+					}
+				}
+			],
+			"Edit Fields": [
+				{
+					"hints": [],
+					"startTime": 1731079118049,
+					"executionTime": 0,
+					"source": [
+						{
+							"previousNode": "When clicking ‘Test workflow’"
+						}
+					],
+					"executionStatus": "success",
+					"data": {
+						"main": [
+							[
+								{
+									"json": {
+										"foo": "bar"
+									},
+									"pairedItem": {
+										"item": 0
+									}
+								},
+								{
+									"json": {
+										"foo": "bar"
+									},
+									"pairedItem": {
+										"item": 1
+									}
+								},
+								{
+									"json": {
+										"foo": "bar"
+									},
+									"pairedItem": {
+										"item": 2
+									}
+								}
+							]
+						]
+					}
+				}
+			],
+			"Code": [
+				{
+					"hints": [],
+					"startTime": 1731079118049,
+					"executionTime": 3,
+					"source": [
+						{
+							"previousNode": "Edit Fields"
+						}
+					],
+					"executionStatus": "success",
+					"data": {
+						"main": [
+							[
+								{
+									"json": {
+										"foo": "bar",
+										"random": 0.6315509336851373
+									},
+									"pairedItem": {
+										"item": 0
+									}
+								},
+								{
+									"json": {
+										"foo": "bar",
+										"random": 0.3336315687359024
+									},
+									"pairedItem": {
+										"item": 1
+									}
+								},
+								{
+									"json": {
+										"foo": "bar",
+										"random": 0.4241870158917733
+									},
+									"pairedItem": {
+										"item": 2
+									}
+								}
+							]
+						]
+					}
+				}
+			]
+		},
+		"pinData": {
+			"When clicking ‘Test workflow’": [
+				{
+					"json": {
+						"query": "First item"
+					},
+					"pairedItem": {
+						"item": 0
+					}
+				},
+				{
+					"json": {
+						"query": "Second item"
+					},
+					"pairedItem": {
+						"item": 0
+					}
+				},
+				{
+					"json": {
+						"query": "Third item"
+					},
+					"pairedItem": {
+						"item": 0
+					}
+				}
+			]
+		},
+		"lastNodeExecuted": "Code"
+	},
+	"executionData": {
+		"contextData": {},
+		"nodeExecutionStack": [],
+		"metadata": {},
+		"waitingExecution": {},
+		"waitingExecutionSource": {}
+	}
+}

packages/cli/src/evaluation/test-runner/__tests__/mock-data/workflow.evaluation.json

@@ -0,0 +1,124 @@
+{
+	"name": "Evaluation Workflow",
+	"nodes": [
+		{
+			"parameters": {},
+			"id": "285ac92b-256f-4bb2-a450-6486b01593cb",
+			"name": "Execute Workflow Trigger",
+			"type": "n8n-nodes-base.executeWorkflowTrigger",
+			"typeVersion": 1,
+			"position": [520, 340]
+		},
+		{
+			"parameters": {
+				"conditions": {
+					"options": {
+						"caseSensitive": true,
+						"leftValue": "",
+						"typeValidation": "strict",
+						"version": 2
+					},
+					"conditions": [
+						{
+							"id": "9d3abc8d-3270-4bec-9a59-82622d5dbb5a",
+							"leftValue": "={{ $json.actual.Code[0].data.main[0].length }}",
+							"rightValue": 3,
+							"operator": {
+								"type": "number",
+								"operation": "gte"
+							}
+						},
+						{
+							"id": "894ce84b-13a4-4415-99c0-0c25182903bb",
+							"leftValue": "={{ $json.actual.Code[0].data.main[0][0].json.random }}",
+							"rightValue": 0.7,
+							"operator": {
+								"type": "number",
+								"operation": "lt"
+							}
+						}
+					],
+					"combinator": "and"
+				},
+				"options": {}
+			},
+			"id": "320b0355-3886-41df-b039-4666bf28e47b",
+			"name": "If",
+			"type": "n8n-nodes-base.if",
+			"typeVersion": 2.2,
+			"position": [740, 340]
+		},
+		{
+			"parameters": {
+				"assignments": {
+					"assignments": [
+						{
+							"id": "3b65d55a-158f-40c6-9853-a1c44b7ba1e5",
+							"name": "success",
+							"value": true,
+							"type": "boolean"
+						}
+					]
+				},
+				"options": {}
+			},
+			"id": "0c7a1ee8-0cf0-4d7f-99a3-186bbcd8815a",
+			"name": "Success",
+			"type": "n8n-nodes-base.set",
+			"typeVersion": 3.4,
+			"position": [980, 220]
+		},
+		{
+			"parameters": {
+				"assignments": {
+					"assignments": [
+						{
+							"id": "6cc8b402-4a30-4873-b825-963a1f1b8b82",
+							"name": "success",
+							"value": false,
+							"type": "boolean"
+						}
+					]
+				},
+				"options": {}
+			},
+			"id": "50d3f84a-d99f-4e04-bdbd-3e8c2668e708",
+			"name": "Fail",
+			"type": "n8n-nodes-base.set",
+			"typeVersion": 3.4,
+			"position": [980, 420]
+		}
+	],
+	"connections": {
+		"Execute Workflow Trigger": {
+			"main": [
+				[
+					{
+						"node": "If",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		},
+		"If": {
+			"main": [
+				[
+					{
+						"node": "Success",
+						"type": "main",
+						"index": 0
+					}
+				],
+				[
+					{
+						"node": "Fail",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		}
+	},
+	"pinData": {}
+}

packages/cli/src/evaluation/test-runner/__tests__/mock-data/workflow.under-test.json

@@ -0,0 +1,78 @@
+{
+	"name": "Workflow Under Test",
+	"nodes": [
+		{
+			"parameters": {},
+			"type": "n8n-nodes-base.manualTrigger",
+			"typeVersion": 1,
+			"position": [-80, 0],
+			"id": "72256d90-3a67-4e29-b032-47df4e5768af",
+			"name": "When clicking ‘Test workflow’"
+		},
+		{
+			"parameters": {
+				"assignments": {
+					"assignments": [
+						{
+							"id": "acfeecbe-443c-4220-b63b-d44d69216902",
+							"name": "foo",
+							"value": "bar",
+							"type": "string"
+						}
+					]
+				},
+				"options": {}
+			},
+			"type": "n8n-nodes-base.set",
+			"typeVersion": 3.4,
+			"position": [140, 0],
+			"id": "319f29bc-1dd4-4122-b223-c584752151a4",
+			"name": "Edit Fields"
+		},
+		{
+			"parameters": {
+				"jsCode": "for (const item of $input.all()) {\n  item.json.random = Math.random();\n}\n\nreturn $input.all();"
+			},
+			"type": "n8n-nodes-base.code",
+			"typeVersion": 2,
+			"position": [380, 0],
+			"id": "d2474215-63af-40a4-a51e-0ea30d762621",
+			"name": "Code"
+		}
+	],
+	"connections": {
+		"When clicking ‘Test workflow’": {
+			"main": [
+				[
+					{
+						"node": "Edit Fields",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		},
+		"Edit Fields": {
+			"main": [
+				[
+					{
+						"node": "Wait",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		},
+		"Wait": {
+			"main": [
+				[
+					{
+						"node": "Code",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		}
+	}
+}

packages/cli/src/evaluation/test-runner/__tests__/test-runner.service.ee.test.ts

@@ -0,0 +1,102 @@
+import type { SelectQueryBuilder } from '@n8n/typeorm';
+import { stringify } from 'flatted';
+import { readFileSync } from 'fs';
+import { mock, mockDeep } from 'jest-mock-extended';
+import path from 'path';
+
+import type { ActiveExecutions } from '@/active-executions';
+import type { ExecutionEntity } from '@/databases/entities/execution-entity';
+import type { TestDefinition } from '@/databases/entities/test-definition.ee';
+import type { User } from '@/databases/entities/user';
+import type { ExecutionRepository } from '@/databases/repositories/execution.repository';
+import type { WorkflowRepository } from '@/databases/repositories/workflow.repository';
+import type { WorkflowRunner } from '@/workflow-runner';
+
+import { TestRunnerService } from '../test-runner.service.ee';
+
+const wfUnderTestJson = JSON.parse(
+	readFileSync(path.join(__dirname, './mock-data/workflow.under-test.json'), { encoding: 'utf-8' }),
+);
+
+const executionDataJson = JSON.parse(
+	readFileSync(path.join(__dirname, './mock-data/execution-data.json'), { encoding: 'utf-8' }),
+);
+
+const executionMocks = [
+	mock<ExecutionEntity>({
+		id: 'some-execution-id',
+		workflowId: 'workflow-under-test-id',
+		status: 'success',
+		executionData: {
+			data: stringify(executionDataJson),
+		},
+	}),
+	mock<ExecutionEntity>({
+		id: 'some-execution-id-2',
+		workflowId: 'workflow-under-test-id',
+		status: 'success',
+		executionData: {
+			data: stringify(executionDataJson),
+		},
+	}),
+];
+
+describe('TestRunnerService', () => {
+	const executionRepository = mock<ExecutionRepository>();
+	const workflowRepository = mock<WorkflowRepository>();
+	const workflowRunner = mock<WorkflowRunner>();
+	const activeExecutions = mock<ActiveExecutions>();
+
+	beforeEach(() => {
+		const executionsQbMock = mockDeep<SelectQueryBuilder<ExecutionEntity>>({
+			fallbackMockImplementation: jest.fn().mockReturnThis(),
+		});
+
+		executionsQbMock.getMany.mockResolvedValueOnce(executionMocks);
+		executionRepository.createQueryBuilder.mockReturnValueOnce(executionsQbMock);
+		executionRepository.findOne
+			.calledWith(expect.objectContaining({ where: { id: 'some-execution-id' } }))
+			.mockResolvedValueOnce(executionMocks[0]);
+		executionRepository.findOne
+			.calledWith(expect.objectContaining({ where: { id: 'some-execution-id-2' } }))
+			.mockResolvedValueOnce(executionMocks[1]);
+	});
+
+	test('should create an instance of TestRunnerService', async () => {
+		const testRunnerService = new TestRunnerService(
+			workflowRepository,
+			workflowRunner,
+			executionRepository,
+			activeExecutions,
+		);
+
+		expect(testRunnerService).toBeInstanceOf(TestRunnerService);
+	});
+
+	test('should create and run test cases from past executions', async () => {
+		const testRunnerService = new TestRunnerService(
+			workflowRepository,
+			workflowRunner,
+			executionRepository,
+			activeExecutions,
+		);
+
+		workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({
+			id: 'workflow-under-test-id',
+			...wfUnderTestJson,
+		});
+
+		workflowRunner.run.mockResolvedValue('test-execution-id');
+
+		await testRunnerService.runTest(
+			mock<User>(),
+			mock<TestDefinition>({
+				workflowId: 'workflow-under-test-id',
+			}),
+		);
+
+		expect(executionRepository.createQueryBuilder).toHaveBeenCalledTimes(1);
+		expect(executionRepository.findOne).toHaveBeenCalledTimes(2);
+		expect(workflowRunner.run).toHaveBeenCalledTimes(2);
+	});
+});

packages/cli/src/evaluation/test-runner/test-runner.service.ee.ts

@@ -0,0 +1,129 @@
+import { parse } from 'flatted';
+import type { IPinData, IRun, IWorkflowExecutionDataProcess } from 'n8n-workflow';
+import assert from 'node:assert';
+import { Service } from 'typedi';
+
+import { ActiveExecutions } from '@/active-executions';
+import type { ExecutionEntity } from '@/databases/entities/execution-entity';
+import type { TestDefinition } from '@/databases/entities/test-definition.ee';
+import type { User } from '@/databases/entities/user';
+import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
+import { ExecutionRepository } from '@/databases/repositories/execution.repository';
+import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
+import type { IExecutionResponse } from '@/interfaces';
+import { WorkflowRunner } from '@/workflow-runner';
+
+/**
+ * This service orchestrates the running of test cases.
+ * It uses the test definitions to find
+ * past executions, creates pin data from them,
+ * and runs the workflow-under-test with the pin data.
+ * TODO: Evaluation workflows
+ * TODO: Node pinning
+ * TODO: Collect metrics
+ */
+@Service()
+export class TestRunnerService {
+	constructor(
+		private readonly workflowRepository: WorkflowRepository,
+		private readonly workflowRunner: WorkflowRunner,
+		private readonly executionRepository: ExecutionRepository,
+		private readonly activeExecutions: ActiveExecutions,
+	) {}
+
+	/**
+	 * Creates a pin data object from the past execution data
+	 * for the given workflow.
+	 * For now, it only pins trigger nodes.
+	 */
+	private createPinDataFromExecution(
+		workflow: WorkflowEntity,
+		execution: ExecutionEntity,
+	): IPinData {
+		const executionData = parse(execution.executionData.data) as IExecutionResponse['data'];
+
+		const triggerNodes = workflow.nodes.filter((node) => /trigger$/i.test(node.type));
+
+		const pinData = {} as IPinData;
+
+		for (const triggerNode of triggerNodes) {
+			const triggerData = executionData.resultData.runData[triggerNode.name];
+			if (triggerData?.[0]?.data?.main?.[0]) {
+				pinData[triggerNode.name] = triggerData[0]?.data?.main?.[0];
+			}
+		}
+
+		return pinData;
+	}
+
+	/**
+	 * Runs a test case with the given pin data.
+	 * Waits for the workflow under test to finish execution.
+	 */
+	private async runTestCase(
+		workflow: WorkflowEntity,
+		testCasePinData: IPinData,
+		userId: string,
+	): Promise<IRun | undefined> {
+		const data: IWorkflowExecutionDataProcess = {
+			executionMode: 'evaluation',
+			runData: {},
+			pinData: testCasePinData,
+			workflowData: workflow,
+			partialExecutionVersion: '-1',
+			userId,
+		};
+
+		// Trigger the workflow under test with mocked data
+		const executionId = await this.workflowRunner.run(data);
+		assert(executionId);
+
+		// Wait for the workflow to finish execution
+		const executePromise = this.activeExecutions.getPostExecutePromise(executionId);
+
+		return await executePromise;
+	}
+
+	/**
+	 * Creates a new test run for the given test definition.
+	 */
+	public async runTest(user: User, test: TestDefinition): Promise<void> {
+		const workflow = await this.workflowRepository.findById(test.workflowId);
+		assert(workflow, 'Workflow not found');
+
+		// 1. Make test cases from previous executions
+
+		// Select executions with the annotation tag and workflow ID of the test.
+		// Fetch only ids to reduce the data transfer.
+		const pastExecutions: ReadonlyArray<Pick<ExecutionEntity, 'id'>> =
+			await this.executionRepository
+				.createQueryBuilder('execution')
+				.select('execution.id')
+				.leftJoin('execution.annotation', 'annotation')
+				.leftJoin('annotation.tags', 'annotationTag')
+				.where('annotationTag.id = :tagId', { tagId: test.annotationTagId })
+				.andWhere('execution.workflowId = :workflowId', { workflowId: test.workflowId })
+				.getMany();
+
+		// 2. Run the test cases
+
+		for (const { id: pastExecutionId } of pastExecutions) {
+			const pastExecution = await this.executionRepository.findOne({
+				where: { id: pastExecutionId },
+				relations: ['executionData', 'metadata'],
+			});
+			assert(pastExecution, 'Execution not found');
+
+			const pinData = this.createPinDataFromExecution(workflow, pastExecution);
+
+			// Run the test case and wait for it to finish
+			const execution = await this.runTestCase(workflow, pinData, user.id);
+
+			if (!execution) {
+				continue;
+			}
+
+			// TODO: 2.3 Collect the run data
+		}
+	}
+}

packages/cli/test/integration/evaluation/test-definitions.api.test.ts

@@ -1,9 +1,11 @@
+import { mockInstance } from 'n8n-core/test/utils';
 import { Container } from 'typedi';
 
 import type { AnnotationTagEntity } from '@/databases/entities/annotation-tag-entity.ee';
 import type { User } from '@/databases/entities/user';
 import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
 import { TestDefinitionRepository } from '@/databases/repositories/test-definition.repository.ee';
+import { TestRunnerService } from '@/evaluation/test-runner/test-runner.service.ee';
 import { createAnnotationTags } from '@test-integration/db/executions';
 
 import { createUserShell } from './../shared/db/users';
@@ -12,6 +14,8 @@ import * as testDb from './../shared/test-db';
 import type { SuperAgentTest } from './../shared/types';
 import * as utils from './../shared/utils/';
 
+const testRunner = mockInstance(TestRunnerService);
+
 let authOwnerAgent: SuperAgentTest;
 let workflowUnderTest: WorkflowEntity;
 let workflowUnderTest2: WorkflowEntity;
@@ -426,3 +430,24 @@ describe('DELETE /evaluation/test-definitions/:id', () => {
 		expect(resp.body.message).toBe('Test definition not found');
 	});
 });
+
+describe('POST /evaluation/test-definitions/:id/run', () => {
+	test('should trigger the test run', async () => {
+		const newTest = Container.get(TestDefinitionRepository).create({
+			name: 'test',
+			workflow: { id: workflowUnderTest.id },
+		});
+		await Container.get(TestDefinitionRepository).save(newTest);
+
+		const resp = await authOwnerAgent.post(`/evaluation/test-definitions/${newTest.id}/run`);
+
+		expect(resp.statusCode).toBe(202);
+		expect(resp.body).toEqual(
+			expect.objectContaining({
+				success: true,
+			}),
+		);
+
+		expect(testRunner.runTest).toHaveBeenCalledTimes(1);
+	});
+});

packages/workflow/src/Interfaces.ts

@@ -2371,7 +2371,8 @@ export type WorkflowExecuteMode =
 	| 'manual'
 	| 'retry'
 	| 'trigger'
-	| 'webhook';
+	| 'webhook'
+	| 'evaluation';
 
 export type WorkflowActivateMode =
 	| 'init'