wip: test runner draft

packages/cli/src/databases/repositories/execution.repository.ts

@@ -163,7 +163,13 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
 			if (!queryParams.relations) {
 				queryParams.relations = [];
 			}
-			(queryParams.relations as string[]).push('executionData', 'metadata');
+
+			if (Array.isArray(queryParams.relations)) {
+				queryParams.relations.push('executionData', 'metadata');
+			} else {
+				queryParams.relations.executionData = true;
+				queryParams.relations.metadata = true;
+			}
 		}
 
 		const executions = await this.find(queryParams);

packages/cli/src/evaluation/test-definitions.controller.ee.ts

@@ -9,6 +9,7 @@ import {
 	testDefinitionCreateRequestBodySchema,
 	testDefinitionPatchRequestBodySchema,
 } from '@/evaluation/test-definition.schema';
+import { TestRunnerService } from '@/evaluation/test-runner/test-runner.service.ee';
 import { listQueryMiddleware } from '@/middlewares';
 import { getSharedWorkflowIds } from '@/public-api/v1/handlers/workflows/workflows.service';
 import { isPositiveInteger } from '@/utils';
@@ -26,7 +27,10 @@ export class TestDefinitionsController {
 		return Number(id);
 	}
 
-	constructor(private readonly testDefinitionService: TestDefinitionService) {}
+	constructor(
+		private readonly testDefinitionService: TestDefinitionService,
+		private readonly testRunnerService: TestRunnerService,
+	) {}
 
 	@Get('/', { middlewares: listQueryMiddleware })
 	async getMany(req: TestDefinitionsRequest.GetMany) {
@@ -135,4 +139,16 @@ export class TestDefinitionsController {
 
 		return testDefinition;
 	}
+
+	@Post('/:id/run')
+	async runTest(req: TestDefinitionsRequest.Run) {
+		if (!isPositiveInteger(req.params.id)) {
+			throw new BadRequestError('Test ID is not a number');
+		}
+
+		const workflowIds = await getSharedWorkflowIds(req.user, ['workflow:read']);
+
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-return
+		return await this.testRunnerService.runTest(req.user, Number(req.params.id), workflowIds);
+	}
 }

packages/cli/src/evaluation/test-definitions.types.ee.ts

@@ -30,4 +30,6 @@ export declare namespace TestDefinitionsRequest {
 	>;
 
 	type Delete = AuthenticatedRequest<RouteParams.TestId>;
+
+	type Run = AuthenticatedRequest<RouteParams.TestId>;
 }

packages/cli/src/evaluation/test-runner/test-runner.service.ee.ts

@@ -0,0 +1,100 @@
+import { parse } from 'flatted';
+import type { IPinData, IWorkflowExecutionDataProcess } from 'n8n-workflow';
+import assert from 'node:assert';
+import { Container, Service } from 'typedi';
+
+import { ActiveExecutions } from '@/active-executions';
+import type { User } from '@/databases/entities/user';
+import { ExecutionRepository } from '@/databases/repositories/execution.repository';
+import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
+import { NotFoundError } from '@/errors/response-errors/not-found.error';
+import { TestDefinitionService } from '@/evaluation/test-definition.service.ee';
+import type { IExecutionDb, IExecutionResponse } from '@/interfaces';
+import { WorkflowRunner } from '@/workflow-runner';
+// import { WorkflowExecutionService } from '@/workflows/workflow-execution.service';
+
+@Service()
+export class TestRunnerService {
+	constructor(
+		private readonly testDefinitionsService: TestDefinitionService,
+		private readonly workflowRepository: WorkflowRepository,
+		// private readonly workflowExecutionService: WorkflowExecutionService,
+		private readonly workflowRunner: WorkflowRunner,
+		private readonly executionRepository: ExecutionRepository,
+	) {}
+
+	public async runTest(user: User, testId: number, accessibleWorkflowIds: string[]): Promise<any> {
+		const test = await this.testDefinitionsService.findOne(testId, accessibleWorkflowIds);
+
+		console.log({ test });
+
+		if (!test) {
+			throw new NotFoundError('Test definition not found');
+		}
+
+		const workflow = await this.workflowRepository.findById(test.workflowId);
+		assert(workflow, 'Workflow not found');
+
+		// Make a list of test cases
+		// const executions = await this.executionRepository.findManyByRangeQuery({
+		// 	kind: 'range',
+		// 	range: {
+		// 		limit: 99,
+		// 	},
+		// 	annotationTags: [test.annotationTagId],
+		// 	accessibleWorkflowIds,
+		// });
+		const executions = await this.executionRepository
+			.createQueryBuilder('execution')
+			.leftJoin('execution.annotation', 'annotation')
+			.leftJoin('annotation.tags', 'annotationTag')
+			.leftJoinAndSelect('execution.executionData', 'executionData')
+			.leftJoinAndSelect('execution.metadata', 'metadata')
+			.where('annotationTag.id = :tagId', { tagId: test.annotationTagId })
+			.andWhere('execution.workflowId = :workflowId', { workflowId: test.workflowId })
+			.getMany();
+
+		const testCases = executions.map((execution) => {
+			const executionData = parse(execution.executionData.data) as IExecutionResponse['data'];
+
+			return {
+				pinData: {
+					'When clicking ‘Test workflow’':
+						executionData.resultData.runData['When clicking ‘Test workflow’'][0]?.data?.main?.[0],
+				} as IPinData,
+			};
+		});
+
+		console.log({ testCases });
+
+		for (const testCase of testCases) {
+			// Start the workflow
+			const data: IWorkflowExecutionDataProcess = {
+				executionMode: 'evaluation',
+				runData: {},
+				pinData: testCase.pinData,
+				workflowData: workflow,
+				userId: user.id,
+				partialExecutionVersion: '-1',
+			};
+
+			// if (pinnedTrigger && !hasRunData(pinnedTrigger)) {
+			// 	data.startNodes = [{ name: pinnedTrigger.name, sourceData: null }];
+			// }
+
+			const executionId = await this.workflowRunner.run(data);
+
+			assert(executionId);
+
+			const executePromise = Container.get(ActiveExecutions).getPostExecutePromise(
+				executionId,
+			) as Promise<IExecutionDb | undefined>;
+
+			const execution = await executePromise;
+			console.log({ execution });
+			console.log(execution?.data.resultData.runData.Code?.[0].data?.main[0]);
+		}
+
+		return { success: true };
+	}
+}

packages/workflow/src/Interfaces.ts

@@ -2346,7 +2346,8 @@ export type WorkflowExecuteMode =
 	| 'manual'
 	| 'retry'
 	| 'trigger'
-	| 'webhook';
+	| 'webhook'
+	| 'evaluation';
 
 export type WorkflowActivateMode =
 	| 'init'