diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 280e6a5bca..5bef675a79 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -163,7 +163,13 @@ export class ExecutionRepository extends Repository { if (!queryParams.relations) { queryParams.relations = []; } - (queryParams.relations as string[]).push('executionData', 'metadata'); + + if (Array.isArray(queryParams.relations)) { + queryParams.relations.push('executionData', 'metadata'); + } else { + queryParams.relations.executionData = true; + queryParams.relations.metadata = true; + } } const executions = await this.find(queryParams); diff --git a/packages/cli/src/evaluation/test-definitions.controller.ee.ts b/packages/cli/src/evaluation/test-definitions.controller.ee.ts index eaa7745c51..ef4a3ed461 100644 --- a/packages/cli/src/evaluation/test-definitions.controller.ee.ts +++ b/packages/cli/src/evaluation/test-definitions.controller.ee.ts @@ -8,6 +8,7 @@ import { testDefinitionCreateRequestBodySchema, testDefinitionPatchRequestBodySchema, } from '@/evaluation/test-definition.schema'; +import { TestRunnerService } from '@/evaluation/test-runner/test-runner.service.ee'; import { listQueryMiddleware } from '@/middlewares'; import { getSharedWorkflowIds } from '@/public-api/v1/handlers/workflows/workflows.service'; @@ -16,7 +17,10 @@ import { TestDefinitionsRequest } from './test-definitions.types.ee'; @RestController('/evaluation/test-definitions') export class TestDefinitionsController { - constructor(private readonly testDefinitionService: TestDefinitionService) {} + constructor( + private readonly testDefinitionService: TestDefinitionService, + private readonly testRunnerService: TestRunnerService, + ) {} @Get('/', { middlewares: listQueryMiddleware }) async getMany(req: TestDefinitionsRequest.GetMany) { @@ -125,4 +129,20 @@ export class TestDefinitionsController { return testDefinition; } + + @Post('/:id/run') + async runTest(req: TestDefinitionsRequest.Run, res: express.Response) { + const { id: testDefinitionId } = req.params; + + const workflowIds = await getSharedWorkflowIds(req.user, ['workflow:read']); + + // Check test definition exists + const testDefinition = await this.testDefinitionService.findOne(testDefinitionId, workflowIds); + if (!testDefinition) throw new NotFoundError('Test definition not found'); + + // We do not await for the test run to complete + void this.testRunnerService.runTest(req.user, testDefinition); + + res.status(202).json({ success: true }); + } } diff --git a/packages/cli/src/evaluation/test-definitions.types.ee.ts b/packages/cli/src/evaluation/test-definitions.types.ee.ts index 2814e6bb7f..2b784b10ef 100644 --- a/packages/cli/src/evaluation/test-definitions.types.ee.ts +++ b/packages/cli/src/evaluation/test-definitions.types.ee.ts @@ -30,4 +30,6 @@ export declare namespace TestDefinitionsRequest { >; type Delete = AuthenticatedRequest; + + type Run = AuthenticatedRequest; } diff --git a/packages/cli/src/evaluation/test-runner/__tests__/mock-data/execution-data.json b/packages/cli/src/evaluation/test-runner/__tests__/mock-data/execution-data.json new file mode 100644 index 0000000000..1c974205d0 --- /dev/null +++ b/packages/cli/src/evaluation/test-runner/__tests__/mock-data/execution-data.json @@ -0,0 +1,171 @@ +{ + "startData": {}, + "resultData": { + "runData": { + "When clicking ‘Test workflow’": [ + { + "hints": [], + "startTime": 1731079118048, + "executionTime": 0, + "source": [], + "executionStatus": "success", + "data": { + "main": [ + [ + { + "json": { + "query": "First item" + }, + "pairedItem": { + "item": 0 + } + }, + { + "json": { + "query": "Second item" + }, + "pairedItem": { + "item": 0 + } + }, + { + "json": { + "query": "Third item" + }, + "pairedItem": { + "item": 0 + } + } + ] + ] + } + } + ], + "Edit Fields": [ + { + "hints": [], + "startTime": 1731079118049, + "executionTime": 0, + "source": [ + { + "previousNode": "When clicking ‘Test workflow’" + } + ], + "executionStatus": "success", + "data": { + "main": [ + [ + { + "json": { + "foo": "bar" + }, + "pairedItem": { + "item": 0 + } + }, + { + "json": { + "foo": "bar" + }, + "pairedItem": { + "item": 1 + } + }, + { + "json": { + "foo": "bar" + }, + "pairedItem": { + "item": 2 + } + } + ] + ] + } + } + ], + "Code": [ + { + "hints": [], + "startTime": 1731079118049, + "executionTime": 3, + "source": [ + { + "previousNode": "Edit Fields" + } + ], + "executionStatus": "success", + "data": { + "main": [ + [ + { + "json": { + "foo": "bar", + "random": 0.6315509336851373 + }, + "pairedItem": { + "item": 0 + } + }, + { + "json": { + "foo": "bar", + "random": 0.3336315687359024 + }, + "pairedItem": { + "item": 1 + } + }, + { + "json": { + "foo": "bar", + "random": 0.4241870158917733 + }, + "pairedItem": { + "item": 2 + } + } + ] + ] + } + } + ] + }, + "pinData": { + "When clicking ‘Test workflow’": [ + { + "json": { + "query": "First item" + }, + "pairedItem": { + "item": 0 + } + }, + { + "json": { + "query": "Second item" + }, + "pairedItem": { + "item": 0 + } + }, + { + "json": { + "query": "Third item" + }, + "pairedItem": { + "item": 0 + } + } + ] + }, + "lastNodeExecuted": "Code" + }, + "executionData": { + "contextData": {}, + "nodeExecutionStack": [], + "metadata": {}, + "waitingExecution": {}, + "waitingExecutionSource": {} + } +} diff --git a/packages/cli/src/evaluation/test-runner/__tests__/mock-data/workflow.evaluation.json b/packages/cli/src/evaluation/test-runner/__tests__/mock-data/workflow.evaluation.json new file mode 100644 index 0000000000..ceec74d809 --- /dev/null +++ b/packages/cli/src/evaluation/test-runner/__tests__/mock-data/workflow.evaluation.json @@ -0,0 +1,124 @@ +{ + "name": "Evaluation Workflow", + "nodes": [ + { + "parameters": {}, + "id": "285ac92b-256f-4bb2-a450-6486b01593cb", + "name": "Execute Workflow Trigger", + "type": "n8n-nodes-base.executeWorkflowTrigger", + "typeVersion": 1, + "position": [520, 340] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict", + "version": 2 + }, + "conditions": [ + { + "id": "9d3abc8d-3270-4bec-9a59-82622d5dbb5a", + "leftValue": "={{ $json.actual.Code[0].data.main[0].length }}", + "rightValue": 3, + "operator": { + "type": "number", + "operation": "gte" + } + }, + { + "id": "894ce84b-13a4-4415-99c0-0c25182903bb", + "leftValue": "={{ $json.actual.Code[0].data.main[0][0].json.random }}", + "rightValue": 0.7, + "operator": { + "type": "number", + "operation": "lt" + } + } + ], + "combinator": "and" + }, + "options": {} + }, + "id": "320b0355-3886-41df-b039-4666bf28e47b", + "name": "If", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [740, 340] + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "3b65d55a-158f-40c6-9853-a1c44b7ba1e5", + "name": "success", + "value": true, + "type": "boolean" + } + ] + }, + "options": {} + }, + "id": "0c7a1ee8-0cf0-4d7f-99a3-186bbcd8815a", + "name": "Success", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [980, 220] + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "6cc8b402-4a30-4873-b825-963a1f1b8b82", + "name": "success", + "value": false, + "type": "boolean" + } + ] + }, + "options": {} + }, + "id": "50d3f84a-d99f-4e04-bdbd-3e8c2668e708", + "name": "Fail", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [980, 420] + } + ], + "connections": { + "Execute Workflow Trigger": { + "main": [ + [ + { + "node": "If", + "type": "main", + "index": 0 + } + ] + ] + }, + "If": { + "main": [ + [ + { + "node": "Success", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "Fail", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "pinData": {} +} diff --git a/packages/cli/src/evaluation/test-runner/__tests__/mock-data/workflow.under-test.json b/packages/cli/src/evaluation/test-runner/__tests__/mock-data/workflow.under-test.json new file mode 100644 index 0000000000..0204283a1b --- /dev/null +++ b/packages/cli/src/evaluation/test-runner/__tests__/mock-data/workflow.under-test.json @@ -0,0 +1,78 @@ +{ + "name": "Workflow Under Test", + "nodes": [ + { + "parameters": {}, + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [-80, 0], + "id": "72256d90-3a67-4e29-b032-47df4e5768af", + "name": "When clicking ‘Test workflow’" + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "acfeecbe-443c-4220-b63b-d44d69216902", + "name": "foo", + "value": "bar", + "type": "string" + } + ] + }, + "options": {} + }, + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [140, 0], + "id": "319f29bc-1dd4-4122-b223-c584752151a4", + "name": "Edit Fields" + }, + { + "parameters": { + "jsCode": "for (const item of $input.all()) {\n item.json.random = Math.random();\n}\n\nreturn $input.all();" + }, + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [380, 0], + "id": "d2474215-63af-40a4-a51e-0ea30d762621", + "name": "Code" + } + ], + "connections": { + "When clicking ‘Test workflow’": { + "main": [ + [ + { + "node": "Edit Fields", + "type": "main", + "index": 0 + } + ] + ] + }, + "Edit Fields": { + "main": [ + [ + { + "node": "Wait", + "type": "main", + "index": 0 + } + ] + ] + }, + "Wait": { + "main": [ + [ + { + "node": "Code", + "type": "main", + "index": 0 + } + ] + ] + } + } +} diff --git a/packages/cli/src/evaluation/test-runner/__tests__/test-runner.service.ee.test.ts b/packages/cli/src/evaluation/test-runner/__tests__/test-runner.service.ee.test.ts new file mode 100644 index 0000000000..1cd413370e --- /dev/null +++ b/packages/cli/src/evaluation/test-runner/__tests__/test-runner.service.ee.test.ts @@ -0,0 +1,102 @@ +import type { SelectQueryBuilder } from '@n8n/typeorm'; +import { stringify } from 'flatted'; +import { readFileSync } from 'fs'; +import { mock, mockDeep } from 'jest-mock-extended'; +import path from 'path'; + +import type { ActiveExecutions } from '@/active-executions'; +import type { ExecutionEntity } from '@/databases/entities/execution-entity'; +import type { TestDefinition } from '@/databases/entities/test-definition.ee'; +import type { User } from '@/databases/entities/user'; +import type { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import type { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import type { WorkflowRunner } from '@/workflow-runner'; + +import { TestRunnerService } from '../test-runner.service.ee'; + +const wfUnderTestJson = JSON.parse( + readFileSync(path.join(__dirname, './mock-data/workflow.under-test.json'), { encoding: 'utf-8' }), +); + +const executionDataJson = JSON.parse( + readFileSync(path.join(__dirname, './mock-data/execution-data.json'), { encoding: 'utf-8' }), +); + +const executionMocks = [ + mock({ + id: 'some-execution-id', + workflowId: 'workflow-under-test-id', + status: 'success', + executionData: { + data: stringify(executionDataJson), + }, + }), + mock({ + id: 'some-execution-id-2', + workflowId: 'workflow-under-test-id', + status: 'success', + executionData: { + data: stringify(executionDataJson), + }, + }), +]; + +describe('TestRunnerService', () => { + const executionRepository = mock(); + const workflowRepository = mock(); + const workflowRunner = mock(); + const activeExecutions = mock(); + + beforeEach(() => { + const executionsQbMock = mockDeep>({ + fallbackMockImplementation: jest.fn().mockReturnThis(), + }); + + executionsQbMock.getMany.mockResolvedValueOnce(executionMocks); + executionRepository.createQueryBuilder.mockReturnValueOnce(executionsQbMock); + executionRepository.findOne + .calledWith(expect.objectContaining({ where: { id: 'some-execution-id' } })) + .mockResolvedValueOnce(executionMocks[0]); + executionRepository.findOne + .calledWith(expect.objectContaining({ where: { id: 'some-execution-id-2' } })) + .mockResolvedValueOnce(executionMocks[1]); + }); + + test('should create an instance of TestRunnerService', async () => { + const testRunnerService = new TestRunnerService( + workflowRepository, + workflowRunner, + executionRepository, + activeExecutions, + ); + + expect(testRunnerService).toBeInstanceOf(TestRunnerService); + }); + + test('should create and run test cases from past executions', async () => { + const testRunnerService = new TestRunnerService( + workflowRepository, + workflowRunner, + executionRepository, + activeExecutions, + ); + + workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ + id: 'workflow-under-test-id', + ...wfUnderTestJson, + }); + + workflowRunner.run.mockResolvedValue('test-execution-id'); + + await testRunnerService.runTest( + mock(), + mock({ + workflowId: 'workflow-under-test-id', + }), + ); + + expect(executionRepository.createQueryBuilder).toHaveBeenCalledTimes(1); + expect(executionRepository.findOne).toHaveBeenCalledTimes(2); + expect(workflowRunner.run).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/cli/src/evaluation/test-runner/test-runner.service.ee.ts b/packages/cli/src/evaluation/test-runner/test-runner.service.ee.ts new file mode 100644 index 0000000000..b57f89bea3 --- /dev/null +++ b/packages/cli/src/evaluation/test-runner/test-runner.service.ee.ts @@ -0,0 +1,129 @@ +import { parse } from 'flatted'; +import type { IPinData, IRun, IWorkflowExecutionDataProcess } from 'n8n-workflow'; +import assert from 'node:assert'; +import { Service } from 'typedi'; + +import { ActiveExecutions } from '@/active-executions'; +import type { ExecutionEntity } from '@/databases/entities/execution-entity'; +import type { TestDefinition } from '@/databases/entities/test-definition.ee'; +import type { User } from '@/databases/entities/user'; +import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; +import { ExecutionRepository } from '@/databases/repositories/execution.repository'; +import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; +import type { IExecutionResponse } from '@/interfaces'; +import { WorkflowRunner } from '@/workflow-runner'; + +/** + * This service orchestrates the running of test cases. + * It uses the test definitions to find + * past executions, creates pin data from them, + * and runs the workflow-under-test with the pin data. + * TODO: Evaluation workflows + * TODO: Node pinning + * TODO: Collect metrics + */ +@Service() +export class TestRunnerService { + constructor( + private readonly workflowRepository: WorkflowRepository, + private readonly workflowRunner: WorkflowRunner, + private readonly executionRepository: ExecutionRepository, + private readonly activeExecutions: ActiveExecutions, + ) {} + + /** + * Creates a pin data object from the past execution data + * for the given workflow. + * For now, it only pins trigger nodes. + */ + private createPinDataFromExecution( + workflow: WorkflowEntity, + execution: ExecutionEntity, + ): IPinData { + const executionData = parse(execution.executionData.data) as IExecutionResponse['data']; + + const triggerNodes = workflow.nodes.filter((node) => /trigger$/i.test(node.type)); + + const pinData = {} as IPinData; + + for (const triggerNode of triggerNodes) { + const triggerData = executionData.resultData.runData[triggerNode.name]; + if (triggerData?.[0]?.data?.main?.[0]) { + pinData[triggerNode.name] = triggerData[0]?.data?.main?.[0]; + } + } + + return pinData; + } + + /** + * Runs a test case with the given pin data. + * Waits for the workflow under test to finish execution. + */ + private async runTestCase( + workflow: WorkflowEntity, + testCasePinData: IPinData, + userId: string, + ): Promise { + const data: IWorkflowExecutionDataProcess = { + executionMode: 'evaluation', + runData: {}, + pinData: testCasePinData, + workflowData: workflow, + partialExecutionVersion: '-1', + userId, + }; + + // Trigger the workflow under test with mocked data + const executionId = await this.workflowRunner.run(data); + assert(executionId); + + // Wait for the workflow to finish execution + const executePromise = this.activeExecutions.getPostExecutePromise(executionId); + + return await executePromise; + } + + /** + * Creates a new test run for the given test definition. + */ + public async runTest(user: User, test: TestDefinition): Promise { + const workflow = await this.workflowRepository.findById(test.workflowId); + assert(workflow, 'Workflow not found'); + + // 1. Make test cases from previous executions + + // Select executions with the annotation tag and workflow ID of the test. + // Fetch only ids to reduce the data transfer. + const pastExecutions: ReadonlyArray> = + await this.executionRepository + .createQueryBuilder('execution') + .select('execution.id') + .leftJoin('execution.annotation', 'annotation') + .leftJoin('annotation.tags', 'annotationTag') + .where('annotationTag.id = :tagId', { tagId: test.annotationTagId }) + .andWhere('execution.workflowId = :workflowId', { workflowId: test.workflowId }) + .getMany(); + + // 2. Run the test cases + + for (const { id: pastExecutionId } of pastExecutions) { + const pastExecution = await this.executionRepository.findOne({ + where: { id: pastExecutionId }, + relations: ['executionData', 'metadata'], + }); + assert(pastExecution, 'Execution not found'); + + const pinData = this.createPinDataFromExecution(workflow, pastExecution); + + // Run the test case and wait for it to finish + const execution = await this.runTestCase(workflow, pinData, user.id); + + if (!execution) { + continue; + } + + // TODO: 2.3 Collect the run data + } + } +} diff --git a/packages/cli/test/integration/evaluation/test-definitions.api.test.ts b/packages/cli/test/integration/evaluation/test-definitions.api.test.ts index 436363246d..b644cbe4ea 100644 --- a/packages/cli/test/integration/evaluation/test-definitions.api.test.ts +++ b/packages/cli/test/integration/evaluation/test-definitions.api.test.ts @@ -1,9 +1,11 @@ +import { mockInstance } from 'n8n-core/test/utils'; import { Container } from 'typedi'; import type { AnnotationTagEntity } from '@/databases/entities/annotation-tag-entity.ee'; import type { User } from '@/databases/entities/user'; import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; import { TestDefinitionRepository } from '@/databases/repositories/test-definition.repository.ee'; +import { TestRunnerService } from '@/evaluation/test-runner/test-runner.service.ee'; import { createAnnotationTags } from '@test-integration/db/executions'; import { createUserShell } from './../shared/db/users'; @@ -12,6 +14,8 @@ import * as testDb from './../shared/test-db'; import type { SuperAgentTest } from './../shared/types'; import * as utils from './../shared/utils/'; +const testRunner = mockInstance(TestRunnerService); + let authOwnerAgent: SuperAgentTest; let workflowUnderTest: WorkflowEntity; let workflowUnderTest2: WorkflowEntity; @@ -426,3 +430,24 @@ describe('DELETE /evaluation/test-definitions/:id', () => { expect(resp.body.message).toBe('Test definition not found'); }); }); + +describe('POST /evaluation/test-definitions/:id/run', () => { + test('should trigger the test run', async () => { + const newTest = Container.get(TestDefinitionRepository).create({ + name: 'test', + workflow: { id: workflowUnderTest.id }, + }); + await Container.get(TestDefinitionRepository).save(newTest); + + const resp = await authOwnerAgent.post(`/evaluation/test-definitions/${newTest.id}/run`); + + expect(resp.statusCode).toBe(202); + expect(resp.body).toEqual( + expect.objectContaining({ + success: true, + }), + ); + + expect(testRunner.runTest).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index 6aac7f9317..e766a4aa21 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -2371,7 +2371,8 @@ export type WorkflowExecuteMode = | 'manual' | 'retry' | 'trigger' - | 'webhook'; + | 'webhook' + | 'evaluation'; export type WorkflowActivateMode = | 'init'