mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-03-05 20:50:17 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(core): Initial TestRunner service with basic test execution (no-changelog) (#11735)

Browse source
This commit is contained in:
Eugene 2024-11-26 16:04:24 +01:00 committed by GitHub
parent 6b23ad0c12
commit 845ba6c917
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 661 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
packages/cli/src/databases/repositories/execution.repository.ts
View file

@ -163,7 +163,13 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
if (!queryParams.relations) {
queryParams.relations = [];
}
(queryParams.relations as string[]).push('executionData', 'metadata');
if (Array.isArray(queryParams.relations)) {
queryParams.relations.push('executionData', 'metadata');
} else {
queryParams.relations.executionData = true;
queryParams.relations.metadata = true;
}
}
const executions = await this.find(queryParams);

22
packages/cli/src/evaluation/test-definitions.controller.ee.ts
View file

@ -8,6 +8,7 @@ import {
testDefinitionCreateRequestBodySchema,
testDefinitionPatchRequestBodySchema,
} from '@/evaluation/test-definition.schema';
import { TestRunnerService } from '@/evaluation/test-runner/test-runner.service.ee';
import { listQueryMiddleware } from '@/middlewares';
import { getSharedWorkflowIds } from '@/public-api/v1/handlers/workflows/workflows.service';
@ -16,7 +17,10 @@ import { TestDefinitionsRequest } from './test-definitions.types.ee';
@RestController('/evaluation/test-definitions')
export class TestDefinitionsController {
constructor(private readonly testDefinitionService: TestDefinitionService) {}
constructor(
private readonly testDefinitionService: TestDefinitionService,
private readonly testRunnerService: TestRunnerService,
) {}
@Get('/', { middlewares: listQueryMiddleware })
async getMany(req: TestDefinitionsRequest.GetMany) {
@ -125,4 +129,20 @@ export class TestDefinitionsController {
return testDefinition;
}
@Post('/:id/run')
async runTest(req: TestDefinitionsRequest.Run, res: express.Response) {
const { id: testDefinitionId } = req.params;
const workflowIds = await getSharedWorkflowIds(req.user, ['workflow:read']);
// Check test definition exists
const testDefinition = await this.testDefinitionService.findOne(testDefinitionId, workflowIds);
if (!testDefinition) throw new NotFoundError('Test definition not found');
// We do not await for the test run to complete
void this.testRunnerService.runTest(req.user, testDefinition);
res.status(202).json({ success: true });
}
}

2
packages/cli/src/evaluation/test-definitions.types.ee.ts
View file

@ -30,4 +30,6 @@ export declare namespace TestDefinitionsRequest {
>;
type Delete = AuthenticatedRequest<RouteParams.TestId>;
type Run = AuthenticatedRequest<RouteParams.TestId>;
}

171
packages/cli/src/evaluation/test-runner/__tests__/mock-data/execution-data.json Normal file
View file

@ -0,0 +1,171 @@
{
"startData": {},
"resultData": {
"runData": {
"When clicking ‘Test workflow’": [
{
"hints": [],
"startTime": 1731079118048,
"executionTime": 0,
"source": [],
"executionStatus": "success",
"data": {
"main": [
[
{
"json": {
"query": "First item"
},
"pairedItem": {
"item": 0
}
},
{
"json": {
"query": "Second item"
},
"pairedItem": {
"item": 0
}
},
{
"json": {
"query": "Third item"
},
"pairedItem": {
"item": 0
}
}
]
]
}
}
],
"Edit Fields": [
{
"hints": [],
"startTime": 1731079118049,
"executionTime": 0,
"source": [
{
"previousNode": "When clicking ‘Test workflow’"
}
],
"executionStatus": "success",
"data": {
"main": [
[
{
"json": {
"foo": "bar"
},
"pairedItem": {
"item": 0
}
},
{
"json": {
"foo": "bar"
},
"pairedItem": {
"item": 1
}
},
{
"json": {
"foo": "bar"
},
"pairedItem": {
"item": 2
}
}
]
]
}
}
],
"Code": [
{
"hints": [],
"startTime": 1731079118049,
"executionTime": 3,
"source": [
{
"previousNode": "Edit Fields"
}
],
"executionStatus": "success",
"data": {
"main": [
[
{
"json": {
"foo": "bar",
"random": 0.6315509336851373
},
"pairedItem": {
"item": 0
}
},
{
"json": {
"foo": "bar",
"random": 0.3336315687359024
},
"pairedItem": {
"item": 1
}
},
{
"json": {
"foo": "bar",
"random": 0.4241870158917733
},
"pairedItem": {
"item": 2
}
}
]
]
}
}
]
},
"pinData": {
"When clicking ‘Test workflow’": [
{
"json": {
"query": "First item"
},
"pairedItem": {
"item": 0
}
},
{
"json": {
"query": "Second item"
},
"pairedItem": {
"item": 0
}
},
{
"json": {
"query": "Third item"
},
"pairedItem": {
"item": 0
}
}
]
},
"lastNodeExecuted": "Code"
},
"executionData": {
"contextData": {},
"nodeExecutionStack": [],
"metadata": {},
"waitingExecution": {},
"waitingExecutionSource": {}
}
}

124
packages/cli/src/evaluation/test-runner/__tests__/mock-data/workflow.evaluation.json Normal file
View file

@ -0,0 +1,124 @@
{
"name": "Evaluation Workflow",
"nodes": [
{
"parameters": {},
"id": "285ac92b-256f-4bb2-a450-6486b01593cb",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"typeVersion": 1,
"position": [520, 340]
},
{
"parameters": {
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "strict",
"version": 2
},
"conditions": [
{
"id": "9d3abc8d-3270-4bec-9a59-82622d5dbb5a",
"leftValue": "={{ $json.actual.Code[0].data.main[0].length }}",
"rightValue": 3,
"operator": {
"type": "number",
"operation": "gte"
}
},
{
"id": "894ce84b-13a4-4415-99c0-0c25182903bb",
"leftValue": "={{ $json.actual.Code[0].data.main[0][0].json.random }}",
"rightValue": 0.7,
"operator": {
"type": "number",
"operation": "lt"
}
}
],
"combinator": "and"
},
"options": {}
},
"id": "320b0355-3886-41df-b039-4666bf28e47b",
"name": "If",
"type": "n8n-nodes-base.if",
"typeVersion": 2.2,
"position": [740, 340]
},
{
"parameters": {
"assignments": {
"assignments": [
{
"id": "3b65d55a-158f-40c6-9853-a1c44b7ba1e5",
"name": "success",
"value": true,
"type": "boolean"
}
]
},
"options": {}
},
"id": "0c7a1ee8-0cf0-4d7f-99a3-186bbcd8815a",
"name": "Success",
"type": "n8n-nodes-base.set",
"typeVersion": 3.4,
"position": [980, 220]
},
{
"parameters": {
"assignments": {
"assignments": [
{
"id": "6cc8b402-4a30-4873-b825-963a1f1b8b82",
"name": "success",
"value": false,
"type": "boolean"
}
]
},
"options": {}
},
"id": "50d3f84a-d99f-4e04-bdbd-3e8c2668e708",
"name": "Fail",
"type": "n8n-nodes-base.set",
"typeVersion": 3.4,
"position": [980, 420]
}
],
"connections": {
"Execute Workflow Trigger": {
"main": [
[
{
"node": "If",
"type": "main",
"index": 0
}
]
]
},
"If": {
"main": [
[
{
"node": "Success",
"type": "main",
"index": 0
}
],
[
{
"node": "Fail",
"type": "main",
"index": 0
}
]
]
}
},
"pinData": {}
}

78
packages/cli/src/evaluation/test-runner/__tests__/mock-data/workflow.under-test.json Normal file
View file

@ -0,0 +1,78 @@
{
"name": "Workflow Under Test",
"nodes": [
{
"parameters": {},
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [-80, 0],
"id": "72256d90-3a67-4e29-b032-47df4e5768af",
"name": "When clicking ‘Test workflow’"
},
{
"parameters": {
"assignments": {
"assignments": [
{
"id": "acfeecbe-443c-4220-b63b-d44d69216902",
"name": "foo",
"value": "bar",
"type": "string"
}
]
},
"options": {}
},
"type": "n8n-nodes-base.set",
"typeVersion": 3.4,
"position": [140, 0],
"id": "319f29bc-1dd4-4122-b223-c584752151a4",
"name": "Edit Fields"
},
{
"parameters": {
"jsCode": "for (const item of $input.all()) {\n item.json.random = Math.random();\n}\n\nreturn $input.all();"
},
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [380, 0],
"id": "d2474215-63af-40a4-a51e-0ea30d762621",
"name": "Code"
}
],
"connections": {
"When clicking ‘Test workflow’": {
"main": [
[
{
"node": "Edit Fields",
"type": "main",
"index": 0
}
]
]
},
"Edit Fields": {
"main": [
[
{
"node": "Wait",
"type": "main",
"index": 0
}
]
]
},
"Wait": {
"main": [
[
{
"node": "Code",
"type": "main",
"index": 0
}
]
]
}
}
}

102
packages/cli/src/evaluation/test-runner/__tests__/test-runner.service.ee.test.ts Normal file
View file

@ -0,0 +1,102 @@
import type { SelectQueryBuilder } from '@n8n/typeorm';
import { stringify } from 'flatted';
import { readFileSync } from 'fs';
import { mock, mockDeep } from 'jest-mock-extended';
import path from 'path';
import type { ActiveExecutions } from '@/active-executions';
import type { ExecutionEntity } from '@/databases/entities/execution-entity';
import type { TestDefinition } from '@/databases/entities/test-definition.ee';
import type { User } from '@/databases/entities/user';
import type { ExecutionRepository } from '@/databases/repositories/execution.repository';
import type { WorkflowRepository } from '@/databases/repositories/workflow.repository';
import type { WorkflowRunner } from '@/workflow-runner';
import { TestRunnerService } from '../test-runner.service.ee';
const wfUnderTestJson = JSON.parse(
readFileSync(path.join(__dirname, './mock-data/workflow.under-test.json'), { encoding: 'utf-8' }),
);
const executionDataJson = JSON.parse(
readFileSync(path.join(__dirname, './mock-data/execution-data.json'), { encoding: 'utf-8' }),
);
const executionMocks = [
mock<ExecutionEntity>({
id: 'some-execution-id',
workflowId: 'workflow-under-test-id',
status: 'success',
executionData: {
data: stringify(executionDataJson),
},
}),
mock<ExecutionEntity>({
id: 'some-execution-id-2',
workflowId: 'workflow-under-test-id',
status: 'success',
executionData: {
data: stringify(executionDataJson),
},
}),
];
describe('TestRunnerService', () => {
const executionRepository = mock<ExecutionRepository>();
const workflowRepository = mock<WorkflowRepository>();
const workflowRunner = mock<WorkflowRunner>();
const activeExecutions = mock<ActiveExecutions>();
beforeEach(() => {
const executionsQbMock = mockDeep<SelectQueryBuilder<ExecutionEntity>>({
fallbackMockImplementation: jest.fn().mockReturnThis(),
});
executionsQbMock.getMany.mockResolvedValueOnce(executionMocks);
executionRepository.createQueryBuilder.mockReturnValueOnce(executionsQbMock);
executionRepository.findOne
.calledWith(expect.objectContaining({ where: { id: 'some-execution-id' } }))
.mockResolvedValueOnce(executionMocks[0]);
executionRepository.findOne
.calledWith(expect.objectContaining({ where: { id: 'some-execution-id-2' } }))
.mockResolvedValueOnce(executionMocks[1]);
});
test('should create an instance of TestRunnerService', async () => {
const testRunnerService = new TestRunnerService(
workflowRepository,
workflowRunner,
executionRepository,
activeExecutions,
);
expect(testRunnerService).toBeInstanceOf(TestRunnerService);
});
test('should create and run test cases from past executions', async () => {
const testRunnerService = new TestRunnerService(
workflowRepository,
workflowRunner,
executionRepository,
activeExecutions,
);
workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({
id: 'workflow-under-test-id',
...wfUnderTestJson,
});
workflowRunner.run.mockResolvedValue('test-execution-id');
await testRunnerService.runTest(
mock<User>(),
mock<TestDefinition>({
workflowId: 'workflow-under-test-id',
}),
);
expect(executionRepository.createQueryBuilder).toHaveBeenCalledTimes(1);
expect(executionRepository.findOne).toHaveBeenCalledTimes(2);
expect(workflowRunner.run).toHaveBeenCalledTimes(2);
});
});

129
packages/cli/src/evaluation/test-runner/test-runner.service.ee.ts Normal file
View file

@ -0,0 +1,129 @@
import { parse } from 'flatted';
import type { IPinData, IRun, IWorkflowExecutionDataProcess } from 'n8n-workflow';
import assert from 'node:assert';
import { Service } from 'typedi';
import { ActiveExecutions } from '@/active-executions';
import type { ExecutionEntity } from '@/databases/entities/execution-entity';
import type { TestDefinition } from '@/databases/entities/test-definition.ee';
import type { User } from '@/databases/entities/user';
import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
import { ExecutionRepository } from '@/databases/repositories/execution.repository';
import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
import type { IExecutionResponse } from '@/interfaces';
import { WorkflowRunner } from '@/workflow-runner';
/**
* This service orchestrates the running of test cases.
* It uses the test definitions to find
* past executions, creates pin data from them,
* and runs the workflow-under-test with the pin data.
* TODO: Evaluation workflows
* TODO: Node pinning
* TODO: Collect metrics
*/
@Service()
export class TestRunnerService {
constructor(
private readonly workflowRepository: WorkflowRepository,
private readonly workflowRunner: WorkflowRunner,
private readonly executionRepository: ExecutionRepository,
private readonly activeExecutions: ActiveExecutions,
) {}
/**
* Creates a pin data object from the past execution data
* for the given workflow.
* For now, it only pins trigger nodes.
*/
private createPinDataFromExecution(
workflow: WorkflowEntity,
execution: ExecutionEntity,
): IPinData {
const executionData = parse(execution.executionData.data) as IExecutionResponse['data'];
const triggerNodes = workflow.nodes.filter((node) => /trigger$/i.test(node.type));
const pinData = {} as IPinData;
for (const triggerNode of triggerNodes) {
const triggerData = executionData.resultData.runData[triggerNode.name];
if (triggerData?.[0]?.data?.main?.[0]) {
pinData[triggerNode.name] = triggerData[0]?.data?.main?.[0];
}
}
return pinData;
}
/**
* Runs a test case with the given pin data.
* Waits for the workflow under test to finish execution.
*/
private async runTestCase(
workflow: WorkflowEntity,
testCasePinData: IPinData,
userId: string,
): Promise<IRun | undefined> {
const data: IWorkflowExecutionDataProcess = {
executionMode: 'evaluation',
runData: {},
pinData: testCasePinData,
workflowData: workflow,
partialExecutionVersion: '-1',
userId,
};
// Trigger the workflow under test with mocked data
const executionId = await this.workflowRunner.run(data);
assert(executionId);
// Wait for the workflow to finish execution
const executePromise = this.activeExecutions.getPostExecutePromise(executionId);
return await executePromise;
}
/**
* Creates a new test run for the given test definition.
*/
public async runTest(user: User, test: TestDefinition): Promise<void> {
const workflow = await this.workflowRepository.findById(test.workflowId);
assert(workflow, 'Workflow not found');
// 1. Make test cases from previous executions
// Select executions with the annotation tag and workflow ID of the test.
// Fetch only ids to reduce the data transfer.
const pastExecutions: ReadonlyArray<Pick<ExecutionEntity, 'id'>> =
await this.executionRepository
.createQueryBuilder('execution')
.select('execution.id')
.leftJoin('execution.annotation', 'annotation')
.leftJoin('annotation.tags', 'annotationTag')
.where('annotationTag.id = :tagId', { tagId: test.annotationTagId })
.andWhere('execution.workflowId = :workflowId', { workflowId: test.workflowId })
.getMany();
// 2. Run the test cases
for (const { id: pastExecutionId } of pastExecutions) {
const pastExecution = await this.executionRepository.findOne({
where: { id: pastExecutionId },
relations: ['executionData', 'metadata'],
});
assert(pastExecution, 'Execution not found');
const pinData = this.createPinDataFromExecution(workflow, pastExecution);
// Run the test case and wait for it to finish
const execution = await this.runTestCase(workflow, pinData, user.id);
if (!execution) {
continue;
}
// TODO: 2.3 Collect the run data
}
}
}

25
packages/cli/test/integration/evaluation/test-definitions.api.test.ts
View file

@ -1,9 +1,11 @@
import { mockInstance } from 'n8n-core/test/utils';
import { Container } from 'typedi';
import type { AnnotationTagEntity } from '@/databases/entities/annotation-tag-entity.ee';
import type { User } from '@/databases/entities/user';
import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
import { TestDefinitionRepository } from '@/databases/repositories/test-definition.repository.ee';
import { TestRunnerService } from '@/evaluation/test-runner/test-runner.service.ee';
import { createAnnotationTags } from '@test-integration/db/executions';
import { createUserShell } from './../shared/db/users';
@ -12,6 +14,8 @@ import * as testDb from './../shared/test-db';
import type { SuperAgentTest } from './../shared/types';
import * as utils from './../shared/utils/';
const testRunner = mockInstance(TestRunnerService);
let authOwnerAgent: SuperAgentTest;
let workflowUnderTest: WorkflowEntity;
let workflowUnderTest2: WorkflowEntity;
@ -426,3 +430,24 @@ describe('DELETE /evaluation/test-definitions/:id', () => {
expect(resp.body.message).toBe('Test definition not found');
});
});
describe('POST /evaluation/test-definitions/:id/run', () => {
test('should trigger the test run', async () => {
const newTest = Container.get(TestDefinitionRepository).create({
name: 'test',
workflow: { id: workflowUnderTest.id },
});
await Container.get(TestDefinitionRepository).save(newTest);
const resp = await authOwnerAgent.post(`/evaluation/test-definitions/${newTest.id}/run`);
expect(resp.statusCode).toBe(202);
expect(resp.body).toEqual(
expect.objectContaining({
success: true,
}),
);
expect(testRunner.runTest).toHaveBeenCalledTimes(1);
});
});

3
packages/workflow/src/Interfaces.ts
View file

@ -2371,7 +2371,8 @@ export type WorkflowExecuteMode =
| 'manual'
| 'retry'
| 'trigger'
| 'webhook';
| 'webhook'
| 'evaluation';
export type WorkflowActivateMode =
| 'init'