n8n/packages/cli/test/integration/evaluation/test-definitions.api.test.ts

import { mockInstance } from 'n8n-core/test/utils';
import { Container } from 'typedi';

import type { AnnotationTagEntity } from '@/databases/entities/annotation-tag-entity.ee';
import type { User } from '@/databases/entities/user';
import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
import { TestDefinitionRepository } from '@/databases/repositories/test-definition.repository.ee';
import { TestRunnerService } from '@/evaluation/test-runner/test-runner.service.ee';
import { createAnnotationTags } from '@test-integration/db/executions';

import { createUserShell } from './../shared/db/users';
import { createWorkflow } from './../shared/db/workflows';
import * as testDb from './../shared/test-db';
import type { SuperAgentTest } from './../shared/types';
import * as utils from './../shared/utils/';

const testRunner = mockInstance(TestRunnerService);

let authOwnerAgent: SuperAgentTest;
let workflowUnderTest: WorkflowEntity;
let workflowUnderTest2: WorkflowEntity;
let evaluationWorkflow: WorkflowEntity;
let otherWorkflow: WorkflowEntity;
let ownerShell: User;
let annotationTag: AnnotationTagEntity;
const testServer = utils.setupTestServer({ endpointGroups: ['evaluation'] });

beforeAll(async () => {
	ownerShell = await createUserShell('global:owner');
	authOwnerAgent = testServer.authAgentFor(ownerShell);
});

beforeEach(async () => {
	await testDb.truncate(['TestDefinition', 'Workflow', 'AnnotationTag']);

	workflowUnderTest = await createWorkflow({ name: 'workflow-under-test' }, ownerShell);
	workflowUnderTest2 = await createWorkflow({ name: 'workflow-under-test-2' }, ownerShell);
	evaluationWorkflow = await createWorkflow({ name: 'evaluation-workflow' }, ownerShell);
	otherWorkflow = await createWorkflow({ name: 'other-workflow' });
	annotationTag = (await createAnnotationTags(['test-tag']))[0];
});

describe('GET /evaluation/test-definitions', () => {
	test('should retrieve empty test definitions list', async () => {
		const resp = await authOwnerAgent.get('/evaluation/test-definitions');
		expect(resp.statusCode).toBe(200);
		expect(resp.body.data.count).toBe(0);
		expect(resp.body.data.testDefinitions).toHaveLength(0);
	});

	test('should retrieve test definitions list', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.get('/evaluation/test-definitions');

		expect(resp.statusCode).toBe(200);
		expect(resp.body.data).toEqual({
			count: 1,
			testDefinitions: [
				expect.objectContaining({
					name: 'test',
					workflowId: workflowUnderTest.id,
					evaluationWorkflowId: null,
				}),
			],
		});
	});

	test('should retrieve test definitions list with pagination', async () => {
		// Add a bunch of test definitions
		const testDefinitions = [];

		for (let i = 0; i < 15; i++) {
			const newTest = Container.get(TestDefinitionRepository).create({
				name: `test-${i}`,
				workflow: { id: workflowUnderTest.id },
			});
			testDefinitions.push(newTest);
		}

		await Container.get(TestDefinitionRepository).save(testDefinitions);

		// Fetch the first page
		let resp = await authOwnerAgent.get('/evaluation/test-definitions?take=10');
		expect(resp.statusCode).toBe(200);
		expect(resp.body.data.count).toBe(15);
		expect(resp.body.data.testDefinitions).toHaveLength(10);

		// Fetch the second page
		resp = await authOwnerAgent.get('/evaluation/test-definitions?take=10&skip=10');
		expect(resp.statusCode).toBe(200);
		expect(resp.body.data.count).toBe(15);
		expect(resp.body.data.testDefinitions).toHaveLength(5);
	});

	test('should retrieve test definitions list for a workflow', async () => {
		// Add a bunch of test definitions for two different workflows
		const testDefinitions = [];

		for (let i = 0; i < 15; i++) {
			const newTest = Container.get(TestDefinitionRepository).create({
				name: `test-${i}`,
				workflow: { id: workflowUnderTest.id },
			});

			const newTest2 = Container.get(TestDefinitionRepository).create({
				name: `test-${i * 2}`,
				workflow: { id: workflowUnderTest2.id },
			});

			testDefinitions.push(newTest, newTest2);
		}

		await Container.get(TestDefinitionRepository).save(testDefinitions);

		// Fetch test definitions of a second workflow
		let resp = await authOwnerAgent.get(
			`/evaluation/test-definitions?filter=${JSON.stringify({ workflowId: workflowUnderTest2.id })}`,
		);

		expect(resp.statusCode).toBe(200);
		expect(resp.body.data.count).toBe(15);
	});

	test('should return error if user has no access to the workflowId specified in filter', async () => {
		let resp = await authOwnerAgent.get(
			`/evaluation/test-definitions?filter=${JSON.stringify({ workflowId: otherWorkflow.id })}`,
		);

		expect(resp.statusCode).toBe(403);
		expect(resp.body.message).toBe('User does not have access to the workflow');
	});
});

describe('GET /evaluation/test-definitions/:id', () => {
	test('should retrieve test definition', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.get(`/evaluation/test-definitions/${newTest.id}`);

		expect(resp.statusCode).toBe(200);
		expect(resp.body.data.name).toBe('test');
		expect(resp.body.data.workflowId).toBe(workflowUnderTest.id);
		expect(resp.body.data.evaluationWorkflowId).toBe(null);
	});

	test('should return 404 for non-existent test definition', async () => {
		const resp = await authOwnerAgent.get('/evaluation/test-definitions/123');

		expect(resp.statusCode).toBe(404);
	});

	test('should retrieve test definition with evaluation workflow', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
			evaluationWorkflow: { id: evaluationWorkflow.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.get(`/evaluation/test-definitions/${newTest.id}`);

		expect(resp.statusCode).toBe(200);
		expect(resp.body.data.name).toBe('test');
		expect(resp.body.data.workflowId).toBe(workflowUnderTest.id);
		expect(resp.body.data.evaluationWorkflowId).toBe(evaluationWorkflow.id);
	});

	test('should not retrieve test definition if user does not have access to workflow under test', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: otherWorkflow.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.get(`/evaluation/test-definitions/${newTest.id}`);

		expect(resp.statusCode).toBe(404);
	});
});

describe('POST /evaluation/test-definitions', () => {
	test('should create test definition', async () => {
		const resp = await authOwnerAgent.post('/evaluation/test-definitions').send({
			name: 'test',
			workflowId: workflowUnderTest.id,
		});

		expect(resp.statusCode).toBe(200);
		expect(resp.body.data.name).toBe('test');
		expect(resp.body.data.workflowId).toBe(workflowUnderTest.id);
	});

	test('should create test definition with evaluation workflow', async () => {
		const resp = await authOwnerAgent.post('/evaluation/test-definitions').send({
			name: 'test',
			workflowId: workflowUnderTest.id,
			evaluationWorkflowId: evaluationWorkflow.id,
		});

		expect(resp.statusCode).toBe(200);
		expect(resp.body.data).toEqual(
			expect.objectContaining({
				name: 'test',
				workflowId: workflowUnderTest.id,
				evaluationWorkflowId: evaluationWorkflow.id,
			}),
		);
	});

	test('should create test definition with all fields', async () => {
		const resp = await authOwnerAgent.post('/evaluation/test-definitions').send({
			name: 'test',
			description: 'test description',
			workflowId: workflowUnderTest.id,
			evaluationWorkflowId: evaluationWorkflow.id,
			annotationTagId: annotationTag.id,
		});

		expect(resp.statusCode).toBe(200);
		expect(resp.body.data).toEqual(
			expect.objectContaining({
				name: 'test',
				description: 'test description',
				workflowId: workflowUnderTest.id,
				evaluationWorkflowId: evaluationWorkflow.id,
				annotationTag: expect.objectContaining({
					id: annotationTag.id,
				}),
			}),
		);
	});

	test('should return error if name is empty', async () => {
		const resp = await authOwnerAgent.post('/evaluation/test-definitions').send({
			name: '',
			workflowId: workflowUnderTest.id,
		});

		expect(resp.statusCode).toBe(400);
		expect(resp.body.errors).toEqual(
			expect.arrayContaining([
				expect.objectContaining({
					code: 'too_small',
					path: ['name'],
				}),
			]),
		);
	});

	test('should return error if user has no access to the workflow', async () => {
		const resp = await authOwnerAgent.post('/evaluation/test-definitions').send({
			name: 'test',
			workflowId: otherWorkflow.id,
		});

		expect(resp.statusCode).toBe(403);
		expect(resp.body.message).toBe('User does not have access to the workflow');
	});

	test('should return error if user has no access to the evaluation workflow', async () => {
		const resp = await authOwnerAgent.post('/evaluation/test-definitions').send({
			name: 'test',
			workflowId: workflowUnderTest.id,
			evaluationWorkflowId: otherWorkflow.id,
		});

		expect(resp.statusCode).toBe(403);
		expect(resp.body.message).toBe('User does not have access to the evaluation workflow');
	});
});

describe('PATCH /evaluation/test-definitions/:id', () => {
	test('should update test definition', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
			name: 'updated-test',
		});

		expect(resp.statusCode).toBe(200);
		expect(resp.body.data.name).toBe('updated-test');
	});

	test('should return 404 if user has no access to the workflow', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: otherWorkflow.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
			name: 'updated-test',
		});

		expect(resp.statusCode).toBe(404);
		expect(resp.body.message).toBe('Test definition not found');
	});

	test('should update test definition with evaluation workflow', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
			name: 'updated-test',
			evaluationWorkflowId: evaluationWorkflow.id,
		});

		expect(resp.statusCode).toBe(200);
		expect(resp.body.data.name).toBe('updated-test');
		expect(resp.body.data.evaluationWorkflowId).toBe(evaluationWorkflow.id);
	});

	test('should return error if user has no access to the evaluation workflow', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
			name: 'updated-test',
			evaluationWorkflowId: otherWorkflow.id,
		});

		expect(resp.statusCode).toBe(403);
		expect(resp.body.message).toBe('User does not have access to the evaluation workflow');
	});

	test('should disallow workflowId', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
			name: 'updated-test',
			workflowId: otherWorkflow.id,
		});

		expect(resp.statusCode).toBe(400);
		expect(resp.body.errors).toEqual(
			expect.arrayContaining([
				expect.objectContaining({
					code: 'unrecognized_keys',
					keys: ['workflowId'],
				}),
			]),
		);
	});

	test('should update annotationTagId', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
			annotationTagId: annotationTag.id,
		});

		expect(resp.statusCode).toBe(200);
		expect(resp.body.data.annotationTag.id).toBe(annotationTag.id);
	});

	test('should return error if annotationTagId is invalid', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
			annotationTagId: '123',
		});

		expect(resp.statusCode).toBe(400);
		expect(resp.body.message).toBe('Annotation tag not found');
	});

	test('should update pinned nodes', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
			mockedNodes: [
				{
					name: 'Schedule Trigger',
				},
			],
		});

		expect(resp.statusCode).toBe(200);
		expect(resp.body.data.mockedNodes).toEqual([{ name: 'Schedule Trigger' }]);
	});

	test('should return error if pinned nodes are invalid', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
			mockedNodes: ['Simple string'],
		});

		expect(resp.statusCode).toBe(400);
	});

	test('should return error if pinned nodes are not in the workflow', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
			mockedNodes: [
				{
					name: 'Invalid Node',
				},
			],
		});

		expect(resp.statusCode).toBe(400);
	});
});

describe('DELETE /evaluation/test-definitions/:id', () => {
	test('should delete test definition', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.delete(`/evaluation/test-definitions/${newTest.id}`);

		expect(resp.statusCode).toBe(200);
		expect(resp.body.data.success).toBe(true);
	});

	test('should return 404 if test definition does not exist', async () => {
		const resp = await authOwnerAgent.delete('/evaluation/test-definitions/123');

		expect(resp.statusCode).toBe(404);
		expect(resp.body.message).toBe('Test definition not found');
	});

	test('should return 404 if user has no access to the workflow', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: otherWorkflow.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.delete(`/evaluation/test-definitions/${newTest.id}`);

		expect(resp.statusCode).toBe(404);
		expect(resp.body.message).toBe('Test definition not found');
	});
});

describe('POST /evaluation/test-definitions/:id/run', () => {
	test('should trigger the test run', async () => {
		const newTest = Container.get(TestDefinitionRepository).create({
			name: 'test',
			workflow: { id: workflowUnderTest.id },
		});
		await Container.get(TestDefinitionRepository).save(newTest);

		const resp = await authOwnerAgent.post(`/evaluation/test-definitions/${newTest.id}/run`);

		expect(resp.statusCode).toBe(202);
		expect(resp.body).toEqual(
			expect.objectContaining({
				success: true,
			}),
		);

		expect(testRunner.runTest).toHaveBeenCalledTimes(1);
	});
});