feat(core): API endpoints for Test Runs (no-changelog) (#11882)

packages/cli/src/databases/repositories/test-run.repository.ee.ts

@@ -1,8 +1,10 @@
+import type { FindManyOptions } from '@n8n/typeorm';
 import { DataSource, Repository } from '@n8n/typeorm';
 import { Service } from 'typedi';
 
 import type { AggregatedTestRunMetrics } from '@/databases/entities/test-run.ee';
 import { TestRun } from '@/databases/entities/test-run.ee';
+import type { ListQuery } from '@/requests';
 
 @Service()
 export class TestRunRepository extends Repository<TestRun> {
@@ -26,4 +28,18 @@ export class TestRunRepository extends Repository<TestRun> {
 	public async markAsCompleted(id: string, metrics: AggregatedTestRunMetrics) {
 		return await this.update(id, { status: 'completed', completedAt: new Date(), metrics });
 	}
+
+	public async getMany(testDefinitionId: string, options: ListQuery.Options) {
+		const findManyOptions: FindManyOptions<TestRun> = {
+			where: { testDefinition: { id: testDefinitionId } },
+			order: { createdAt: 'DESC' },
+		};
+
+		if (options?.take) {
+			findManyOptions.skip = options.skip;
+			findManyOptions.take = options.take;
+		}
+
+		return await this.find(findManyOptions);
+	}
 }

packages/cli/src/evaluation/metrics.controller.ts

@@ -112,7 +112,7 @@ export class TestMetricsController {
 	}
 
 	@Delete('/:testDefinitionId/metrics/:id')
-	async delete(req: TestMetricsRequest.GetOne) {
+	async delete(req: TestMetricsRequest.Delete) {
 		const { id: metricId, testDefinitionId } = req.params;
 
 		await this.getTestDefinition(req);

packages/cli/src/evaluation/test-definitions.types.ee.ts

@@ -13,7 +13,7 @@ export declare namespace TestDefinitionsRequest {
 
 	type GetOne = AuthenticatedRequest<RouteParams.TestId>;
 
-	type GetMany = AuthenticatedRequest<{}, {}, {}, ListQuery.Params & { includeScopes?: string }> & {
+	type GetMany = AuthenticatedRequest<{}, {}, {}, ListQuery.Params> & {
 		listQueryOptions: ListQuery.Options;
 	};
 
@@ -63,3 +63,27 @@ export declare namespace TestMetricsRequest {
 
 	type Delete = AuthenticatedRequest<RouteParams.TestDefinitionId & RouteParams.TestMetricId>;
 }
+
+// ----------------------------------
+// 					 /test-definitions/:testDefinitionId/runs
+// ----------------------------------
+
+export declare namespace TestRunsRequest {
+	namespace RouteParams {
+		type TestId = {
+			testDefinitionId: string;
+		};
+
+		type TestRunId = {
+			id: string;
+		};
+	}
+
+	type GetMany = AuthenticatedRequest<RouteParams.TestId, {}, {}, ListQuery.Params> & {
+		listQueryOptions: ListQuery.Options;
+	};
+
+	type GetOne = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
+
+	type Delete = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
+}

packages/cli/src/evaluation/test-runs.controller.ee.ts

@@ -0,0 +1,77 @@
+import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee';
+import { Delete, Get, RestController } from '@/decorators';
+import { NotFoundError } from '@/errors/response-errors/not-found.error';
+import { TestRunsRequest } from '@/evaluation/test-definitions.types.ee';
+import { listQueryMiddleware } from '@/middlewares';
+import { getSharedWorkflowIds } from '@/public-api/v1/handlers/workflows/workflows.service';
+
+import { TestDefinitionService } from './test-definition.service.ee';
+
+@RestController('/evaluation/test-definitions')
+export class TestRunsController {
+	constructor(
+		private readonly testDefinitionService: TestDefinitionService,
+		private readonly testRunRepository: TestRunRepository,
+	) {}
+
+	/** This method is used in multiple places in the controller to get the test definition
+	 * (or just check that it exists and the user has access to it).
+	 */
+	private async getTestDefinition(
+		req: TestRunsRequest.GetOne | TestRunsRequest.GetMany | TestRunsRequest.Delete,
+	) {
+		const { testDefinitionId } = req.params;
+
+		const userAccessibleWorkflowIds = await getSharedWorkflowIds(req.user, ['workflow:read']);
+
+		const testDefinition = await this.testDefinitionService.findOne(
+			testDefinitionId,
+			userAccessibleWorkflowIds,
+		);
+
+		if (!testDefinition) throw new NotFoundError('Test definition not found');
+
+		return testDefinition;
+	}
+
+	@Get('/:testDefinitionId/runs', { middlewares: listQueryMiddleware })
+	async getMany(req: TestRunsRequest.GetMany) {
+		const { testDefinitionId } = req.params;
+
+		await this.getTestDefinition(req);
+
+		return await this.testRunRepository.getMany(testDefinitionId, req.listQueryOptions);
+	}
+
+	@Get('/:testDefinitionId/runs/:id')
+	async getOne(req: TestRunsRequest.GetOne) {
+		const { id: testRunId, testDefinitionId } = req.params;
+
+		await this.getTestDefinition(req);
+
+		const testRun = await this.testRunRepository.findOne({
+			where: { id: testRunId, testDefinition: { id: testDefinitionId } },
+		});
+
+		if (!testRun) throw new NotFoundError('Test run not found');
+
+		return testRun;
+	}
+
+	@Delete('/:testDefinitionId/runs/:id')
+	async delete(req: TestRunsRequest.Delete) {
+		const { id: testRunId, testDefinitionId } = req.params;
+
+		await this.getTestDefinition(req);
+
+		const testRun = await this.testRunRepository.findOne({
+			where: { id: testRunId, testDefinition: { id: testDefinitionId } },
+		});
+
+		if (!testRun) throw new NotFoundError('Test run not found');
+
+		await this.testRunRepository.delete({ id: testRunId });
+
+		return { success: true };
+	}
+}

packages/cli/src/server.ts

@@ -65,6 +65,7 @@ import '@/external-secrets/external-secrets.controller.ee';
 import '@/license/license.controller';
 import '@/evaluation/test-definitions.controller.ee';
 import '@/evaluation/metrics.controller';
+import '@/evaluation/test-runs.controller.ee';
 import '@/workflows/workflow-history/workflow-history.controller.ee';
 import '@/workflows/workflows.controller';
 

packages/cli/test/integration/evaluation/test-runs.api.test.ts

@@ -0,0 +1,239 @@
+import { Container } from 'typedi';
+
+import type { TestDefinition } from '@/databases/entities/test-definition.ee';
+import type { User } from '@/databases/entities/user';
+import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
+import { ProjectRepository } from '@/databases/repositories/project.repository';
+import { TestDefinitionRepository } from '@/databases/repositories/test-definition.repository.ee';
+import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee';
+import { createUserShell } from '@test-integration/db/users';
+import { createWorkflow } from '@test-integration/db/workflows';
+import * as testDb from '@test-integration/test-db';
+import type { SuperAgentTest } from '@test-integration/types';
+import * as utils from '@test-integration/utils';
+
+let authOwnerAgent: SuperAgentTest;
+let workflowUnderTest: WorkflowEntity;
+let otherWorkflow: WorkflowEntity;
+let testDefinition: TestDefinition;
+let otherTestDefinition: TestDefinition;
+let ownerShell: User;
+
+const testServer = utils.setupTestServer({
+	endpointGroups: ['workflows', 'evaluation'],
+	enabledFeatures: ['feat:sharing'],
+});
+
+beforeAll(async () => {
+	ownerShell = await createUserShell('global:owner');
+	authOwnerAgent = testServer.authAgentFor(ownerShell);
+});
+
+beforeEach(async () => {
+	await testDb.truncate(['TestDefinition', 'TestRun', 'Workflow', 'SharedWorkflow']);
+
+	workflowUnderTest = await createWorkflow({ name: 'workflow-under-test' }, ownerShell);
+
+	testDefinition = Container.get(TestDefinitionRepository).create({
+		name: 'test',
+		workflow: { id: workflowUnderTest.id },
+	});
+	await Container.get(TestDefinitionRepository).save(testDefinition);
+
+	otherWorkflow = await createWorkflow({ name: 'other-workflow' });
+
+	otherTestDefinition = Container.get(TestDefinitionRepository).create({
+		name: 'other-test',
+		workflow: { id: otherWorkflow.id },
+	});
+	await Container.get(TestDefinitionRepository).save(otherTestDefinition);
+});
+
+describe('GET /evaluation/test-definitions/:testDefinitionId/runs', () => {
+	test('should retrieve empty list of runs for a test definition', async () => {
+		const resp = await authOwnerAgent.get(`/evaluation/test-definitions/${testDefinition.id}/runs`);
+
+		expect(resp.statusCode).toBe(200);
+		expect(resp.body.data).toEqual([]);
+	});
+
+	test('should retrieve 404 if test definition does not exist', async () => {
+		const resp = await authOwnerAgent.get('/evaluation/test-definitions/123/runs');
+
+		expect(resp.statusCode).toBe(404);
+	});
+
+	test('should retrieve 404 if user does not have access to test definition', async () => {
+		const resp = await authOwnerAgent.get(
+			`/evaluation/test-definitions/${otherTestDefinition.id}/runs`,
+		);
+
+		expect(resp.statusCode).toBe(404);
+	});
+
+	test('should retrieve list of runs for a test definition', async () => {
+		const testRunRepository = Container.get(TestRunRepository);
+		const testRun = await testRunRepository.createTestRun(testDefinition.id);
+
+		const resp = await authOwnerAgent.get(`/evaluation/test-definitions/${testDefinition.id}/runs`);
+
+		expect(resp.statusCode).toBe(200);
+		expect(resp.body.data).toEqual([
+			expect.objectContaining({
+				id: testRun.id,
+				status: 'new',
+				testDefinitionId: testDefinition.id,
+				runAt: null,
+				completedAt: null,
+			}),
+		]);
+	});
+
+	test('should retrieve list of runs for a test definition with pagination', async () => {
+		const testRunRepository = Container.get(TestRunRepository);
+		const testRun1 = await testRunRepository.createTestRun(testDefinition.id);
+		// Mark as running just to make a slight delay between the runs
+		await testRunRepository.markAsRunning(testRun1.id);
+		const testRun2 = await testRunRepository.createTestRun(testDefinition.id);
+
+		// Fetch the first page
+		const resp = await authOwnerAgent.get(
+			`/evaluation/test-definitions/${testDefinition.id}/runs?take=1`,
+		);
+
+		expect(resp.statusCode).toBe(200);
+		expect(resp.body.data).toEqual([
+			expect.objectContaining({
+				id: testRun2.id,
+				status: 'new',
+				testDefinitionId: testDefinition.id,
+				runAt: null,
+				completedAt: null,
+			}),
+		]);
+
+		// Fetch the second page
+		const resp2 = await authOwnerAgent.get(
+			`/evaluation/test-definitions/${testDefinition.id}/runs?take=1&skip=1`,
+		);
+
+		expect(resp2.statusCode).toBe(200);
+		expect(resp2.body.data).toEqual([
+			expect.objectContaining({
+				id: testRun1.id,
+				status: 'running',
+				testDefinitionId: testDefinition.id,
+				runAt: expect.any(String),
+				completedAt: null,
+			}),
+		]);
+	});
+});
+
+describe('GET /evaluation/test-definitions/:testDefinitionId/runs/:id', () => {
+	test('should retrieve test run for a test definition', async () => {
+		const testRunRepository = Container.get(TestRunRepository);
+		const testRun = await testRunRepository.createTestRun(testDefinition.id);
+
+		const resp = await authOwnerAgent.get(
+			`/evaluation/test-definitions/${testDefinition.id}/runs/${testRun.id}`,
+		);
+
+		expect(resp.statusCode).toBe(200);
+		expect(resp.body.data).toEqual(
+			expect.objectContaining({
+				id: testRun.id,
+				status: 'new',
+				testDefinitionId: testDefinition.id,
+				runAt: null,
+				completedAt: null,
+			}),
+		);
+	});
+
+	test('should retrieve 404 if test run does not exist', async () => {
+		const resp = await authOwnerAgent.get(
+			`/evaluation/test-definitions/${testDefinition.id}/runs/123`,
+		);
+
+		expect(resp.statusCode).toBe(404);
+	});
+
+	test('should retrieve 404 if user does not have access to test definition', async () => {
+		const testRunRepository = Container.get(TestRunRepository);
+		const testRun = await testRunRepository.createTestRun(otherTestDefinition.id);
+
+		const resp = await authOwnerAgent.get(
+			`/evaluation/test-definitions/${otherTestDefinition.id}/runs/${testRun.id}`,
+		);
+
+		expect(resp.statusCode).toBe(404);
+	});
+
+	test('should retrieve test run for a test definition of a shared workflow', async () => {
+		const memberShell = await createUserShell('global:member');
+		const memberAgent = testServer.authAgentFor(memberShell);
+		const memberPersonalProject = await Container.get(
+			ProjectRepository,
+		).getPersonalProjectForUserOrFail(memberShell.id);
+
+		// Share workflow with a member
+		const sharingResponse = await authOwnerAgent
+			.put(`/workflows/${workflowUnderTest.id}/share`)
+			.send({ shareWithIds: [memberPersonalProject.id] });
+
+		expect(sharingResponse.statusCode).toBe(200);
+
+		// Create a test run for the shared workflow
+		const testRunRepository = Container.get(TestRunRepository);
+		const testRun = await testRunRepository.createTestRun(testDefinition.id);
+
+		// Check if member can retrieve the test run of a shared workflow
+		const resp = await memberAgent.get(
+			`/evaluation/test-definitions/${testDefinition.id}/runs/${testRun.id}`,
+		);
+
+		expect(resp.statusCode).toBe(200);
+		expect(resp.body.data).toEqual(
+			expect.objectContaining({
+				id: testRun.id,
+			}),
+		);
+	});
+});
+
+describe('DELETE /evaluation/test-definitions/:testDefinitionId/runs/:id', () => {
+	test('should delete test run for a test definition', async () => {
+		const testRunRepository = Container.get(TestRunRepository);
+		const testRun = await testRunRepository.createTestRun(testDefinition.id);
+
+		const resp = await authOwnerAgent.delete(
+			`/evaluation/test-definitions/${testDefinition.id}/runs/${testRun.id}`,
+		);
+
+		expect(resp.statusCode).toBe(200);
+		expect(resp.body.data).toEqual({ success: true });
+
+		const testRunAfterDelete = await testRunRepository.findOne({ where: { id: testRun.id } });
+		expect(testRunAfterDelete).toBeNull();
+	});
+
+	test('should retrieve 404 if test run does not exist', async () => {
+		const resp = await authOwnerAgent.delete(
+			`/evaluation/test-definitions/${testDefinition.id}/runs/123`,
+		);
+
+		expect(resp.statusCode).toBe(404);
+	});
+
+	test('should retrieve 404 if user does not have access to test definition', async () => {
+		const testRunRepository = Container.get(TestRunRepository);
+		const testRun = await testRunRepository.createTestRun(otherTestDefinition.id);
+
+		const resp = await authOwnerAgent.delete(
+			`/evaluation/test-definitions/${otherTestDefinition.id}/runs/${testRun.id}`,
+		);
+
+		expect(resp.statusCode).toBe(404);
+	});
+});

packages/cli/test/integration/shared/test-db.ts

@@ -76,6 +76,7 @@ const repositories = [
 	'Tag',
 	'TestDefinition',
 	'TestMetric',
+	'TestRun',
 	'User',
 	'Variables',
 	'Webhook',

packages/cli/test/integration/shared/utils/test-server.ts

@@ -281,6 +281,7 @@ export const setupTestServer = ({
 					case 'evaluation':
 						await import('@/evaluation/metrics.controller');
 						await import('@/evaluation/test-definitions.controller.ee');
+						await import('@/evaluation/test-runs.controller.ee');
 						break;
 				}
 			}