chore: Add mockedNodes property to TestDefinition (no-changelog) (#12001)

Co-authored-by: Tomi Turtiainen <10324676+tomi@users.noreply.github.com>

packages/cli/src/databases/entities/test-definition.ee.ts

@@ -5,7 +5,12 @@ import { AnnotationTagEntity } from '@/databases/entities/annotation-tag-entity.
 import type { TestMetric } from '@/databases/entities/test-metric.ee';
 import { WorkflowEntity } from '@/databases/entities/workflow-entity';
 
-import { WithTimestampsAndStringId } from './abstract-entity';
+import { jsonColumnType, WithTimestampsAndStringId } from './abstract-entity';
+
+// Entity representing a node in a workflow under test, for which data should be mocked during test execution
+export type MockedNodeItem = {
+	name: string;
+};
 
 /**
  * Entity representing a Test Definition
@@ -27,6 +32,9 @@ export class TestDefinition extends WithTimestampsAndStringId {
 	@Column('text')
 	description: string;
 
+	@Column(jsonColumnType, { default: '[]' })
+	mockedNodes: MockedNodeItem[];
+
 	/**
 	 * Relation to the workflow under test
 	 */

packages/cli/src/databases/migrations/common/1733133775640-AddMockedNodesColumnToTestDefinition.ts

@@ -0,0 +1,22 @@
+import type { MigrationContext, ReversibleMigration } from '@/databases/types';
+
+// We have to use raw query migration instead of schemaBuilder helpers,
+// because the typeorm schema builder implements addColumns by a table recreate for sqlite
+// which causes weird issues with the migration
+export class AddMockedNodesColumnToTestDefinition1733133775640 implements ReversibleMigration {
+	async up({ escape, runQuery }: MigrationContext) {
+		const tableName = escape.tableName('test_definition');
+		const mockedNodesColumnName = escape.columnName('mockedNodes');
+
+		await runQuery(
+			`ALTER TABLE ${tableName} ADD COLUMN ${mockedNodesColumnName} JSON DEFAULT '[]' NOT NULL`,
+		);
+	}
+
+	async down({ escape, runQuery }: MigrationContext) {
+		const tableName = escape.tableName('test_definition');
+		const columnName = escape.columnName('mockedNodes');
+
+		await runQuery(`ALTER TABLE ${tableName} DROP COLUMN ${columnName}`);
+	}
+}

packages/cli/src/databases/migrations/mysqldb/index.ts

@@ -73,6 +73,7 @@ import { CreateTestDefinitionTable1730386903556 } from '../common/1730386903556-
 import { AddDescriptionToTestDefinition1731404028106 } from '../common/1731404028106-AddDescriptionToTestDefinition';
 import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-CreateTestMetricTable';
 import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable';
+import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 
 export const mysqlMigrations: Migration[] = [
 	InitialMigration1588157391238,
@@ -148,4 +149,5 @@ export const mysqlMigrations: Migration[] = [
 	MigrateTestDefinitionKeyToString1731582748663,
 	CreateTestMetricTable1732271325258,
 	CreateTestRun1732549866705,
+	AddMockedNodesColumnToTestDefinition1733133775640,
 ];

packages/cli/src/databases/migrations/postgresdb/index.ts

@@ -73,6 +73,7 @@ import { CreateTestDefinitionTable1730386903556 } from '../common/1730386903556-
 import { AddDescriptionToTestDefinition1731404028106 } from '../common/1731404028106-AddDescriptionToTestDefinition';
 import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-CreateTestMetricTable';
 import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable';
+import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 
 export const postgresMigrations: Migration[] = [
 	InitialMigration1587669153312,
@@ -148,4 +149,5 @@ export const postgresMigrations: Migration[] = [
 	MigrateTestDefinitionKeyToString1731582748663,
 	CreateTestMetricTable1732271325258,
 	CreateTestRun1732549866705,
+	AddMockedNodesColumnToTestDefinition1733133775640,
 ];

packages/cli/src/databases/migrations/sqlite/index.ts

@@ -70,6 +70,7 @@ import { UpdateProcessedDataValueColumnToText1729607673464 } from '../common/172
 import { CreateTestDefinitionTable1730386903556 } from '../common/1730386903556-CreateTestDefinitionTable';
 import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-CreateTestMetricTable';
 import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable';
+import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 
 const sqliteMigrations: Migration[] = [
 	InitialMigration1588102412422,
@@ -142,6 +143,7 @@ const sqliteMigrations: Migration[] = [
 	MigrateTestDefinitionKeyToString1731582748663,
 	CreateTestMetricTable1732271325258,
 	CreateTestRun1732549866705,
+	AddMockedNodesColumnToTestDefinition1733133775640,
 ];
 
 export { sqliteMigrations };

packages/cli/src/evaluation/test-definition.schema.ts

@@ -16,5 +16,6 @@ export const testDefinitionPatchRequestBodySchema = z
 		description: z.string().optional(),
 		evaluationWorkflowId: z.string().min(1).optional(),
 		annotationTagId: z.string().min(1).optional(),
+		mockedNodes: z.array(z.object({ name: z.string() })).optional(),
 	})
 	.strict();

packages/cli/src/evaluation/test-definition.service.ee.ts

@@ -1,6 +1,6 @@
 import { Service } from 'typedi';
 
-import type { TestDefinition } from '@/databases/entities/test-definition.ee';
+import type { MockedNodeItem, TestDefinition } from '@/databases/entities/test-definition.ee';
 import { AnnotationTagRepository } from '@/databases/repositories/annotation-tag.repository.ee';
 import { TestDefinitionRepository } from '@/databases/repositories/test-definition.repository.ee';
 import { BadRequestError } from '@/errors/response-errors/bad-request.error';
@@ -31,6 +31,7 @@ export class TestDefinitionService {
 		evaluationWorkflowId?: string;
 		annotationTagId?: string;
 		id?: string;
+		mockedNodes?: MockedNodeItem[];
 	}) {
 		const entity: TestDefinitionLike = {};
 
@@ -64,6 +65,10 @@ export class TestDefinitionService {
 			};
 		}
 
+		if (attrs.mockedNodes) {
+			entity.mockedNodes = attrs.mockedNodes;
+		}
+
 		return entity;
 	}
 
@@ -107,6 +112,24 @@ export class TestDefinitionService {
 			}
 		}
 
+		// If there are mocked nodes, validate them
+		if (attrs.mockedNodes && attrs.mockedNodes.length > 0) {
+			const existingTestDefinition = await this.testDefinitionRepository.findOneOrFail({
+				where: {
+					id,
+				},
+				relations: ['workflow'],
+			});
+
+			const existingNodeNames = new Set(existingTestDefinition.workflow.nodes.map((n) => n.name));
+
+			attrs.mockedNodes.forEach((node) => {
+				if (!existingNodeNames.has(node.name)) {
+					throw new BadRequestError(`Pinned node not found in the workflow: ${node.name}`);
+				}
+			});
+		}
+
 		// Update the test definition
 		const queryResult = await this.testDefinitionRepository.update(id, this.toEntityLike(attrs));
 

packages/cli/src/evaluation/test-definitions.types.ee.ts

@@ -1,3 +1,4 @@
+import type { MockedNodeItem } from '@/databases/entities/test-definition.ee';
 import type { AuthenticatedRequest, ListQuery } from '@/requests';
 
 // ----------------------------------
@@ -26,7 +27,12 @@ export declare namespace TestDefinitionsRequest {
 	type Patch = AuthenticatedRequest<
 		RouteParams.TestId,
 		{},
-		{ name?: string; evaluationWorkflowId?: string; annotationTagId?: string }
+		{
+			name?: string;
+			evaluationWorkflowId?: string;
+			annotationTagId?: string;
+			mockedNodes?: MockedNodeItem[];
+		}
 	>;
 
 	type Delete = AuthenticatedRequest<RouteParams.TestId>;

packages/cli/src/evaluation/test-runner/__tests__/get-start-node.ee.test.ts

@@ -1,7 +1,7 @@
 import { readFileSync } from 'fs';
 import path from 'path';
 
-import { getPastExecutionStartNode } from '../utils.ee';
+import { getPastExecutionTriggerNode } from '../utils.ee';
 
 const executionDataJson = JSON.parse(
 	readFileSync(path.join(__dirname, './mock-data/execution-data.json'), { encoding: 'utf-8' }),
@@ -21,19 +21,19 @@ const executionDataMultipleTriggersJson2 = JSON.parse(
 
 describe('getPastExecutionStartNode', () => {
 	test('should return the start node of the past execution', () => {
-		const startNode = getPastExecutionStartNode(executionDataJson);
+		const startNode = getPastExecutionTriggerNode(executionDataJson);
 
 		expect(startNode).toEqual('When clicking ‘Test workflow’');
 	});
 
 	test('should return the start node of the past execution with multiple triggers', () => {
-		const startNode = getPastExecutionStartNode(executionDataMultipleTriggersJson);
+		const startNode = getPastExecutionTriggerNode(executionDataMultipleTriggersJson);
 
 		expect(startNode).toEqual('When clicking ‘Test workflow’');
 	});
 
 	test('should return the start node of the past execution with multiple triggers - chat trigger', () => {
-		const startNode = getPastExecutionStartNode(executionDataMultipleTriggersJson2);
+		const startNode = getPastExecutionTriggerNode(executionDataMultipleTriggersJson2);
 
 		expect(startNode).toEqual('When chat message received');
 	});

packages/cli/src/evaluation/test-runner/test-runner.service.ee.ts

@@ -22,7 +22,7 @@ import { getRunData } from '@/workflow-execute-additional-data';
 import { WorkflowRunner } from '@/workflow-runner';
 
 import { EvaluationMetrics } from './evaluation-metrics.ee';
-import { createPinData, getPastExecutionStartNode } from './utils.ee';
+import { createPinData, getPastExecutionTriggerNode } from './utils.ee';
 
 /**
  * This service orchestrates the running of test cases.
@@ -58,7 +58,7 @@ export class TestRunnerService {
 		const pinData = createPinData(workflow, pastExecutionData);
 
 		// Determine the start node of the past execution
-		const pastExecutionStartNode = getPastExecutionStartNode(pastExecutionData);
+		const pastExecutionStartNode = getPastExecutionTriggerNode(pastExecutionData);
 
 		// Prepare the data to run the workflow
 		const data: IWorkflowExecutionDataProcess = {

packages/cli/src/evaluation/test-runner/utils.ee.ts

@@ -26,7 +26,7 @@ export function createPinData(workflow: WorkflowEntity, executionData: IRunExecu
  * Returns the start node of the past execution.
  * The start node is the node that has no source and has run data.
  */
-export function getPastExecutionStartNode(executionData: IRunExecutionData) {
+export function getPastExecutionTriggerNode(executionData: IRunExecutionData) {
 	return Object.keys(executionData.resultData.runData).find((nodeName) => {
 		const data = executionData.resultData.runData[nodeName];
 		return !data[0].source || data[0].source.length === 0 || data[0].source[0] === null;

packages/cli/test/integration/evaluation/test-definitions.api.test.ts

@@ -394,6 +394,57 @@ describe('PATCH /evaluation/test-definitions/:id', () => {
 		expect(resp.statusCode).toBe(400);
 		expect(resp.body.message).toBe('Annotation tag not found');
 	});
+
+	test('should update pinned nodes', async () => {
+		const newTest = Container.get(TestDefinitionRepository).create({
+			name: 'test',
+			workflow: { id: workflowUnderTest.id },
+		});
+		await Container.get(TestDefinitionRepository).save(newTest);
+
+		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
+			mockedNodes: [
+				{
+					name: 'Schedule Trigger',
+				},
+			],
+		});
+
+		expect(resp.statusCode).toBe(200);
+		expect(resp.body.data.mockedNodes).toEqual([{ name: 'Schedule Trigger' }]);
+	});
+
+	test('should return error if pinned nodes are invalid', async () => {
+		const newTest = Container.get(TestDefinitionRepository).create({
+			name: 'test',
+			workflow: { id: workflowUnderTest.id },
+		});
+		await Container.get(TestDefinitionRepository).save(newTest);
+
+		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
+			mockedNodes: ['Simple string'],
+		});
+
+		expect(resp.statusCode).toBe(400);
+	});
+
+	test('should return error if pinned nodes are not in the workflow', async () => {
+		const newTest = Container.get(TestDefinitionRepository).create({
+			name: 'test',
+			workflow: { id: workflowUnderTest.id },
+		});
+		await Container.get(TestDefinitionRepository).save(newTest);
+
+		const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({
+			mockedNodes: [
+				{
+					name: 'Invalid Node',
+				},
+			],
+		});
+
+		expect(resp.statusCode).toBe(400);
+	});
 });
 
 describe('DELETE /evaluation/test-definitions/:id', () => {