diff --git a/packages/cli/src/databases/entities/test-definition.ee.ts b/packages/cli/src/databases/entities/test-definition.ee.ts index 77f8ca2bdc..a7cb393b5d 100644 --- a/packages/cli/src/databases/entities/test-definition.ee.ts +++ b/packages/cli/src/databases/entities/test-definition.ee.ts @@ -5,7 +5,12 @@ import { AnnotationTagEntity } from '@/databases/entities/annotation-tag-entity. import type { TestMetric } from '@/databases/entities/test-metric.ee'; import { WorkflowEntity } from '@/databases/entities/workflow-entity'; -import { WithTimestampsAndStringId } from './abstract-entity'; +import { jsonColumnType, WithTimestampsAndStringId } from './abstract-entity'; + +// Entity representing a node in a workflow under test, for which data should be mocked during test execution +export type MockedNodeItem = { + name: string; +}; /** * Entity representing a Test Definition @@ -27,6 +32,9 @@ export class TestDefinition extends WithTimestampsAndStringId { @Column('text') description: string; + @Column(jsonColumnType, { default: '[]' }) + mockedNodes: MockedNodeItem[]; + /** * Relation to the workflow under test */ diff --git a/packages/cli/src/databases/migrations/common/1733133775640-AddMockedNodesColumnToTestDefinition.ts b/packages/cli/src/databases/migrations/common/1733133775640-AddMockedNodesColumnToTestDefinition.ts new file mode 100644 index 0000000000..09ce45722c --- /dev/null +++ b/packages/cli/src/databases/migrations/common/1733133775640-AddMockedNodesColumnToTestDefinition.ts @@ -0,0 +1,22 @@ +import type { MigrationContext, ReversibleMigration } from '@/databases/types'; + +// We have to use raw query migration instead of schemaBuilder helpers, +// because the typeorm schema builder implements addColumns by a table recreate for sqlite +// which causes weird issues with the migration +export class AddMockedNodesColumnToTestDefinition1733133775640 implements ReversibleMigration { + async up({ escape, runQuery }: MigrationContext) { + const tableName = escape.tableName('test_definition'); + const mockedNodesColumnName = escape.columnName('mockedNodes'); + + await runQuery( + `ALTER TABLE ${tableName} ADD COLUMN ${mockedNodesColumnName} JSON DEFAULT '[]' NOT NULL`, + ); + } + + async down({ escape, runQuery }: MigrationContext) { + const tableName = escape.tableName('test_definition'); + const columnName = escape.columnName('mockedNodes'); + + await runQuery(`ALTER TABLE ${tableName} DROP COLUMN ${columnName}`); + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index d962042333..b977f6b013 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -73,6 +73,7 @@ import { CreateTestDefinitionTable1730386903556 } from '../common/1730386903556- import { AddDescriptionToTestDefinition1731404028106 } from '../common/1731404028106-AddDescriptionToTestDefinition'; import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-CreateTestMetricTable'; import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable'; +import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; export const mysqlMigrations: Migration[] = [ InitialMigration1588157391238, @@ -148,4 +149,5 @@ export const mysqlMigrations: Migration[] = [ MigrateTestDefinitionKeyToString1731582748663, CreateTestMetricTable1732271325258, CreateTestRun1732549866705, + AddMockedNodesColumnToTestDefinition1733133775640, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index 012b18e31d..985e6964e1 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -73,6 +73,7 @@ import { CreateTestDefinitionTable1730386903556 } from '../common/1730386903556- import { AddDescriptionToTestDefinition1731404028106 } from '../common/1731404028106-AddDescriptionToTestDefinition'; import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-CreateTestMetricTable'; import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable'; +import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; export const postgresMigrations: Migration[] = [ InitialMigration1587669153312, @@ -148,4 +149,5 @@ export const postgresMigrations: Migration[] = [ MigrateTestDefinitionKeyToString1731582748663, CreateTestMetricTable1732271325258, CreateTestRun1732549866705, + AddMockedNodesColumnToTestDefinition1733133775640, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index 7c8fcbf86f..34d548b684 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -70,6 +70,7 @@ import { UpdateProcessedDataValueColumnToText1729607673464 } from '../common/172 import { CreateTestDefinitionTable1730386903556 } from '../common/1730386903556-CreateTestDefinitionTable'; import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-CreateTestMetricTable'; import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable'; +import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; const sqliteMigrations: Migration[] = [ InitialMigration1588102412422, @@ -142,6 +143,7 @@ const sqliteMigrations: Migration[] = [ MigrateTestDefinitionKeyToString1731582748663, CreateTestMetricTable1732271325258, CreateTestRun1732549866705, + AddMockedNodesColumnToTestDefinition1733133775640, ]; export { sqliteMigrations }; diff --git a/packages/cli/src/evaluation/test-definition.schema.ts b/packages/cli/src/evaluation/test-definition.schema.ts index 8f71ee6858..7760ae9dac 100644 --- a/packages/cli/src/evaluation/test-definition.schema.ts +++ b/packages/cli/src/evaluation/test-definition.schema.ts @@ -16,5 +16,6 @@ export const testDefinitionPatchRequestBodySchema = z description: z.string().optional(), evaluationWorkflowId: z.string().min(1).optional(), annotationTagId: z.string().min(1).optional(), + mockedNodes: z.array(z.object({ name: z.string() })).optional(), }) .strict(); diff --git a/packages/cli/src/evaluation/test-definition.service.ee.ts b/packages/cli/src/evaluation/test-definition.service.ee.ts index 55b7339ebe..e9a31e7eee 100644 --- a/packages/cli/src/evaluation/test-definition.service.ee.ts +++ b/packages/cli/src/evaluation/test-definition.service.ee.ts @@ -1,6 +1,6 @@ import { Service } from 'typedi'; -import type { TestDefinition } from '@/databases/entities/test-definition.ee'; +import type { MockedNodeItem, TestDefinition } from '@/databases/entities/test-definition.ee'; import { AnnotationTagRepository } from '@/databases/repositories/annotation-tag.repository.ee'; import { TestDefinitionRepository } from '@/databases/repositories/test-definition.repository.ee'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; @@ -31,6 +31,7 @@ export class TestDefinitionService { evaluationWorkflowId?: string; annotationTagId?: string; id?: string; + mockedNodes?: MockedNodeItem[]; }) { const entity: TestDefinitionLike = {}; @@ -64,6 +65,10 @@ export class TestDefinitionService { }; } + if (attrs.mockedNodes) { + entity.mockedNodes = attrs.mockedNodes; + } + return entity; } @@ -107,6 +112,24 @@ export class TestDefinitionService { } } + // If there are mocked nodes, validate them + if (attrs.mockedNodes && attrs.mockedNodes.length > 0) { + const existingTestDefinition = await this.testDefinitionRepository.findOneOrFail({ + where: { + id, + }, + relations: ['workflow'], + }); + + const existingNodeNames = new Set(existingTestDefinition.workflow.nodes.map((n) => n.name)); + + attrs.mockedNodes.forEach((node) => { + if (!existingNodeNames.has(node.name)) { + throw new BadRequestError(`Pinned node not found in the workflow: ${node.name}`); + } + }); + } + // Update the test definition const queryResult = await this.testDefinitionRepository.update(id, this.toEntityLike(attrs)); diff --git a/packages/cli/src/evaluation/test-definitions.types.ee.ts b/packages/cli/src/evaluation/test-definitions.types.ee.ts index 1beb415276..b7441a2763 100644 --- a/packages/cli/src/evaluation/test-definitions.types.ee.ts +++ b/packages/cli/src/evaluation/test-definitions.types.ee.ts @@ -1,3 +1,4 @@ +import type { MockedNodeItem } from '@/databases/entities/test-definition.ee'; import type { AuthenticatedRequest, ListQuery } from '@/requests'; // ---------------------------------- @@ -26,7 +27,12 @@ export declare namespace TestDefinitionsRequest { type Patch = AuthenticatedRequest< RouteParams.TestId, {}, - { name?: string; evaluationWorkflowId?: string; annotationTagId?: string } + { + name?: string; + evaluationWorkflowId?: string; + annotationTagId?: string; + mockedNodes?: MockedNodeItem[]; + } >; type Delete = AuthenticatedRequest; diff --git a/packages/cli/src/evaluation/test-runner/__tests__/get-start-node.ee.test.ts b/packages/cli/src/evaluation/test-runner/__tests__/get-start-node.ee.test.ts index 20f75fcf57..107cea80c6 100644 --- a/packages/cli/src/evaluation/test-runner/__tests__/get-start-node.ee.test.ts +++ b/packages/cli/src/evaluation/test-runner/__tests__/get-start-node.ee.test.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs'; import path from 'path'; -import { getPastExecutionStartNode } from '../utils.ee'; +import { getPastExecutionTriggerNode } from '../utils.ee'; const executionDataJson = JSON.parse( readFileSync(path.join(__dirname, './mock-data/execution-data.json'), { encoding: 'utf-8' }), @@ -21,19 +21,19 @@ const executionDataMultipleTriggersJson2 = JSON.parse( describe('getPastExecutionStartNode', () => { test('should return the start node of the past execution', () => { - const startNode = getPastExecutionStartNode(executionDataJson); + const startNode = getPastExecutionTriggerNode(executionDataJson); expect(startNode).toEqual('When clicking ‘Test workflow’'); }); test('should return the start node of the past execution with multiple triggers', () => { - const startNode = getPastExecutionStartNode(executionDataMultipleTriggersJson); + const startNode = getPastExecutionTriggerNode(executionDataMultipleTriggersJson); expect(startNode).toEqual('When clicking ‘Test workflow’'); }); test('should return the start node of the past execution with multiple triggers - chat trigger', () => { - const startNode = getPastExecutionStartNode(executionDataMultipleTriggersJson2); + const startNode = getPastExecutionTriggerNode(executionDataMultipleTriggersJson2); expect(startNode).toEqual('When chat message received'); }); diff --git a/packages/cli/src/evaluation/test-runner/test-runner.service.ee.ts b/packages/cli/src/evaluation/test-runner/test-runner.service.ee.ts index 5aaaf25558..92f5e4394e 100644 --- a/packages/cli/src/evaluation/test-runner/test-runner.service.ee.ts +++ b/packages/cli/src/evaluation/test-runner/test-runner.service.ee.ts @@ -22,7 +22,7 @@ import { getRunData } from '@/workflow-execute-additional-data'; import { WorkflowRunner } from '@/workflow-runner'; import { EvaluationMetrics } from './evaluation-metrics.ee'; -import { createPinData, getPastExecutionStartNode } from './utils.ee'; +import { createPinData, getPastExecutionTriggerNode } from './utils.ee'; /** * This service orchestrates the running of test cases. @@ -58,7 +58,7 @@ export class TestRunnerService { const pinData = createPinData(workflow, pastExecutionData); // Determine the start node of the past execution - const pastExecutionStartNode = getPastExecutionStartNode(pastExecutionData); + const pastExecutionStartNode = getPastExecutionTriggerNode(pastExecutionData); // Prepare the data to run the workflow const data: IWorkflowExecutionDataProcess = { diff --git a/packages/cli/src/evaluation/test-runner/utils.ee.ts b/packages/cli/src/evaluation/test-runner/utils.ee.ts index a6a4dc5ec2..c2ef68a5dc 100644 --- a/packages/cli/src/evaluation/test-runner/utils.ee.ts +++ b/packages/cli/src/evaluation/test-runner/utils.ee.ts @@ -26,7 +26,7 @@ export function createPinData(workflow: WorkflowEntity, executionData: IRunExecu * Returns the start node of the past execution. * The start node is the node that has no source and has run data. */ -export function getPastExecutionStartNode(executionData: IRunExecutionData) { +export function getPastExecutionTriggerNode(executionData: IRunExecutionData) { return Object.keys(executionData.resultData.runData).find((nodeName) => { const data = executionData.resultData.runData[nodeName]; return !data[0].source || data[0].source.length === 0 || data[0].source[0] === null; diff --git a/packages/cli/test/integration/evaluation/test-definitions.api.test.ts b/packages/cli/test/integration/evaluation/test-definitions.api.test.ts index b644cbe4ea..fe977fbfd3 100644 --- a/packages/cli/test/integration/evaluation/test-definitions.api.test.ts +++ b/packages/cli/test/integration/evaluation/test-definitions.api.test.ts @@ -394,6 +394,57 @@ describe('PATCH /evaluation/test-definitions/:id', () => { expect(resp.statusCode).toBe(400); expect(resp.body.message).toBe('Annotation tag not found'); }); + + test('should update pinned nodes', async () => { + const newTest = Container.get(TestDefinitionRepository).create({ + name: 'test', + workflow: { id: workflowUnderTest.id }, + }); + await Container.get(TestDefinitionRepository).save(newTest); + + const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({ + mockedNodes: [ + { + name: 'Schedule Trigger', + }, + ], + }); + + expect(resp.statusCode).toBe(200); + expect(resp.body.data.mockedNodes).toEqual([{ name: 'Schedule Trigger' }]); + }); + + test('should return error if pinned nodes are invalid', async () => { + const newTest = Container.get(TestDefinitionRepository).create({ + name: 'test', + workflow: { id: workflowUnderTest.id }, + }); + await Container.get(TestDefinitionRepository).save(newTest); + + const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({ + mockedNodes: ['Simple string'], + }); + + expect(resp.statusCode).toBe(400); + }); + + test('should return error if pinned nodes are not in the workflow', async () => { + const newTest = Container.get(TestDefinitionRepository).create({ + name: 'test', + workflow: { id: workflowUnderTest.id }, + }); + await Container.get(TestDefinitionRepository).save(newTest); + + const resp = await authOwnerAgent.patch(`/evaluation/test-definitions/${newTest.id}`).send({ + mockedNodes: [ + { + name: 'Invalid Node', + }, + ], + }); + + expect(resp.statusCode).toBe(400); + }); }); describe('DELETE /evaluation/test-definitions/:id', () => {