From 3d990eb55591211cb36fe2b75b2eafe3ef2b4530 Mon Sep 17 00:00:00 2001 From: oleg Date: Tue, 7 Jan 2025 12:52:44 +0100 Subject: [PATCH 01/30] feat(editor): Add workflow evaluation run views (no-changelog) (#12258) --- .../test-runner/test-runner.service.ee.ts | 12 +- .../src/components/N8nSelect/Select.vue | 5 + .../editor-ui/src/api/testDefinition.ee.ts | 174 ++++- .../src/components/MainHeader/MainHeader.vue | 55 +- .../editor-ui/src/components/TagsDropdown.vue | 31 +- .../EditDefinition/EvaluationHeader.vue | 15 +- .../EditDefinition/EvaluationStep.vue | 40 +- .../EditDefinition/MetricsInput.vue | 28 +- .../EditDefinition/NodesPinning.vue | 194 ++++++ .../EditDefinition/TagsInput.vue | 45 +- .../ListDefinition/TestItem.vue | 34 +- .../ListDefinition/TestsList.vue | 2 +- .../TestDefinition/ListRuns/MetricsChart.vue | 108 +++ .../TestDefinition/ListRuns/TestRunsTable.vue | 119 ++++ .../composables/useMetricsChart.ts | 145 ++++ .../composables/useTestDefinitionForm.ts | 198 +++--- .../TestDefinition/shared/TableCell.vue | 74 ++ .../shared/TestDefinitionTable.vue | 95 +++ .../TestDefinition/tests/MetricsInput.test.ts | 101 ++- .../tests/useMetricsChart.test.ts | 127 ++++ .../tests/useTestDefinitionForm.test.ts | 237 +++++-- .../src/components/TestDefinition/types.ts | 22 + .../src/components/WorkflowTagsDropdown.vue | 2 +- .../src/components/canvas/Canvas.vue | 10 +- .../canvas/elements/nodes/CanvasNode.vue | 30 +- packages/editor-ui/src/constants.ts | 4 + .../src/plugins/i18n/locales/en.json | 40 +- packages/editor-ui/src/router.ts | 28 + .../editor-ui/src/stores/executions.store.ts | 1 + .../stores/testDefinition.store.ee.test.ts | 635 +++++++++++++----- .../src/stores/testDefinition.store.ee.ts | 214 +++++- packages/editor-ui/src/stores/ui.store.ts | 12 +- packages/editor-ui/src/stores/ui.utils.ts | 10 +- packages/editor-ui/src/types/canvas.ts | 1 + .../TestDefinition/TestDefinitionEditView.vue | 288 ++++++-- .../TestDefinition/TestDefinitionListView.vue | 96 ++- .../TestDefinitionRunDetailView.vue | 292 ++++++++ .../TestDefinitionRunsListView.vue | 129 ++++ .../tests/TestDefinitionEditView.test.ts | 296 +++++--- .../tests/TestDefinitionListView.test.ts | 174 +++++ .../tests/TestDefinitionRunDetailView.test.ts | 267 ++++++++ 41 files changed, 3811 insertions(+), 579 deletions(-) create mode 100644 packages/editor-ui/src/components/TestDefinition/EditDefinition/NodesPinning.vue create mode 100644 packages/editor-ui/src/components/TestDefinition/ListRuns/MetricsChart.vue create mode 100644 packages/editor-ui/src/components/TestDefinition/ListRuns/TestRunsTable.vue create mode 100644 packages/editor-ui/src/components/TestDefinition/composables/useMetricsChart.ts create mode 100644 packages/editor-ui/src/components/TestDefinition/shared/TableCell.vue create mode 100644 packages/editor-ui/src/components/TestDefinition/shared/TestDefinitionTable.vue create mode 100644 packages/editor-ui/src/components/TestDefinition/tests/useMetricsChart.test.ts create mode 100644 packages/editor-ui/src/views/TestDefinition/TestDefinitionRunDetailView.vue create mode 100644 packages/editor-ui/src/views/TestDefinition/TestDefinitionRunsListView.vue create mode 100644 packages/editor-ui/src/views/TestDefinition/tests/TestDefinitionListView.test.ts create mode 100644 packages/editor-ui/src/views/TestDefinition/tests/TestDefinitionRunDetailView.test.ts diff --git a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts index 926bc29b70..c1e6e33942 100644 --- a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts +++ b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts @@ -134,6 +134,7 @@ export class TestRunnerService { evaluationWorkflow: WorkflowEntity, expectedData: IRunData, actualData: IRunData, + testRunId?: string, ) { // Prepare the evaluation wf input data. // Provide both the expected data and the actual data @@ -146,7 +147,13 @@ export class TestRunnerService { // Prepare the data to run the evaluation workflow const data = await getRunData(evaluationWorkflow, [evaluationInputData]); - + // FIXME: This is a hack to add the testRunId to the evaluation workflow execution data + // So that we can fetch all execution runs for a test run + if (testRunId && data.executionData) { + data.executionData.resultData.metadata = { + testRunId, + }; + } data.executionMode = 'evaluation'; // Trigger the evaluation workflow @@ -264,10 +271,9 @@ export class TestRunnerService { evaluationWorkflow, originalRunData, testCaseRunData, + testRun.id, ); assert(evalExecution); - - // Extract the output of the last node executed in the evaluation workflow metrics.addResults(this.extractEvaluationResult(evalExecution)); } diff --git a/packages/design-system/src/components/N8nSelect/Select.vue b/packages/design-system/src/components/N8nSelect/Select.vue index 7cb4fafcc9..80a065d429 100644 --- a/packages/design-system/src/components/N8nSelect/Select.vue +++ b/packages/design-system/src/components/N8nSelect/Select.vue @@ -31,6 +31,10 @@ const props = defineProps({ multiple: { type: Boolean, }, + multipleLimit: { + type: Number, + default: 0, + }, filterMethod: { type: Function, }, @@ -120,6 +124,7 @@ defineExpose({ ; } + interface CreateTestDefinitionParams { name: string; workflowId: string; @@ -21,31 +25,63 @@ export interface UpdateTestDefinitionParams { evaluationWorkflowId?: string | null; annotationTagId?: string | null; description?: string | null; + mockedNodes?: Array<{ name: string }>; } + export interface UpdateTestResponse { createdAt: string; updatedAt: string; id: string; name: string; workflowId: string; - description: string | null; - annotationTag: string | null; - evaluationWorkflowId: string | null; - annotationTagId: string | null; + description?: string | null; + annotationTag?: string | null; + evaluationWorkflowId?: string | null; + annotationTagId?: string | null; +} + +export interface TestRunRecord { + id: string; + testDefinitionId: string; + status: 'new' | 'running' | 'completed' | 'error'; + metrics?: Record; + createdAt: string; + updatedAt: string; + runAt: string; + completedAt: string; +} + +interface GetTestRunParams { + testDefinitionId: string; + runId: string; +} + +interface DeleteTestRunParams { + testDefinitionId: string; + runId: string; } const endpoint = '/evaluation/test-definitions'; +const getMetricsEndpoint = (testDefinitionId: string, metricId?: string) => + `${endpoint}/${testDefinitionId}/metrics${metricId ? `/${metricId}` : ''}`; -export async function getTestDefinitions(context: IRestApiContext) { +export async function getTestDefinitions( + context: IRestApiContext, + params?: { workflowId?: string }, +) { + let url = endpoint; + if (params?.workflowId) { + url += `?filter=${JSON.stringify({ workflowId: params.workflowId })}`; + } return await makeRestApiRequest<{ count: number; testDefinitions: TestDefinitionRecord[] }>( context, 'GET', - endpoint, + url, ); } export async function getTestDefinition(context: IRestApiContext, id: string) { - return await makeRestApiRequest<{ id: string }>(context, 'GET', `${endpoint}/${id}`); + return await makeRestApiRequest(context, 'GET', `${endpoint}/${id}`); } export async function createTestDefinition( @@ -71,3 +107,125 @@ export async function updateTestDefinition( export async function deleteTestDefinition(context: IRestApiContext, id: string) { return await makeRestApiRequest<{ success: boolean }>(context, 'DELETE', `${endpoint}/${id}`); } + +// Metrics +export interface TestMetricRecord { + id: string; + name: string; + testDefinitionId: string; + createdAt?: string; + updatedAt?: string; +} + +export interface CreateTestMetricParams { + testDefinitionId: string; + name: string; +} + +export interface UpdateTestMetricParams { + name: string; + id: string; + testDefinitionId: string; +} + +export interface DeleteTestMetricParams { + testDefinitionId: string; + id: string; +} + +export const getTestMetrics = async (context: IRestApiContext, testDefinitionId: string) => { + return await makeRestApiRequest( + context, + 'GET', + getMetricsEndpoint(testDefinitionId), + ); +}; + +export const getTestMetric = async ( + context: IRestApiContext, + testDefinitionId: string, + id: string, +) => { + return await makeRestApiRequest( + context, + 'GET', + getMetricsEndpoint(testDefinitionId, id), + ); +}; + +export const createTestMetric = async ( + context: IRestApiContext, + params: CreateTestMetricParams, +) => { + return await makeRestApiRequest( + context, + 'POST', + getMetricsEndpoint(params.testDefinitionId), + { name: params.name }, + ); +}; + +export const updateTestMetric = async ( + context: IRestApiContext, + params: UpdateTestMetricParams, +) => { + return await makeRestApiRequest( + context, + 'PATCH', + getMetricsEndpoint(params.testDefinitionId, params.id), + { name: params.name }, + ); +}; + +export const deleteTestMetric = async ( + context: IRestApiContext, + params: DeleteTestMetricParams, +) => { + return await makeRestApiRequest( + context, + 'DELETE', + getMetricsEndpoint(params.testDefinitionId, params.id), + ); +}; + +const getRunsEndpoint = (testDefinitionId: string, runId?: string) => + `${endpoint}/${testDefinitionId}/runs${runId ? `/${runId}` : ''}`; + +// Get all test runs for a test definition +export const getTestRuns = async (context: IRestApiContext, testDefinitionId: string) => { + return await makeRestApiRequest( + context, + 'GET', + getRunsEndpoint(testDefinitionId), + ); +}; + +// Get specific test run +export const getTestRun = async (context: IRestApiContext, params: GetTestRunParams) => { + return await makeRestApiRequest( + context, + 'GET', + getRunsEndpoint(params.testDefinitionId, params.runId), + ); +}; + +// Start a new test run +export const startTestRun = async (context: IRestApiContext, testDefinitionId: string) => { + const response = await request({ + method: 'POST', + baseURL: context.baseUrl, + endpoint: `${endpoint}/${testDefinitionId}/run`, + headers: { 'push-ref': context.pushRef }, + }); + // CLI is returning the response without wrapping it in `data` key + return response as { success: boolean }; +}; + +// Delete a test run +export const deleteTestRun = async (context: IRestApiContext, params: DeleteTestRunParams) => { + return await makeRestApiRequest<{ success: boolean }>( + context, + 'DELETE', + getRunsEndpoint(params.testDefinitionId, params.runId), + ); +}; diff --git a/packages/editor-ui/src/components/MainHeader/MainHeader.vue b/packages/editor-ui/src/components/MainHeader/MainHeader.vue index 1a595312c9..7db2b3a478 100644 --- a/packages/editor-ui/src/components/MainHeader/MainHeader.vue +++ b/packages/editor-ui/src/components/MainHeader/MainHeader.vue @@ -43,6 +43,25 @@ const executionToReturnTo = ref(''); const dirtyState = ref(false); const githubButtonHidden = useLocalStorage(LOCAL_STORAGE_HIDE_GITHUB_STAR_BUTTON, false); +// Track the routes that are used for the tabs +// This is used to determine which tab to show when the route changes +// TODO: It might be easier to manage this in the router config, by passing meta information to the routes +// This would allow us to specify it just once on the root route, and then have the tabs be determined for children +const testDefinitionRoutes: VIEWS[] = [ + VIEWS.TEST_DEFINITION, + VIEWS.TEST_DEFINITION_EDIT, + VIEWS.TEST_DEFINITION_RUNS, + VIEWS.TEST_DEFINITION_RUNS_DETAIL, + VIEWS.TEST_DEFINITION_RUNS_COMPARE, +]; + +const workflowRoutes: VIEWS[] = [VIEWS.WORKFLOW, VIEWS.NEW_WORKFLOW, VIEWS.EXECUTION_DEBUG]; + +const executionRoutes: VIEWS[] = [ + VIEWS.EXECUTION_HOME, + VIEWS.WORKFLOW_EXECUTIONS, + VIEWS.EXECUTION_PREVIEW, +]; const tabBarItems = computed(() => { const items = [ { value: MAIN_HEADER_TABS.WORKFLOW, label: locale.baseText('generic.editor') }, @@ -92,24 +111,30 @@ onMounted(async () => { syncTabsWithRoute(route); }); +function isViewRoute(name: unknown): name is VIEWS { + return ( + typeof name === 'string' && + [testDefinitionRoutes, workflowRoutes, executionRoutes].flat().includes(name as VIEWS) + ); +} + function syncTabsWithRoute(to: RouteLocation, from?: RouteLocation): void { - if (to.matched.some((record) => record.name === VIEWS.TEST_DEFINITION)) { - activeHeaderTab.value = MAIN_HEADER_TABS.TEST_DEFINITION; - } - if ( - to.name === VIEWS.EXECUTION_HOME || - to.name === VIEWS.WORKFLOW_EXECUTIONS || - to.name === VIEWS.EXECUTION_PREVIEW - ) { - activeHeaderTab.value = MAIN_HEADER_TABS.EXECUTIONS; - } else if ( - to.name === VIEWS.WORKFLOW || - to.name === VIEWS.NEW_WORKFLOW || - to.name === VIEWS.EXECUTION_DEBUG - ) { - activeHeaderTab.value = MAIN_HEADER_TABS.WORKFLOW; + // Map route types to their corresponding tab in the header + const routeTabMapping = [ + { routes: testDefinitionRoutes, tab: MAIN_HEADER_TABS.TEST_DEFINITION }, + { routes: executionRoutes, tab: MAIN_HEADER_TABS.EXECUTIONS }, + { routes: workflowRoutes, tab: MAIN_HEADER_TABS.WORKFLOW }, + ]; + + // Update the active tab based on the current route + if (to.name && isViewRoute(to.name)) { + const matchingTab = routeTabMapping.find(({ routes }) => routes.includes(to.name as VIEWS)); + if (matchingTab) { + activeHeaderTab.value = matchingTab.tab; + } } + // Store the current workflow ID, but only if it's not a new workflow if (to.params.name !== 'new' && typeof to.params.name === 'string') { workflowToReturnTo.value = to.params.name; } diff --git a/packages/editor-ui/src/components/TagsDropdown.vue b/packages/editor-ui/src/components/TagsDropdown.vue index 67799b1657..0a1b342b5c 100644 --- a/packages/editor-ui/src/components/TagsDropdown.vue +++ b/packages/editor-ui/src/components/TagsDropdown.vue @@ -16,7 +16,10 @@ interface TagsDropdownProps { allTags: ITag[]; isLoading: boolean; tagsById: Record; + createEnabled?: boolean; + manageEnabled?: boolean; createTag?: (name: string) => Promise; + multipleLimit?: number; } const i18n = useI18n(); @@ -27,6 +30,10 @@ const props = withDefaults(defineProps(), { placeholder: '', modelValue: () => [], eventBus: null, + createEnabled: true, + manageEnabled: true, + createTag: undefined, + multipleLimit: 0, }); const emit = defineEmits<{ @@ -59,6 +66,17 @@ const appliedTags = computed(() => { return props.modelValue.filter((id: string) => props.tagsById[id]); }); +const containerClasses = computed(() => { + return { 'tags-container': true, focused: focused.value }; +}); + +const dropdownClasses = computed(() => ({ + 'tags-dropdown': true, + [`tags-dropdown-${dropdownId}`]: true, + 'tags-dropdown-create-enabled': props.createEnabled, + 'tags-dropdown-manage-enabled': props.manageEnabled, +})); + watch( () => props.allTags, () => { @@ -189,7 +207,7 @@ onClickOutside(