feat(editor): Add evaluation workflow and enhance workflow selector with pinned data support (#12773)

This commit is contained in:
oleg 2025-01-29 11:03:03 +01:00 committed by GitHub
parent 05b5f95331
commit be967ebec0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 290 additions and 125 deletions

View file

@ -556,6 +556,7 @@ describe('Projects', { disableAutoLogin: true }, () => {
selectResourceLocatorItem('workflowId', 0, 'Create a');
cy.get('body').type('{esc}');
workflowPage.actions.addNodeToCanvas(NOTION_NODE_NAME, true, true);
clickCreateNewCredential();
setCredentialValues({

View file

@ -86,6 +86,8 @@ describe('Workflow Selector Parameter', () => {
cy.stub(win, 'open').as('windowOpen');
});
cy.intercept('POST', '/rest/workflows*').as('createSubworkflow');
ndv.getters.resourceLocator('workflowId').should('be.visible');
ndv.getters.resourceLocatorInput('workflowId').click();
@ -98,10 +100,20 @@ describe('Workflow Selector Parameter', () => {
getVisiblePopper().findChildByTestId('rlc-item').eq(0).click();
const SAMPLE_SUBWORKFLOW_TEMPLATE_ID = 'VMiAxXa3lCAizGB5f7dVZQSFfg3FtHkdTKvLuupqBls=';
cy.get('@windowOpen').should(
'be.calledWith',
`/workflows/onboarding/${SAMPLE_SUBWORKFLOW_TEMPLATE_ID}?sampleSubWorkflows=0`,
);
cy.wait('@createSubworkflow').then((interception) => {
expect(interception.request.body).to.have.property('name').that.includes('Sub-Workflow');
expect(interception.request.body.nodes).to.be.an('array');
expect(interception.request.body.nodes).to.have.length(2);
expect(interception.request.body.nodes[0]).to.have.property(
'name',
'When Executed by Another Workflow',
);
expect(interception.request.body.nodes[1]).to.have.property(
'name',
'Replace me with your logic',
);
});
cy.get('@windowOpen').should('be.calledWithMatch', /\/workflow\/.+/);
});
});

View file

@ -65,7 +65,8 @@ describe('Sub-workflow creation and typed usage', () => {
// **************************
// NAVIGATE TO CHILD WORKFLOW
// **************************
// Close NDV before opening the node creator
cy.get('body').type('{esc}');
openNode('When Executed by Another Workflow');
});
@ -138,41 +139,41 @@ describe('Sub-workflow creation and typed usage', () => {
cy.window().then((win) => {
cy.stub(win, 'open').callsFake((url) => {
cy.visit(url);
selectResourceLocatorItem('workflowId', 0, 'Create a');
openNode('When Executed by Another Workflow');
getParameterInputByName('inputSource').click();
getVisiblePopper()
.getByTestId('parameter-input')
.eq(0)
.type('Using JSON Example{downArrow}{enter}');
const exampleJson =
'{{}' + EXAMPLE_FIELDS.map((x) => `"${x[0]}": ${makeExample(x[1])}`).join(',') + '}';
getParameterInputByName('jsonExample')
.find('.cm-line')
.eq(0)
.type(`{selectAll}{backspace}${exampleJson}{enter}`);
// first one doesn't work for some reason, might need to wait for something?
clickExecuteNode();
validateAndReturnToParent(
DEFAULT_SUBWORKFLOW_NAME_2,
2,
EXAMPLE_FIELDS.map((f) => f[0]),
);
assertOutputTableContent([
['[null]', '[null]', '[null]', '[null]', '[null]', 'false'],
['[null]', '[null]', '[null]', '[null]', '[null]', 'false'],
]);
clickExecuteNode();
});
});
selectResourceLocatorItem('workflowId', 0, 'Create a');
openNode('When Executed by Another Workflow');
getParameterInputByName('inputSource').click();
getVisiblePopper()
.getByTestId('parameter-input')
.eq(0)
.type('Using JSON Example{downArrow}{enter}');
const exampleJson =
'{{}' + EXAMPLE_FIELDS.map((x) => `"${x[0]}": ${makeExample(x[1])}`).join(',') + '}';
getParameterInputByName('jsonExample')
.find('.cm-line')
.eq(0)
.type(`{selectAll}{backspace}${exampleJson}{enter}`);
// first one doesn't work for some reason, might need to wait for something?
clickExecuteNode();
validateAndReturnToParent(
DEFAULT_SUBWORKFLOW_NAME_2,
2,
EXAMPLE_FIELDS.map((f) => f[0]),
);
assertOutputTableContent([
['[null]', '[null]', '[null]', '[null]', '[null]', 'false'],
['[null]', '[null]', '[null]', '[null]', '[null]', 'false'],
]);
clickExecuteNode();
});
it('should show node issue when no fields are defined in manual mode', () => {

View file

@ -1,20 +1,45 @@
<script setup lang="ts">
import { useI18n } from '@/composables/useI18n';
import type { INodeParameterResourceLocator } from 'n8n-workflow';
import { SAMPLE_EVALUATION_WORKFLOW } from '@/constants.workflows';
import type { IWorkflowDataCreate } from '@/Interface';
import type { INodeParameterResourceLocator, IPinData } from 'n8n-workflow';
import { computed } from 'vue';
interface WorkflowSelectorProps {
modelValue: INodeParameterResourceLocator;
examplePinnedData?: IPinData;
sampleWorkflowName?: string;
}
withDefaults(defineProps<WorkflowSelectorProps>(), {
const props = withDefaults(defineProps<WorkflowSelectorProps>(), {
modelValue: () => ({
mode: 'id',
value: '',
__rl: true,
}),
examplePinnedData: () => ({}),
sampleWorkflowName: undefined,
});
defineEmits<{ 'update:modelValue': [value: WorkflowSelectorProps['modelValue']] }>();
const locale = useI18n();
const subworkflowName = computed(() => {
if (props.sampleWorkflowName) {
return locale.baseText('testDefinition.workflowInput.subworkflowName', {
interpolate: { name: props.sampleWorkflowName },
});
}
return locale.baseText('testDefinition.workflowInput.subworkflowName.default');
});
const sampleWorkflow = computed<IWorkflowDataCreate>(() => {
return {
...SAMPLE_EVALUATION_WORKFLOW,
name: subworkflowName.value,
pinData: props.examplePinnedData,
};
});
</script>
<template>
<div>
@ -36,6 +61,7 @@ const locale = useI18n();
:expression-edit-dialog-visible="false"
:path="'workflows'"
allow-new
:sample-workflow="sampleWorkflow"
@update:model-value="$emit('update:modelValue', $event)"
/>
</n8n-input-label>

View file

@ -9,6 +9,7 @@ import type { EditableFormState, EvaluationFormState } from '@/components/TestDe
import type { ITag, ModalState } from '@/Interface';
import { NODE_PINNING_MODAL_KEY } from '@/constants';
import { ref } from 'vue';
import type { IPinData } from 'n8n-workflow';
defineProps<{
showConfig: boolean;
@ -16,6 +17,8 @@ defineProps<{
allTags: ITag[];
tagsById: Record<string, ITag>;
isLoading: boolean;
examplePinnedData?: IPinData;
sampleWorkflowName?: string;
getFieldIssues: (key: string) => Array<{ field: string; message: string }>;
startEditing: (field: keyof EditableFormState) => void;
saveChanges: (field: keyof EditableFormState) => void;
@ -23,6 +26,12 @@ defineProps<{
createTag?: (name: string) => Promise<ITag>;
}>();
const emit = defineEmits<{
openPinningModal: [];
deleteMetric: [metric: Partial<TestMetricRecord>];
}>();
const locale = useI18n();
const changedFieldsKeys = ref<string[]>([]);
const tags = defineModel<EvaluationFormState['tags']>('tags', { required: true });
const evaluationWorkflow = defineModel<EvaluationFormState['evaluationWorkflow']>(
@ -35,12 +44,6 @@ const mockedNodes = defineModel<EvaluationFormState['mockedNodes']>('mockedNodes
});
const nodePinningModal = ref<ModalState | null>(null);
const emit = defineEmits<{
openPinningModal: [];
deleteMetric: [metric: Partial<TestMetricRecord>];
}>();
const locale = useI18n();
function updateChangedFieldsKeys(key: string) {
changedFieldsKeys.value.push(key);
@ -138,7 +141,9 @@ function showFieldIssues(fieldKey: string) {
<template #cardContent>
<WorkflowSelector
v-model="evaluationWorkflow"
:example-pinned-data="examplePinnedData"
:class="{ 'has-issues': getFieldIssues('evaluationWorkflow').length > 0 }"
:sample-workflow-name="sampleWorkflowName"
@update:model-value="updateChangedFieldsKeys('evaluationWorkflow')"
/>
</template>

View file

@ -20,8 +20,9 @@ import { useWorkflowResourceLocatorModes } from './useWorkflowResourceLocatorMod
import { useWorkflowResourcesLocator } from './useWorkflowResourcesLocator';
import { useProjectsStore } from '@/stores/projects.store';
import { useTelemetry } from '@/composables/useTelemetry';
import { NEW_SAMPLE_WORKFLOW_CREATED_CHANNEL } from '@/constants';
import { VIEWS } from '@/constants';
import { SAMPLE_SUBWORKFLOW_WORKFLOW } from '@/constants.workflows';
import type { IWorkflowDataCreate } from '@/Interface';
interface Props {
modelValue: INodeParameterResourceLocator;
@ -34,6 +35,7 @@ interface Props {
forceShowExpression?: boolean;
parameterIssues?: string[];
parameter: INodeProperties;
sampleWorkflow?: IWorkflowDataCreate;
}
const props = withDefaults(defineProps<Props>(), {
@ -44,6 +46,7 @@ const props = withDefaults(defineProps<Props>(), {
forceShowExpression: false,
expressionDisplayValue: '',
parameterIssues: () => [],
sampleWorkflow: () => SAMPLE_SUBWORKFLOW_WORKFLOW,
});
const emit = defineEmits<{
@ -205,36 +208,30 @@ onClickOutside(dropdown, () => {
isDropdownVisible.value = false;
});
const onAddResourceClicked = () => {
const subWorkflowNameRegex = /My\s+Sub-Workflow\s+\d+/;
const urlSearchParams = new URLSearchParams();
if (projectStore.currentProjectId) {
urlSearchParams.set('projectId', projectStore.currentProjectId);
}
const onAddResourceClicked = async () => {
const projectId = projectStore.currentProjectId;
const sampleWorkflow = props.sampleWorkflow;
const workflowName = sampleWorkflow.name ?? 'My Sub-Workflow';
const sampleSubWorkflows = workflowsStore.allWorkflows.filter(
(w) => w.name && subWorkflowNameRegex.test(w.name),
(w) => w.name && new RegExp(workflowName).test(w.name),
);
urlSearchParams.set('sampleSubWorkflows', sampleSubWorkflows.length.toString());
const workflow: IWorkflowDataCreate = {
...sampleWorkflow,
name: `${workflowName} ${sampleSubWorkflows.length + 1}`,
};
if (projectId) {
workflow.projectId = projectId;
}
telemetry.track('User clicked create new sub-workflow button', {}, { withPostHog: true });
const sampleSubworkflowChannel = new BroadcastChannel(NEW_SAMPLE_WORKFLOW_CREATED_CHANNEL);
const newWorkflow = await workflowsStore.createNewWorkflow(workflow);
const { href } = router.resolve({ name: VIEWS.WORKFLOW, params: { name: newWorkflow.id } });
await reloadWorkflows();
onInputChange(newWorkflow.id);
hideDropdown();
sampleSubworkflowChannel.onmessage = async (event: MessageEvent<{ workflowId: string }>) => {
const workflowId = event.data.workflowId;
await reloadWorkflows();
onInputChange(workflowId);
hideDropdown();
};
window.open(
`/workflows/onboarding/${SAMPLE_SUBWORKFLOW_WORKFLOW.meta.templateId}?${urlSearchParams.toString()}`,
'_blank',
);
window.open(href, '_blank');
};
</script>
<template>

View file

@ -904,8 +904,6 @@ export const BROWSER_ID_STORAGE_KEY = 'n8n-browserId';
export const APP_MODALS_ELEMENT_ID = 'app-modals';
export const NEW_SAMPLE_WORKFLOW_CREATED_CHANNEL = 'new-sample-sub-workflow-created';
export const AI_NODES_PACKAGE_NAME = '@n8n/n8n-nodes-langchain';
export const AI_ASSISTANT_MAX_CONTENT_LENGTH = 100; // in kilobytes

View file

@ -1,11 +1,8 @@
import { NodeConnectionType } from 'n8n-workflow';
import type { INodeUi, WorkflowDataWithTemplateId } from './Interface';
import type { INodeUi, IWorkflowDataCreate } from './Interface';
export const SAMPLE_SUBWORKFLOW_WORKFLOW: WorkflowDataWithTemplateId = {
export const SAMPLE_SUBWORKFLOW_WORKFLOW: IWorkflowDataCreate = {
name: 'My Sub-Workflow',
meta: {
templateId: 'VMiAxXa3lCAizGB5f7dVZQSFfg3FtHkdTKvLuupqBls=',
},
nodes: [
{
id: 'c055762a-8fe7-4141-a639-df2372f30060',
@ -41,3 +38,137 @@ export const SAMPLE_SUBWORKFLOW_WORKFLOW: WorkflowDataWithTemplateId = {
},
pinData: {},
};
export const SAMPLE_EVALUATION_WORKFLOW: IWorkflowDataCreate = {
name: 'My Evaluation Sub-Workflow',
nodes: [
{
parameters: {
inputSource: 'passthrough',
},
id: 'ad3156ed-3007-4a09-8527-920505339812',
name: 'When called by a test run',
type: 'n8n-nodes-base.executeWorkflowTrigger',
typeVersion: 1.1,
position: [620, 380],
},
{
parameters: {},
id: '5ff0deaf-6ec9-4a0f-a906-70f1d8375e7c',
name: 'Replace me',
type: 'n8n-nodes-base.noOp',
typeVersion: 1,
position: [860, 380],
},
{
parameters: {
assignments: {
assignments: [
{
id: 'a748051d-ebdb-4fcf-aaed-02756130ce2a',
name: 'my_metric',
value: 1,
type: 'number',
},
],
},
options: {},
},
id: '2cae7e85-7808-4cab-85c0-d233f47701a1',
name: 'Return metric(s)',
type: 'n8n-nodes-base.set',
typeVersion: 3.4,
position: [1100, 380],
},
{
parameters: {
content:
"### 1. Receive execution data\n\nThis workflow will be passed:\n- A past execution from the test\n- The execution produced by re-running it\n\n\nWe've pinned some example data to get you started",
height: 438,
width: 217,
color: 7,
},
id: 'ecb90156-30a3-4a90-93d5-6aca702e2f6b',
name: 'Sticky Note',
type: 'n8n-nodes-base.stickyNote',
typeVersion: 1,
position: [560, 105],
},
{
parameters: {
content: '### 2. Compare actual and expected result\n',
height: 439,
width: 217,
color: 7,
},
id: '556464f8-b86d-41e2-9249-ca6d541c9147',
name: 'Sticky Note1',
type: 'n8n-nodes-base.stickyNote',
typeVersion: 1,
position: [800, 104],
},
{
parameters: {
content: '### 3. Return metrics\n\nMetrics should always be numerical',
height: 439,
width: 217,
color: 7,
},
id: '04c96a00-b360-423a-90a6-b3943c7d832f',
name: 'Sticky Note2',
type: 'n8n-nodes-base.stickyNote',
typeVersion: 1,
position: [1040, 104],
},
{
parameters: {
content:
'## Evaluation workflow\nThis workflow is used to check whether a single past execution being tested gives similar results when re-run',
height: 105,
width: 694,
},
id: '2250a6ec-7c4f-45e4-8dfe-c4b50c98b34b',
name: 'Sticky Note3',
type: 'n8n-nodes-base.stickyNote',
typeVersion: 1,
position: [560, -25],
},
],
pinData: {
'When called by a test run': [
{
json: {
newExecution: {},
originalExecution: {},
},
},
],
},
connections: {
'When called by a test run': {
[NodeConnectionType.Main]: [
[
{
node: 'Replace me',
type: NodeConnectionType.Main,
index: 0,
},
],
],
},
'Replace me': {
[NodeConnectionType.Main]: [
[
{
node: 'Return metric(s)',
type: NodeConnectionType.Main,
index: 0,
},
],
],
},
},
settings: {
executionOrder: 'v1',
},
};

View file

@ -2359,6 +2359,7 @@
"workflows.create.personal.toast.text": "This workflow has been created inside your personal space.",
"workflows.create.project.toast.title": "Workflow successfully created in {projectName}",
"workflows.create.project.toast.text": "All members from {projectName} will have access to this workflow.",
"workflowSelectorParameterInput.createNewSubworkflow.name": "My Sub-Workflow",
"importCurlModal.title": "Import cURL command",
"importCurlModal.input.label": "cURL Command",
"importCurlModal.input.placeholder": "Paste the cURL command here",
@ -2869,6 +2870,8 @@
"testDefinition.configError.noExecutionsAddedToTag": "No executions added to this tag",
"testDefinition.configError.noEvaluationWorkflow": "No evaluation workflow set",
"testDefinition.configError.noMetrics": "No metrics set",
"testDefinition.workflowInput.subworkflowName": "Evaluation workflow for {name}",
"testDefinition.workflowInput.subworkflowName.default": "My Evaluation Sub-Workflow",
"freeAi.credits.callout.claim.title": "Get {credits} free OpenAI API credits",
"freeAi.credits.callout.claim.button.label": "Claim credits",
"freeAi.credits.callout.success.title.part1": "Claimed {credits} free OpenAI API credits! Please note these free credits are only for the following models:",

View file

@ -14,6 +14,10 @@ import type { TestMetricRecord, TestRunRecord } from '@/api/testDefinition.ee';
import { useUIStore } from '@/stores/ui.store';
import { useTestDefinitionStore } from '@/stores/testDefinition.store.ee';
import ConfigSection from '@/components/TestDefinition/EditDefinition/sections/ConfigSection.vue';
import { useExecutionsStore } from '@/stores/executions.store';
import { useWorkflowsStore } from '@/stores/workflows.store';
import type { IPinData } from 'n8n-workflow';
const props = defineProps<{
testId?: string;
}>();
@ -26,6 +30,8 @@ const toast = useToast();
const testDefinitionStore = useTestDefinitionStore();
const tagsStore = useAnnotationTagsStore();
const uiStore = useUIStore();
const executionsStore = useExecutionsStore();
const workflowStore = useWorkflowsStore();
const {
state,
@ -50,11 +56,13 @@ const appliedTheme = computed(() => uiStore.appliedTheme);
const tagUsageCount = computed(
() => tagsStore.tagsById[state.value.tags.value[0]]?.usageCount ?? 0,
);
const workflowName = computed(() => workflowStore.workflow.name);
const hasRuns = computed(() => runs.value.length > 0);
const fieldsIssues = computed(() => testDefinitionStore.getFieldIssues(testId.value) ?? []);
const showConfig = ref(true);
const selectedMetric = ref<string>('');
const fieldsIssues = computed(() => testDefinitionStore.getFieldIssues(testId.value) ?? []);
const examplePinnedData = ref<IPinData>({});
onMounted(async () => {
if (!testDefinitionStore.isFeatureEnabled) {
@ -145,13 +153,36 @@ function toggleConfig() {
showConfig.value = !showConfig.value;
}
async function getExamplePinnedDataForTags() {
const evaluationWorkflowExecutions = await executionsStore.fetchExecutions({
workflowId: currentWorkflowId.value,
annotationTags: state.value.tags.value,
});
if (evaluationWorkflowExecutions.count > 0) {
const firstExecution = evaluationWorkflowExecutions.results[0];
const executionData = await executionsStore.fetchExecution(firstExecution.id);
const resultData = executionData?.data?.resultData.runData;
examplePinnedData.value = {
'When called by a test run': [
{
json: {
originalExecution: resultData,
newExecution: resultData,
},
},
],
};
}
}
// Debounced watchers for auto-saving
watch(
() => state.value.metrics,
debounce(async () => await updateMetrics(testId.value), { debounceTime: 400 }),
{ deep: true },
);
watch(() => state.value.tags, getExamplePinnedDataForTags);
watch(
() => [
state.value.description,
@ -219,6 +250,8 @@ watch(
:start-editing="startEditing"
:save-changes="saveChanges"
:create-tag="handleCreateTag"
:example-pinned-data="examplePinnedData"
:sample-workflow-name="workflowName"
@open-pinning-modal="openPinningModal"
@delete-metric="onDeleteMetric"
/>

View file

@ -1,13 +1,11 @@
<script setup lang="ts">
import { useLoadingService } from '@/composables/useLoadingService';
import { useI18n } from '@/composables/useI18n';
import { NEW_SAMPLE_WORKFLOW_CREATED_CHANNEL, VIEWS } from '@/constants';
import { VIEWS } from '@/constants';
import { useTemplatesStore } from '@/stores/templates.store';
import { useWorkflowsStore } from '@/stores/workflows.store';
import { onMounted } from 'vue';
import { useRoute, useRouter } from 'vue-router';
import type { IWorkflowDataCreate } from '@/Interface';
import { SAMPLE_SUBWORKFLOW_WORKFLOW } from '@/constants.workflows';
const loadingService = useLoadingService();
const templateStore = useTemplatesStore();
@ -17,10 +15,6 @@ const route = useRoute();
const i18n = useI18n();
const openWorkflowTemplate = async (templateId: string) => {
if (templateId === SAMPLE_SUBWORKFLOW_WORKFLOW.meta.templateId) {
await openSampleSubworkflow();
return;
}
try {
loadingService.startLoading();
const template = await templateStore.getFixedWorkflowTemplate(templateId);
@ -58,42 +52,6 @@ const openWorkflowTemplate = async (templateId: string) => {
}
};
const openSampleSubworkflow = async () => {
try {
loadingService.startLoading();
const projectId = route.query?.projectId;
const sampleSubWorkflows = Number(route.query?.sampleSubWorkflows ?? 0);
const workflowName = `${SAMPLE_SUBWORKFLOW_WORKFLOW.name} ${sampleSubWorkflows + 1}`;
const workflow: IWorkflowDataCreate = {
...SAMPLE_SUBWORKFLOW_WORKFLOW,
name: workflowName,
};
if (projectId) {
workflow.projectId = projectId as string;
}
const newWorkflow = await workflowsStore.createNewWorkflow(workflow);
const sampleSubworkflowChannel = new BroadcastChannel(NEW_SAMPLE_WORKFLOW_CREATED_CHANNEL);
sampleSubworkflowChannel.postMessage({ workflowId: newWorkflow.id });
await router.replace({
name: VIEWS.WORKFLOW,
params: { name: newWorkflow.id },
});
loadingService.stopLoading();
} catch (e) {
await router.replace({ name: VIEWS.NEW_WORKFLOW });
loadingService.stopLoading();
}
};
onMounted(async () => {
const templateId = route.params.id;
if (!templateId || typeof templateId !== 'string') {