From 9c6def91975764522fa52cdf21e9cb5bdb4d721d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20G=C3=B3mez=20Morales?= Date: Thu, 28 Nov 2024 11:04:24 +0100 Subject: [PATCH] perf(editor): Virtualize SchemaView (#11694) --- .../@n8n/api-types/src/frontend-settings.ts | 1 + packages/cli/src/config/schema.ts | 7 + packages/cli/src/services/frontend.service.ts | 1 + packages/editor-ui/package.json | 1 + packages/editor-ui/src/__tests__/defaults.ts | 1 + packages/editor-ui/src/components/RunData.vue | 8 +- .../src/components/VirtualSchema.test.ts | 439 ++++++ .../src/components/VirtualSchema.vue | 292 ++++ .../src/components/VirtualSchemaHeader.vue | 104 ++ .../src/components/VirtualSchemaItem.vue | 142 ++ .../__snapshots__/VirtualSchema.test.ts.snap | 1256 +++++++++++++++++ .../src/composables/useDataSchema.test.ts | 38 +- .../src/composables/useDataSchema.ts | 212 ++- .../editor-ui/src/vue-virtual-scroller.d.ts | 112 ++ pnpm-lock.yaml | 140 +- 15 files changed, 2660 insertions(+), 94 deletions(-) create mode 100644 packages/editor-ui/src/components/VirtualSchema.test.ts create mode 100644 packages/editor-ui/src/components/VirtualSchema.vue create mode 100644 packages/editor-ui/src/components/VirtualSchemaHeader.vue create mode 100644 packages/editor-ui/src/components/VirtualSchemaItem.vue create mode 100644 packages/editor-ui/src/components/__snapshots__/VirtualSchema.test.ts.snap create mode 100644 packages/editor-ui/src/vue-virtual-scroller.d.ts diff --git a/packages/@n8n/api-types/src/frontend-settings.ts b/packages/@n8n/api-types/src/frontend-settings.ts index 54b7956821..8f9c740ad6 100644 --- a/packages/@n8n/api-types/src/frontend-settings.ts +++ b/packages/@n8n/api-types/src/frontend-settings.ts @@ -172,4 +172,5 @@ export interface FrontendSettings { blockFileAccessToN8nFiles: boolean; }; betaFeatures: FrontendBetaFeatures[]; + virtualSchemaView: boolean; } diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index 54fa07e7f5..1891d8193d 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -405,4 +405,11 @@ export const schema = { doc: 'Set this to 1 to enable the new partial execution logic by default.', }, }, + + virtualSchemaView: { + doc: 'Whether to display the virtualized schema view', + format: Boolean, + default: false, + env: 'N8N_VIRTUAL_SCHEMA_VIEW', + }, }; diff --git a/packages/cli/src/services/frontend.service.ts b/packages/cli/src/services/frontend.service.ts index 5fad80d83c..79d04b2263 100644 --- a/packages/cli/src/services/frontend.service.ts +++ b/packages/cli/src/services/frontend.service.ts @@ -231,6 +231,7 @@ export class FrontendService { blockFileAccessToN8nFiles: this.securityConfig.blockFileAccessToN8nFiles, }, betaFeatures: this.frontendConfig.betaFeatures, + virtualSchemaView: config.getEnv('virtualSchemaView'), }; } diff --git a/packages/editor-ui/package.json b/packages/editor-ui/package.json index fb279ef889..073440d07c 100644 --- a/packages/editor-ui/package.json +++ b/packages/editor-ui/package.json @@ -79,6 +79,7 @@ "vue-json-pretty": "2.2.4", "vue-markdown-render": "catalog:frontend", "vue-router": "catalog:frontend", + "vue-virtual-scroller": "2.0.0-beta.8", "vue3-touch-events": "^4.1.3", "xss": "catalog:" }, diff --git a/packages/editor-ui/src/__tests__/defaults.ts b/packages/editor-ui/src/__tests__/defaults.ts index 5746ca5a20..dff54f420f 100644 --- a/packages/editor-ui/src/__tests__/defaults.ts +++ b/packages/editor-ui/src/__tests__/defaults.ts @@ -126,4 +126,5 @@ export const defaultSettings: FrontendSettings = { enabled: false, }, betaFeatures: [], + virtualSchemaView: false, }; diff --git a/packages/editor-ui/src/components/RunData.vue b/packages/editor-ui/src/components/RunData.vue index 9288f0731a..803bc86a0e 100644 --- a/packages/editor-ui/src/components/RunData.vue +++ b/packages/editor-ui/src/components/RunData.vue @@ -46,6 +46,7 @@ import { useExternalHooks } from '@/composables/useExternalHooks'; import { useI18n } from '@/composables/useI18n'; import { useNodeHelpers } from '@/composables/useNodeHelpers'; import { useNodeType } from '@/composables/useNodeType'; +import { useSettingsStore } from '@/stores/settings.store'; import type { PinDataSource, UnpinDataSource } from '@/composables/usePinnedData'; import { usePinnedData } from '@/composables/usePinnedData'; import { useTelemetry } from '@/composables/useTelemetry'; @@ -86,8 +87,11 @@ const LazyRunDataTable = defineAsyncComponent( const LazyRunDataJson = defineAsyncComponent( async () => await import('@/components/RunDataJson.vue'), ); -const LazyRunDataSchema = defineAsyncComponent( - async () => await import('@/components/RunDataSchema.vue'), + +const LazyRunDataSchema = defineAsyncComponent(async () => + useSettingsStore().settings.virtualSchemaView + ? await import('@/components/VirtualSchema.vue') + : await import('@/components/RunDataSchema.vue'), ); const LazyRunDataHtml = defineAsyncComponent( async () => await import('@/components/RunDataHtml.vue'), diff --git a/packages/editor-ui/src/components/VirtualSchema.test.ts b/packages/editor-ui/src/components/VirtualSchema.test.ts new file mode 100644 index 0000000000..0a000ea24e --- /dev/null +++ b/packages/editor-ui/src/components/VirtualSchema.test.ts @@ -0,0 +1,439 @@ +import { createComponentRenderer } from '@/__tests__/render'; +import VirtualSchema from '@/components/VirtualSchema.vue'; +import { useWorkflowsStore } from '@/stores/workflows.store'; +import { userEvent } from '@testing-library/user-event'; +import { cleanup, waitFor } from '@testing-library/vue'; +import { createPinia, setActivePinia } from 'pinia'; +import { + createTestNode, + defaultNodeDescriptions, + mockNodeTypeDescription, +} from '@/__tests__/mocks'; +import { IF_NODE_TYPE, SET_NODE_TYPE, MANUAL_TRIGGER_NODE_TYPE } from '@/constants'; +import { useNodeTypesStore } from '@/stores/nodeTypes.store'; +import { mock } from 'vitest-mock-extended'; +import type { IWorkflowDb } from '@/Interface'; +import { NodeConnectionType, type IDataObject, type INodeExecutionData } from 'n8n-workflow'; +import * as nodeHelpers from '@/composables/useNodeHelpers'; +import { useNDVStore } from '@/stores/ndv.store'; +import { fireEvent } from '@testing-library/dom'; +import { useTelemetry } from '@/composables/useTelemetry'; + +const mockNode1 = createTestNode({ + name: 'Manual Trigger', + type: MANUAL_TRIGGER_NODE_TYPE, + typeVersion: 1, + disabled: false, +}); + +const mockNode2 = createTestNode({ + name: 'Set2', + type: SET_NODE_TYPE, + typeVersion: 1, + disabled: false, +}); + +const disabledNode = createTestNode({ + name: 'Disabled Node', + type: SET_NODE_TYPE, + typeVersion: 1, + disabled: true, +}); + +const ifNode = createTestNode({ + name: 'If', + type: IF_NODE_TYPE, + typeVersion: 1, + disabled: false, +}); + +const aiTool = createTestNode({ + name: 'AI Tool', + type: '@n8n/n8n-nodes-langchain.memoryBufferWindow', + typeVersion: 1, + disabled: false, +}); + +const unknownNodeType = createTestNode({ + name: 'Unknown Node Type', + type: 'unknown', +}); + +const defaultNodes = [ + { name: 'Manual Trigger', indicies: [], depth: 1 }, + { name: 'Set2', indicies: [], depth: 2 }, +]; + +async function setupStore() { + const workflow = mock({ + id: '123', + name: 'Test Workflow', + connections: {}, + active: true, + nodes: [mockNode1, mockNode2, disabledNode, ifNode, aiTool, unknownNodeType], + }); + + const pinia = createPinia(); + setActivePinia(pinia); + + const workflowsStore = useWorkflowsStore(); + const nodeTypesStore = useNodeTypesStore(); + + nodeTypesStore.setNodeTypes([ + ...defaultNodeDescriptions, + mockNodeTypeDescription({ + name: MANUAL_TRIGGER_NODE_TYPE, + outputs: [NodeConnectionType.Main], + }), + mockNodeTypeDescription({ + name: IF_NODE_TYPE, + outputs: [NodeConnectionType.Main, NodeConnectionType.Main], + }), + ]); + workflowsStore.workflow = workflow; + + return pinia; +} + +function mockNodeOutputData(nodeName: string, data: IDataObject[], outputIndex = 0) { + const originalNodeHelpers = nodeHelpers.useNodeHelpers(); + vi.spyOn(nodeHelpers, 'useNodeHelpers').mockImplementation(() => { + return { + ...originalNodeHelpers, + getNodeInputData: vi.fn((node, _, output) => { + if (node.name === nodeName && output === outputIndex) { + return data.map((json) => ({ json })); + } + return []; + }), + }; + }); +} + +describe('RunDataSchema.vue', () => { + let renderComponent: ReturnType; + + const DynamicScrollerStub = { + props: { + items: Array, + }, + template: + '
', + methods: { + scrollToItem: vi.fn(), + }, + }; + + const DynamicScrollerItemStub = { + template: '', + }; + + beforeEach(async () => { + cleanup(); + renderComponent = createComponentRenderer(VirtualSchema, { + global: { + stubs: { + DynamicScroller: DynamicScrollerStub, + DynamicScrollerItem: DynamicScrollerItemStub, + FontAwesomeIcon: true, + }, + }, + pinia: await setupStore(), + props: { + mappingEnabled: true, + runIndex: 1, + outputIndex: 0, + totalRuns: 2, + paneType: 'input', + connectionType: 'main', + search: '', + nodes: defaultNodes, + }, + }); + }); + + it('renders schema for empty data', async () => { + const { getAllByText, getAllByTestId } = renderComponent(); + + expect(getAllByText("No fields - item(s) exist, but they're empty").length).toBe(2); + + // Collapse second node + await userEvent.click(getAllByTestId('run-data-schema-header')[1]); + expect(getAllByText("No fields - item(s) exist, but they're empty").length).toBe(1); + }); + + it('renders schema for data', async () => { + useWorkflowsStore().pinData({ + node: mockNode1, + data: [ + { json: { name: 'John', age: 22, hobbies: ['surfing', 'traveling'] } }, + { json: { name: 'Joe', age: 33, hobbies: ['skateboarding', 'gaming'] } }, + ], + }); + useWorkflowsStore().pinData({ + node: mockNode2, + data: [ + { json: { name: 'John', age: 22, hobbies: ['surfing', 'traveling'] } }, + { json: { name: 'Joe', age: 33, hobbies: ['skateboarding', 'gaming'] } }, + ], + }); + + const { getAllByTestId } = renderComponent(); + const headers = getAllByTestId('run-data-schema-header'); + expect(headers.length).toBe(2); + expect(headers[0]).toHaveTextContent('Manual Trigger'); + expect(headers[0]).toHaveTextContent('2 items'); + expect(headers[1]).toHaveTextContent('Set2'); + + const items = getAllByTestId('run-data-schema-item'); + + expect(items[0]).toHaveTextContent('nameJohn'); + expect(items[1]).toHaveTextContent('age22'); + expect(items[2]).toHaveTextContent('hobbies'); + expect(items[3]).toHaveTextContent('hobbies[0]surfing'); + expect(items[4]).toHaveTextContent('hobbies[1]traveling'); + }); + + it('renders schema in output pane', async () => { + const { container } = renderComponent({ + props: { + nodes: [], + paneType: 'output', + node: mockNode1, + data: [ + { name: 'John', age: 22, hobbies: ['surfing', 'traveling'] }, + { name: 'Joe', age: 33, hobbies: ['skateboarding', 'gaming'] }, + ], + }, + }); + + expect(container).toMatchSnapshot(); + }); + + it('renders schema with spaces and dots', () => { + useWorkflowsStore().pinData({ + node: mockNode1, + data: [ + { + json: { + 'hello world': [ + { + test: { + 'more to think about': 1, + }, + 'test.how': 'ignore', + }, + ], + }, + }, + ], + }); + + const { container } = renderComponent(); + expect(container).toMatchSnapshot(); + }); + + it('renders no data to show for data empty objects', () => { + useWorkflowsStore().pinData({ + node: mockNode1, + data: [{ json: {} }, { json: {} }], + }); + + const { getAllByText } = renderComponent(); + expect(getAllByText("No fields - item(s) exist, but they're empty").length).toBe(2); + }); + + // this can happen when setting the output to [{}] + it('renders empty state to show for empty data', () => { + useWorkflowsStore().pinData({ + node: mockNode1, + data: [{} as INodeExecutionData], + }); + + const { getAllByText } = renderComponent({ props: { paneType: 'output' } }); + expect(getAllByText("No fields - item(s) exist, but they're empty").length).toBe(1); + }); + + it('renders disabled nodes correctly', () => { + const { getByTestId } = renderComponent({ + props: { + nodes: [{ name: disabledNode.name, indicies: [], depth: 1 }], + }, + }); + expect(getByTestId('run-data-schema-header')).toHaveTextContent( + `${disabledNode.name} (Deactivated)`, + ); + }); + + it('renders schema for correct output branch', async () => { + mockNodeOutputData( + 'If', + [ + { id: 1, name: 'John' }, + { id: 2, name: 'Jane' }, + ], + 1, + ); + const { getByTestId } = renderComponent({ + props: { + nodes: [{ name: 'If', indicies: [1], depth: 2 }], + }, + }); + + await waitFor(() => { + expect(getByTestId('run-data-schema-header')).toHaveTextContent('If'); + expect(getByTestId('run-data-schema-header')).toHaveTextContent('2 items'); + expect(getByTestId('run-data-schema-header')).toMatchSnapshot(); + }); + }); + + it('renders previous nodes schema for AI tools', async () => { + mockNodeOutputData( + 'If', + [ + { id: 1, name: 'John' }, + { id: 2, name: 'Jane' }, + ], + 0, + ); + const { getByTestId } = renderComponent({ + props: { + nodes: [ + { + name: 'If', + indicies: [], // indicies are not set for AI tools + depth: 2, + }, + ], + node: aiTool, + }, + }); + + await waitFor(() => { + expect(getByTestId('run-data-schema-header')).toHaveTextContent('If'); + expect(getByTestId('run-data-schema-header')).toHaveTextContent('2 items'); + expect(getByTestId('run-data-schema-header')).toMatchSnapshot(); + }); + }); + + it('renders its own data for AI tools in debug mode', async () => { + const { getByTestId } = renderComponent({ + props: { + nodes: [], // in debug mode nodes are empty + node: aiTool, + data: [{ output: 'AI tool output' }], + }, + }); + + expect(getByTestId('run-data-schema-item')).toHaveTextContent('AI tool output'); + }); + + test.each([[[{ tx: false }, { tx: false }]], [[{ tx: '' }, { tx: '' }]], [[{ tx: [] }]]])( + 'renders schema instead of showing no data for %o', + (data) => { + useWorkflowsStore().pinData({ + node: mockNode1, + data: data.map((item) => ({ json: item })), + }); + + const { getAllByTestId } = renderComponent(); + expect(getAllByTestId('run-data-schema-item')[0]).toHaveTextContent('tx'); + }, + ); + + it('should filter invalid connections', () => { + const { pinData } = useWorkflowsStore(); + pinData({ + node: mockNode1, + data: [{ json: { tx: 1 } }], + }); + pinData({ + node: mockNode2, + data: [{ json: { tx: 2 } }], + }); + + const { getAllByTestId } = renderComponent({ + props: { + nodes: [ + { name: mockNode1.name, indicies: [], depth: 1 }, + { name: 'unknown', indicies: [], depth: 1 }, + { name: mockNode2.name, indicies: [], depth: 1 }, + { name: unknownNodeType.name, indicies: [], depth: 1 }, + ], + }, + }); + + expect(getAllByTestId('run-data-schema-item').length).toBe(2); + }); + + it('should show connections', () => { + const ndvStore = useNDVStore(); + vi.spyOn(ndvStore, 'ndvNodeInputNumber', 'get').mockReturnValue({ + [defaultNodes[0].name]: [0], + [defaultNodes[1].name]: [0, 1, 2], + }); + + const { getAllByTestId } = renderComponent(); + const headers = getAllByTestId('run-data-schema-header'); + expect(headers.length).toBe(2); + expect(headers[0]).toHaveTextContent('Input 0'); + expect(headers[1]).toHaveTextContent('Inputs 0, 1, 2'); + }); + + it('should handle drop event', async () => { + const ndvStore = useNDVStore(); + useWorkflowsStore().pinData({ + node: mockNode1, + data: [{ json: { name: 'John', age: 22, hobbies: ['surfing', 'traveling'] } }], + }); + const telemetry = useTelemetry(); + const trackSpy = vi.spyOn(telemetry, 'track'); + const reset = vi.spyOn(ndvStore, 'resetMappingTelemetry'); + const { getAllByTestId } = renderComponent(); + + const items = getAllByTestId('run-data-schema-item'); + expect(items.length).toBe(6); + + expect(items[0].className).toBe('schema-item draggable'); + expect(items[0]).toHaveTextContent('nameJohn'); + + const pill = items[0].querySelector('.pill') as Element; + + fireEvent(pill, new MouseEvent('mousedown', { bubbles: true })); + fireEvent(window, new MouseEvent('mousemove', { bubbles: true })); + expect(reset).toHaveBeenCalled(); + + fireEvent(window, new MouseEvent('mouseup', { bubbles: true })); + + await waitFor(() => + expect(trackSpy).toHaveBeenCalledWith( + 'User dragged data for mapping', + expect.any(Object), + expect.any(Object), + ), + ); + }); + + it('should expand all nodes when searching', async () => { + useWorkflowsStore().pinData({ + node: mockNode1, + data: [{ json: { name: 'John' } }], + }); + useWorkflowsStore().pinData({ + node: mockNode2, + data: [{ json: { name: 'John' } }], + }); + + const { getAllByTestId, queryAllByTestId, rerender } = renderComponent(); + const headers = getAllByTestId('run-data-schema-header'); + expect(headers.length).toBe(2); + expect(getAllByTestId('run-data-schema-item').length).toBe(2); + + // Collapse all nodes + await Promise.all(headers.map(async ($header) => await userEvent.click($header))); + + expect(queryAllByTestId('run-data-schema-item').length).toBe(0); + await rerender({ search: 'John' }); + expect(getAllByTestId('run-data-schema-item').length).toBe(2); + }); +}); diff --git a/packages/editor-ui/src/components/VirtualSchema.vue b/packages/editor-ui/src/components/VirtualSchema.vue new file mode 100644 index 0000000000..2a4c0ebedc --- /dev/null +++ b/packages/editor-ui/src/components/VirtualSchema.vue @@ -0,0 +1,292 @@ + + + + + diff --git a/packages/editor-ui/src/components/VirtualSchemaHeader.vue b/packages/editor-ui/src/components/VirtualSchemaHeader.vue new file mode 100644 index 0000000000..677f88ba5c --- /dev/null +++ b/packages/editor-ui/src/components/VirtualSchemaHeader.vue @@ -0,0 +1,104 @@ + + + + + diff --git a/packages/editor-ui/src/components/VirtualSchemaItem.vue b/packages/editor-ui/src/components/VirtualSchemaItem.vue new file mode 100644 index 0000000000..84e9a49b8c --- /dev/null +++ b/packages/editor-ui/src/components/VirtualSchemaItem.vue @@ -0,0 +1,142 @@ + + + + + diff --git a/packages/editor-ui/src/components/__snapshots__/VirtualSchema.test.ts.snap b/packages/editor-ui/src/components/__snapshots__/VirtualSchema.test.ts.snap new file mode 100644 index 0000000000..7c1fa63139 --- /dev/null +++ b/packages/editor-ui/src/components/__snapshots__/VirtualSchema.test.ts.snap @@ -0,0 +1,1256 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`RunDataSchema.vue > renders previous nodes schema for AI tools 1`] = ` +
+
+ +
+
+
+ + +
+ + +
+ +
+
+
+ If + +
+ +
+ 2 items +
+
+`; + +exports[`RunDataSchema.vue > renders schema for correct output branch 1`] = ` +
+
+ +
+
+
+ + +
+ + +
+ +
+
+
+ If + +
+ +
+ 2 items +
+
+`; + +exports[`RunDataSchema.vue > renders schema in output pane 1`] = ` +
+
+ +
+ +
+ + + +
+
+ +
+
+ + + + + + name + + + +
+ + + + + John + + + +
+ + + + +
+
+ +
+
+ + + + + + age + + + +
+ + + + + 22 + + + +
+ + + + +
+
+
+ +
+
+
+ + + + + + hobbies + + + +
+ + + +
+ + + + +
+
+ +
+
+ + + + + + hobbies[0] + + + +
+ + + + + surfing + + + +
+ + + + +
+
+ +
+
+ + + + + + hobbies[1] + + + +
+ + + + + traveling + + + +
+ + + +
+ + + +
+
+
+`; + +exports[`RunDataSchema.vue > renders schema with spaces and dots 1`] = ` +
+
+ +
+ +
+ + +
+
+ +
+
+
+ + +
+ + +
+ +
+
+
+ Manual Trigger + +
+ +
+ 1 item +
+
+ + + +
+
+
+ +
+
+
+ + + + + + hello world + + + +
+ + + +
+ + + + +
+
+
+ +
+
+
+ + + + + + hello world[0] + + + +
+ + + +
+ + + + +
+
+
+ +
+
+
+ + + + + + test + + + +
+ + + +
+ + + + +
+
+ +
+
+ + + + + + more to think about + + + +
+ + + + + 1 + + + +
+ + + + +
+
+ +
+
+ + + + + + test.how + + + +
+ + + + + ignore + + + +
+ + + +
+
+ +
+
+
+ + +
+ + +
+ +
+
+
+ Set2 + +
+ + +
+ + + +
+
+ +
+ + + + + + No fields - item(s) exist, but they're empty + + + +
+ + + +
+ + + +
+
+
+`; diff --git a/packages/editor-ui/src/composables/useDataSchema.test.ts b/packages/editor-ui/src/composables/useDataSchema.test.ts index 3c685550c7..212f767675 100644 --- a/packages/editor-ui/src/composables/useDataSchema.test.ts +++ b/packages/editor-ui/src/composables/useDataSchema.test.ts @@ -1,5 +1,5 @@ import jp from 'jsonpath'; -import { useDataSchema } from '@/composables/useDataSchema'; +import { useDataSchema, useFlattenSchema } from '@/composables/useDataSchema'; import type { IExecutionResponse, INodeUi, Schema } from '@/Interface'; import { setActivePinia } from 'pinia'; import { createTestingPinia } from '@pinia/testing'; @@ -649,3 +649,39 @@ describe('useDataSchema', () => { ); }); }); + +describe('useFlattenSchema', () => { + it('flattens a schema', () => { + const schema: Schema = { + path: '', + type: 'object', + value: [ + { + key: 'obj', + path: '.obj', + type: 'object', + value: [ + { + key: 'foo', + path: '.obj.foo', + type: 'object', + value: [ + { + key: 'nested', + path: '.obj.foo.nested', + type: 'string', + value: 'bar', + }, + ], + }, + ], + }, + ], + }; + expect( + useFlattenSchema().flattenSchema({ + schema, + }).length, + ).toBe(3); + }); +}); diff --git a/packages/editor-ui/src/composables/useDataSchema.ts b/packages/editor-ui/src/composables/useDataSchema.ts index 8b20ab93fc..8896f8dd7e 100644 --- a/packages/editor-ui/src/composables/useDataSchema.ts +++ b/packages/editor-ui/src/composables/useDataSchema.ts @@ -1,15 +1,18 @@ +import { ref } from 'vue'; import type { Optional, Primitives, Schema, INodeUi } from '@/Interface'; import { type ITaskDataConnections, type IDataObject, type INodeExecutionData, + type INodeTypeDescription, NodeConnectionType, } from 'n8n-workflow'; import { merge } from 'lodash-es'; -import { generatePath } from '@/utils/mappingUtils'; +import { generatePath, getMappedExpression } from '@/utils/mappingUtils'; import { isObj } from '@/utils/typeGuards'; import { useWorkflowsStore } from '@/stores/workflows.store'; -import { isPresent } from '@/utils/typesUtils'; +import { isPresent, shorten } from '@/utils/typesUtils'; +import { useI18n } from '@/composables/useI18n'; export function useDataSchema() { function getSchema( @@ -166,3 +169,208 @@ export function useDataSchema() { filterSchema, }; } + +export type SchemaNode = { + node: INodeUi; + nodeType: INodeTypeDescription; + depth: number; + connectedOutputIndexes: number[]; + itemsCount: number; + schema: Schema; +}; + +export type RenderItem = { + title?: string; + path?: string; + level?: number; + depth?: number; + expression?: string; + value?: string; + id: string; + icon: string; + collapsable?: boolean; + nodeType?: INodeUi['type']; + type: 'item'; +}; + +export type RenderHeader = { + id: string; + title: string; + info?: string; + collapsable: boolean; + nodeType: INodeTypeDescription; + itemCount: number | null; + type: 'header'; +}; + +type Renders = RenderHeader | RenderItem; + +const icons = { + object: 'cube', + array: 'list', + ['string']: 'font', + null: 'font', + ['number']: 'hashtag', + ['boolean']: 'check-square', + function: 'code', + bigint: 'calculator', + symbol: 'sun', + ['undefined']: 'ban', +} as const; + +const getIconBySchemaType = (type: Schema['type']): string => icons[type]; + +const emptyItem = (): RenderItem => ({ + id: `empty-${window.crypto.randomUUID()}`, + icon: '', + value: useI18n().baseText('dataMapping.schemaView.emptyData'), + type: 'item', +}); + +const isDataEmpty = (schema: Schema) => { + // Utilize the generated schema instead of looping over the entire data again + // The schema for empty data is { type: 'object', value: [] } + const isObjectOrArray = schema.type === 'object'; + const isEmpty = Array.isArray(schema.value) && schema.value.length === 0; + + return isObjectOrArray && isEmpty; +}; + +const prefixTitle = (title: string, prefix?: string) => (prefix ? `${prefix}[${title}]` : title); + +export const useFlattenSchema = () => { + const closedNodes = ref>(new Set()); + const headerIds = ref>(new Set()); + const toggleLeaf = (id: string) => { + if (closedNodes.value.has(id)) { + closedNodes.value.delete(id); + } else { + closedNodes.value.add(id); + } + }; + + const toggleNode = (id: string) => { + if (closedNodes.value.has(id)) { + closedNodes.value = new Set(headerIds.value); + closedNodes.value.delete(id); + } else { + closedNodes.value.add(id); + } + }; + + const flattenSchema = ({ + schema, + node = { name: '', type: '' }, + depth = 0, + prefix = '', + level = 0, + }: { + schema: Schema; + node?: { name: string; type: string }; + depth?: number; + prefix?: string; + level?: number; + }): RenderItem[] => { + // only show empty item for the first level + if (isDataEmpty(schema) && depth <= 0) { + return [emptyItem()]; + } + + const expression = getMappedExpression({ + nodeName: node.name, + distanceFromActive: depth, + path: schema.path, + }); + + if (Array.isArray(schema.value)) { + const items: RenderItem[] = []; + + if (schema.key) { + items.push({ + title: prefixTitle(schema.key, prefix), + path: schema.path, + expression, + depth, + level, + icon: getIconBySchemaType(schema.type), + id: expression, + collapsable: true, + nodeType: node.type, + type: 'item', + }); + } + + if (closedNodes.value.has(expression)) { + return items; + } + + return items.concat( + schema.value + .map((item) => { + const itemPrefix = schema.type === 'array' ? schema.key : ''; + return flattenSchema({ + schema: item, + node, + depth, + prefix: itemPrefix, + level: level + 1, + }); + }) + .flat(), + ); + } else if (schema.key) { + return [ + { + title: prefixTitle(schema.key, prefix), + path: schema.path, + expression, + level, + depth, + value: shorten(schema.value, 600, 0), + id: expression, + icon: getIconBySchemaType(schema.type), + collapsable: false, + nodeType: node.type, + type: 'item', + }, + ]; + } + + return []; + }; + + const flattenMultipleSchemas = ( + nodes: SchemaNode[], + additionalInfo: (node: INodeUi) => string, + ) => { + headerIds.value.clear(); + + return nodes.reduce((acc, item) => { + acc.push({ + title: item.node.name, + id: item.node.name, + collapsable: true, + nodeType: item.nodeType, + itemCount: item.itemsCount, + info: additionalInfo(item.node), + type: 'header', + }); + + headerIds.value.add(item.node.name); + + if (closedNodes.value.has(item.node.name)) { + return acc; + } + + if (isDataEmpty(item.schema)) { + acc.push(emptyItem()); + return acc; + } + + acc.push(...flattenSchema(item)); + return acc; + }, []); + }; + + return { closedNodes, toggleLeaf, toggleNode, flattenSchema, flattenMultipleSchemas }; +}; diff --git a/packages/editor-ui/src/vue-virtual-scroller.d.ts b/packages/editor-ui/src/vue-virtual-scroller.d.ts new file mode 100644 index 0000000000..1e6ef322aa --- /dev/null +++ b/packages/editor-ui/src/vue-virtual-scroller.d.ts @@ -0,0 +1,112 @@ +declare module 'vue-virtual-scroller' { + import { + type ObjectEmitsOptions, + type PublicProps, + type SetupContext, + type SlotsType, + type VNode, + } from 'vue'; + + interface RecycleScrollerProps { + items: readonly T[]; + direction?: 'vertical' | 'horizontal'; + itemSize?: number | null; + gridItems?: number; + itemSecondarySize?: number; + minItemSize?: number; + sizeField?: string; + typeField?: string; + keyField?: keyof T; + pageMode?: boolean; + prerender?: number; + buffer?: number; + emitUpdate?: boolean; + updateInterval?: number; + listClass?: string; + itemClass?: string; + listTag?: string; + itemTag?: string; + } + + interface DynamicScrollerProps extends RecycleScrollerProps { + minItemSize: number; + } + + interface RecycleScrollerEmitOptions extends ObjectEmitsOptions { + resize: () => void; + visible: () => void; + hidden: () => void; + update: ( + startIndex: number, + endIndex: number, + visibleStartIndex: number, + visibleEndIndex: number, + ) => void; + 'scroll-start': () => void; + 'scroll-end': () => void; + } + + interface RecycleScrollerSlotProps { + item: T; + index: number; + active: boolean; + } + + interface RecycleScrollerSlots { + default(slotProps: RecycleScrollerSlotProps): unknown; + before(): unknown; + empty(): unknown; + after(): unknown; + } + + export interface RecycleScrollerInstance { + getScroll(): { start: number; end: number }; + scrollToItem(index: number): void; + scrollToPosition(position: number): void; + } + + export const RecycleScroller: ( + props: RecycleScrollerProps & PublicProps, + ctx?: SetupContext>>, + expose?: (exposed: RecycleScrollerInstance) => void, + ) => VNode & { + __ctx?: { + props: RecycleScrollerProps & PublicProps; + expose(exposed: RecycleScrollerInstance): void; + slots: RecycleScrollerSlots; + }; + }; + + export const DynamicScroller: ( + props: DynamicScrollerProps & PublicProps, + ctx?: SetupContext>>, + expose?: (exposed: RecycleScrollerInstance) => void, + ) => VNode & { + __ctx?: { + props: DynamicScrollerProps & PublicProps; + expose(exposed: RecycleScrollerInstance): void; + slots: RecycleScrollerSlots; + }; + }; + + interface DynamicScrollerItemProps { + item: T; + active: boolean; + sizeDependencies?: unknown[]; + watchData?: boolean; + tag?: string; + emitResize?: boolean; + onResize?: () => void; + } + + interface DynamicScrollerItemEmitOptions extends ObjectEmitsOptions { + resize: () => void; + } + + export const DynamicScrollerItem: ( + props: DynamicScrollerItemProps & PublicProps, + ctx?: SetupContext, + ) => VNode; + + export function IdState(options?: { idProp?: (value: any) => unknown }): unknown; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ca720e72fa..3eabd32411 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1114,7 +1114,7 @@ importers: dependencies: '@langchain/core': specifier: 'catalog:' - version: 0.3.15(openai@4.69.0(zod@3.23.8)) + version: 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)) '@n8n/client-oauth2': specifier: workspace:* version: link:../@n8n/client-oauth2 @@ -1513,6 +1513,9 @@ importers: vue-router: specifier: catalog:frontend version: 4.4.5(vue@3.5.11(typescript@5.7.2)) + vue-virtual-scroller: + specifier: 2.0.0-beta.8 + version: 2.0.0-beta.8(vue@3.5.11(typescript@5.7.2)) vue3-touch-events: specifier: ^4.1.3 version: 4.1.3 @@ -1942,7 +1945,7 @@ importers: devDependencies: '@langchain/core': specifier: 'catalog:' - version: 0.3.15(openai@4.69.0) + version: 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)) '@types/deep-equal': specifier: ^1.0.1 version: 1.0.1 @@ -9244,6 +9247,9 @@ packages: resolution: {integrity: sha512-MGZAq0Q3OuRYgZKvlB69z4gLN4G3PvgC4A2zhkCXCXrLD5wm2cCnwNB59xOBVA+srZ0zEes6u+VylcPIkB4SqA==} engines: {node: 6.* || 8.* || >= 10.*} + mitt@2.1.0: + resolution: {integrity: sha512-ILj2TpLiysu2wkBbWjAmww7TkZb65aiQO+DkVdUTBpBXq+MHYiETENkKFMtsJZX1Lf4pe4QOrTSjIfUwN5lRdg==} + mjml-accordion@4.15.3: resolution: {integrity: sha512-LPNVSj1LyUVYT9G1gWwSw3GSuDzDsQCu0tPB2uDsq4VesYNnU6v3iLCQidMiR6azmIt13OEozG700ygAUuA6Ng==} @@ -11004,10 +11010,6 @@ packages: resolution: {integrity: sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==} engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} - source-map-js@1.0.2: - resolution: {integrity: sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==} - engines: {node: '>=0.10.0'} - source-map-js@1.2.0: resolution: {integrity: sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==} engines: {node: '>=0.10.0'} @@ -12134,6 +12136,16 @@ packages: peerDependencies: vue: ^3.3.4 + vue-observe-visibility@2.0.0-alpha.1: + resolution: {integrity: sha512-flFbp/gs9pZniXR6fans8smv1kDScJ8RS7rEpMjhVabiKeq7Qz3D9+eGsypncjfIyyU84saU88XZ0zjbD6Gq/g==} + peerDependencies: + vue: ^3.0.0 + + vue-resize@2.0.0-alpha.1: + resolution: {integrity: sha512-7+iqOueLU7uc9NrMfrzbG8hwMqchfVfSzpVlCMeJQe4pyibqyoifDNbKTZvwxZKDvGkB+PdFeKvnGZMoEb8esg==} + peerDependencies: + vue: ^3.0.0 + vue-router@4.4.5: resolution: {integrity: sha512-4fKZygS8cH1yCyuabAXGUAsyi1b2/o/OKgu/RUb+znIYOxPRxdkytJEx+0wGcpBE1pX6vUgh5jwWOKRGvuA/7Q==} peerDependencies: @@ -12145,6 +12157,11 @@ packages: peerDependencies: typescript: ^5.7.2 + vue-virtual-scroller@2.0.0-beta.8: + resolution: {integrity: sha512-b8/f5NQ5nIEBRTNi6GcPItE4s7kxNHw2AIHLtDp+2QvqdTjVN0FgONwX9cr53jWRgnu+HRLPaWDOR2JPI5MTfQ==} + peerDependencies: + vue: ^3.2.0 + vue3-touch-events@4.1.3: resolution: {integrity: sha512-uXTclRzn7de1mgiDIZ8N4J/wnWl1vBPLTWr60fqoLXu7ifhDKpl83Q2m9qA20KfEiAy+L4X/xXGc5ptGmdPh4A==} @@ -14667,38 +14684,6 @@ snapshots: transitivePeerDependencies: - openai - '@langchain/core@0.3.15(openai@4.69.0(zod@3.23.8))': - dependencies: - ansi-styles: 5.2.0 - camelcase: 6.3.0 - decamelize: 1.2.0 - js-tiktoken: 1.0.12 - langsmith: 0.2.3(openai@4.69.0(zod@3.23.8)) - mustache: 4.2.0 - p-queue: 6.6.2 - p-retry: 4.6.2 - uuid: 10.0.0 - zod: 3.23.8 - zod-to-json-schema: 3.23.3(zod@3.23.8) - transitivePeerDependencies: - - openai - - '@langchain/core@0.3.15(openai@4.69.0)': - dependencies: - ansi-styles: 5.2.0 - camelcase: 6.3.0 - decamelize: 1.2.0 - js-tiktoken: 1.0.12 - langsmith: 0.2.3(openai@4.69.0) - mustache: 4.2.0 - p-queue: 6.6.2 - p-retry: 4.6.2 - uuid: 10.0.0 - zod: 3.23.8 - zod-to-json-schema: 3.23.3(zod@3.23.8) - transitivePeerDependencies: - - openai - '@langchain/google-common@0.1.1(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8)': dependencies: '@langchain/core': 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)) @@ -19264,7 +19249,7 @@ snapshots: eslint-import-resolver-node@0.3.9: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) is-core-module: 2.13.1 resolve: 1.22.8 transitivePeerDependencies: @@ -19289,7 +19274,7 @@ snapshots: eslint-module-utils@2.8.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.7.2))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0): dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) optionalDependencies: '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.7.2) eslint: 8.57.0 @@ -19309,7 +19294,7 @@ snapshots: array.prototype.findlastindex: 1.2.3 array.prototype.flat: 1.3.2 array.prototype.flatmap: 1.3.2 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 @@ -20102,7 +20087,7 @@ snapshots: array-parallel: 0.1.3 array-series: 0.1.5 cross-spawn: 4.0.2 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -20430,7 +20415,7 @@ snapshots: isstream: 0.1.2 jsonwebtoken: 9.0.2 mime-types: 2.1.35 - retry-axios: 2.6.0(axios@1.7.4(debug@4.3.7)) + retry-axios: 2.6.0(axios@1.7.4) tough-cookie: 4.1.3 transitivePeerDependencies: - supports-color @@ -21272,7 +21257,7 @@ snapshots: form-data: 4.0.0 html-encoding-sniffer: 4.0.0 http-proxy-agent: 7.0.0 - https-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.5 is-potential-custom-element-name: 1.0.1 nwsapi: 2.2.7 parse5: 7.1.2 @@ -21467,28 +21452,6 @@ snapshots: optionalDependencies: openai: 4.69.0(encoding@0.1.13)(zod@3.23.8) - langsmith@0.2.3(openai@4.69.0(zod@3.23.8)): - dependencies: - '@types/uuid': 10.0.0 - commander: 10.0.1 - p-queue: 6.6.2 - p-retry: 4.6.2 - semver: 7.6.0 - uuid: 10.0.0 - optionalDependencies: - openai: 4.69.0(zod@3.23.8) - - langsmith@0.2.3(openai@4.69.0): - dependencies: - '@types/uuid': 10.0.0 - commander: 10.0.1 - p-queue: 6.6.2 - p-retry: 4.6.2 - semver: 7.6.0 - uuid: 10.0.0 - optionalDependencies: - openai: 4.69.0(zod@3.23.8) - lazy-ass@1.6.0: {} ldapts@4.2.6: @@ -22002,6 +21965,8 @@ snapshots: lodash: 4.17.21 pretender: 3.4.7 + mitt@2.1.0: {} + mjml-accordion@4.15.3(encoding@0.1.13): dependencies: '@babel/runtime': 7.24.7 @@ -22823,22 +22788,6 @@ snapshots: - encoding - supports-color - openai@4.69.0(zod@3.23.8): - dependencies: - '@types/node': 18.16.16 - '@types/node-fetch': 2.6.4 - abort-controller: 3.0.0 - agentkeepalive: 4.2.1 - form-data-encoder: 1.7.2 - formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) - optionalDependencies: - zod: 3.23.8 - transitivePeerDependencies: - - encoding - - supports-color - optional: true - openapi-sampler@1.5.1: dependencies: '@types/json-schema': 7.0.15 @@ -23019,7 +22968,7 @@ snapshots: pdf-parse@1.1.1: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) node-ensure: 0.0.0 transitivePeerDependencies: - supports-color @@ -23823,9 +23772,9 @@ snapshots: ret@0.1.15: {} - retry-axios@2.6.0(axios@1.7.4(debug@4.3.7)): + retry-axios@2.6.0(axios@1.7.4): dependencies: - axios: 1.7.4(debug@4.3.7) + axios: 1.7.4 retry-request@7.0.2(encoding@0.1.13): dependencies: @@ -23850,7 +23799,7 @@ snapshots: rhea@1.0.24: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -23988,7 +23937,7 @@ snapshots: dependencies: chokidar: 4.0.1 immutable: 4.2.2 - source-map-js: 1.0.2 + source-map-js: 1.2.1 sax@1.2.4: {} @@ -24270,8 +24219,6 @@ snapshots: smart-buffer: 4.2.0 optional: true - source-map-js@1.0.2: {} - source-map-js@1.2.0: {} source-map-js@1.2.1: {} @@ -25494,6 +25441,14 @@ snapshots: markdown-it: 13.0.2 vue: 3.5.11(typescript@5.7.2) + vue-observe-visibility@2.0.0-alpha.1(vue@3.5.11(typescript@5.7.2)): + dependencies: + vue: 3.5.11(typescript@5.7.2) + + vue-resize@2.0.0-alpha.1(vue@3.5.11(typescript@5.7.2)): + dependencies: + vue: 3.5.11(typescript@5.7.2) + vue-router@4.4.5(vue@3.5.11(typescript@5.7.2)): dependencies: '@vue/devtools-api': 6.6.4 @@ -25506,6 +25461,13 @@ snapshots: semver: 7.6.0 typescript: 5.7.2 + vue-virtual-scroller@2.0.0-beta.8(vue@3.5.11(typescript@5.7.2)): + dependencies: + mitt: 2.1.0 + vue: 3.5.11(typescript@5.7.2) + vue-observe-visibility: 2.0.0-alpha.1(vue@3.5.11(typescript@5.7.2)) + vue-resize: 2.0.0-alpha.1(vue@3.5.11(typescript@5.7.2)) + vue3-touch-events@4.1.3: {} vue@3.5.11(typescript@5.7.2):