feat: Add tracking for node errors and update node graph (#11060)

This commit is contained in:
Mutasem Aldmour 2024-10-15 12:38:17 +02:00 committed by GitHub
parent ecbe568d69
commit d3b05f1c54
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 676 additions and 5 deletions

View file

@ -651,7 +651,9 @@ export class TelemetryEventRelay extends EventRelay {
}
if (telemetryProperties.is_manual) {
nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes);
nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes, {
runData: runData.data.resultData?.runData,
});
telemetryProperties.node_graph = nodeGraphResult.nodeGraph;
telemetryProperties.node_graph_string = JSON.stringify(nodeGraphResult.nodeGraph);
@ -663,7 +665,9 @@ export class TelemetryEventRelay extends EventRelay {
if (telemetryProperties.is_manual) {
if (!nodeGraphResult) {
nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes);
nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes, {
runData: runData.data.resultData?.runData,
});
}
let userRole: 'owner' | 'sharee' | undefined = undefined;
@ -688,7 +692,9 @@ export class TelemetryEventRelay extends EventRelay {
};
if (!manualExecEventProperties.node_graph_string) {
nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes);
nodeGraphResult = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes, {
runData: runData.data.resultData?.runData,
});
manualExecEventProperties.node_graph_string = JSON.stringify(nodeGraphResult.nodeGraph);
}

View file

@ -15,6 +15,7 @@ vi.mock('vue-router', () => ({
fullPath: vi.fn(),
}),
RouterLink: vi.fn(),
useRouter: vi.fn(),
}));
let route: ReturnType<typeof useRoute>;

View file

@ -9,6 +9,7 @@ vi.mock('vue-router', () => ({
params: {},
}),
RouterLink: vi.fn(),
useRouter: vi.fn(),
}));
const initialState = {

View file

@ -10,6 +10,7 @@ vi.mock('vue-router', async (importOriginal) => {
useRoute: () => ({
params: {},
}),
useRouter: vi.fn(),
};
});

View file

@ -8,10 +8,12 @@ import {
import { useWorkflowsStore } from '@/stores/workflows.store';
import type { IExecutionResponse, INodeUi, IWorkflowDb, IWorkflowSettings } from '@/Interface';
import { useNodeTypesStore } from '@/stores/nodeTypes.store';
import type { ExecutionSummary, IConnection, INodeExecutionData } from 'n8n-workflow';
import { type ExecutionSummary, type IConnection, type INodeExecutionData } from 'n8n-workflow';
import { stringSizeInBytes } from '@/utils/typesUtils';
import { dataPinningEventBus } from '@/event-bus';
import { useUIStore } from '@/stores/ui.store';
import type { PushPayload } from '@n8n/api-types';
import { flushPromises } from '@vue/test-utils';
vi.mock('@/api/workflows', () => ({
getWorkflows: vi.fn(),
@ -19,12 +21,18 @@ vi.mock('@/api/workflows', () => ({
getNewWorkflow: vi.fn(),
}));
const getNodeType = vi.fn();
vi.mock('@/stores/nodeTypes.store', () => ({
useNodeTypesStore: vi.fn(() => ({
getNodeType: vi.fn(),
getNodeType,
})),
}));
const track = vi.fn();
vi.mock('@/composables/useTelemetry', () => ({
useTelemetry: () => ({ track }),
}));
describe('useWorkflowsStore', () => {
let workflowsStore: ReturnType<typeof useWorkflowsStore>;
let uiStore: ReturnType<typeof useUIStore>;
@ -33,6 +41,7 @@ describe('useWorkflowsStore', () => {
setActivePinia(createPinia());
workflowsStore = useWorkflowsStore();
uiStore = useUIStore();
track.mockReset();
});
it('should initialize with default state', () => {
@ -441,4 +450,197 @@ describe('useWorkflowsStore', () => {
expect(uiStore.stateIsDirty).toBe(true);
});
});
describe('addNodeExecutionData', () => {
const { successEvent, errorEvent, executionReponse } = generateMockExecutionEvents();
it('should throw error if not initalized', () => {
expect(() => workflowsStore.addNodeExecutionData(successEvent)).toThrowError();
});
it('should add node success run data', () => {
workflowsStore.setWorkflowExecutionData(executionReponse);
// ACT
workflowsStore.addNodeExecutionData(successEvent);
expect(workflowsStore.workflowExecutionData).toEqual({
...executionReponse,
data: {
resultData: {
runData: {
[successEvent.nodeName]: [successEvent.data],
},
},
},
});
});
it('should add node error event and track errored executions', async () => {
workflowsStore.setWorkflowExecutionData(executionReponse);
workflowsStore.addNode({
parameters: {},
id: '554c7ff4-7ee2-407c-8931-e34234c5056a',
name: 'Edit Fields',
type: 'n8n-nodes-base.set',
position: [680, 180],
typeVersion: 3.4,
});
getNodeType.mockReturnValue(getMockEditFieldsNode());
// ACT
workflowsStore.addNodeExecutionData(errorEvent);
await flushPromises();
expect(workflowsStore.workflowExecutionData).toEqual({
...executionReponse,
data: {
resultData: {
runData: {
[errorEvent.nodeName]: [errorEvent.data],
},
},
},
});
expect(track).toHaveBeenCalledWith(
'Manual exec errored',
{
error_title: 'invalid syntax',
node_type: 'n8n-nodes-base.set',
node_type_version: 3.4,
node_id: '554c7ff4-7ee2-407c-8931-e34234c5056a',
node_graph_string:
'{"node_types":["n8n-nodes-base.set"],"node_connections":[],"nodes":{"0":{"id":"554c7ff4-7ee2-407c-8931-e34234c5056a","type":"n8n-nodes-base.set","version":3.4,"position":[680,180]}},"notes":{},"is_pinned":false}',
},
{
withPostHog: true,
},
);
});
});
});
function getMockEditFieldsNode() {
return {
displayName: 'Edit Fields (Set)',
name: 'n8n-nodes-base.set',
icon: 'fa:pen',
group: ['input'],
description: 'Modify, add, or remove item fields',
defaultVersion: 3.4,
iconColor: 'blue',
version: [3, 3.1, 3.2, 3.3, 3.4],
subtitle: '={{$parameter["mode"]}}',
defaults: {
name: 'Edit Fields',
},
inputs: ['main'],
outputs: ['main'],
properties: [],
};
}
function generateMockExecutionEvents() {
const executionReponse: IExecutionResponse = {
id: '1',
workflowData: {
id: '1',
name: '',
createdAt: '1',
updatedAt: '1',
nodes: [],
connections: {},
active: false,
versionId: '1',
},
finished: false,
mode: 'cli',
startedAt: new Date(),
status: 'new',
data: {
resultData: {
runData: {},
},
},
};
const successEvent: PushPayload<'nodeExecuteAfter'> = {
executionId: '59',
nodeName: 'When clicking Test workflow',
data: {
hints: [],
startTime: 1727867966633,
executionTime: 1,
source: [],
executionStatus: 'success',
data: {
main: [
[
{
json: {},
pairedItem: {
item: 0,
},
},
],
],
},
},
};
const errorEvent: PushPayload<'nodeExecuteAfter'> = {
executionId: '61',
nodeName: 'Edit Fields',
data: {
hints: [],
startTime: 1727869043441,
executionTime: 2,
source: [
{
previousNode: 'When clicking Test workflow',
},
],
executionStatus: 'error',
// @ts-expect-error simpler data type, not BE class with methods
error: {
level: 'error',
tags: {
packageName: 'workflow',
},
context: {
itemIndex: 0,
},
functionality: 'regular',
name: 'NodeOperationError',
timestamp: 1727869043442,
node: {
parameters: {
mode: 'manual',
duplicateItem: false,
assignments: {
assignments: [
{
id: '87afdb19-4056-4551-93ef-d0126a34eb83',
name: "={{ $('Wh }}",
value: '',
type: 'string',
},
],
},
includeOtherFields: false,
options: {},
},
id: '9fb34d2d-7191-48de-8f18-91a6a28d0230',
name: 'Edit Fields',
type: 'n8n-nodes-base.set',
typeVersion: 3.4,
position: [1120, 180],
},
messages: [],
message: 'invalid syntax',
stack: 'NodeOperationError: invalid syntax',
},
},
};
return { executionReponse, errorEvent, successEvent };
}

View file

@ -80,6 +80,11 @@ import { useProjectsStore } from '@/stores/projects.store';
import type { ProjectSharingData } from '@/types/projects.types';
import type { PushPayload } from '@n8n/api-types';
import { useLocalStorage } from '@vueuse/core';
import { useTelemetry } from '@/composables/useTelemetry';
import { TelemetryHelpers } from 'n8n-workflow';
import { useWorkflowHelpers } from '@/composables/useWorkflowHelpers';
import { useRouter } from 'vue-router';
import { useSettingsStore } from './settings.store';
const defaults: Omit<IWorkflowDb, 'id'> & { settings: NonNullable<IWorkflowDb['settings']> } = {
name: '',
@ -107,6 +112,10 @@ let cachedWorkflow: Workflow | null = null;
export const useWorkflowsStore = defineStore(STORES.WORKFLOWS, () => {
const uiStore = useUIStore();
const telemetry = useTelemetry();
const router = useRouter();
const workflowHelpers = useWorkflowHelpers({ router });
const settingsStore = useSettingsStore();
// -1 means the backend chooses the default
// 0 is the old flow
// 1 is the new flow
@ -1188,6 +1197,33 @@ export const useWorkflowsStore = defineStore(STORES.WORKFLOWS, () => {
}
}
async function trackNodeExecution(pushData: PushPayload<'nodeExecuteAfter'>): Promise<void> {
const nodeName = pushData.nodeName;
if (pushData.data.error) {
const node = getNodeByName(nodeName);
telemetry.track(
'Manual exec errored',
{
error_title: pushData.data.error.message,
node_type: node?.type,
node_type_version: node?.typeVersion,
node_id: node?.id,
node_graph_string: JSON.stringify(
TelemetryHelpers.generateNodesGraph(
await workflowHelpers.getWorkflowDataToSave(),
workflowHelpers.getNodeTypes(),
{
isCloudDeployment: settingsStore.isCloudDeployment,
},
).nodeGraph,
),
},
{ withPostHog: true },
);
}
}
function addNodeExecutionData(pushData: PushPayload<'nodeExecuteAfter'>): void {
if (!workflowExecutionData.value?.data) {
throw new Error('The "workflowExecutionData" is not initialized!');
@ -1209,6 +1245,8 @@ export const useWorkflowsStore = defineStore(STORES.WORKFLOWS, () => {
};
}
workflowExecutionData.value.data!.resultData.runData[pushData.nodeName].push(pushData.data);
void trackNodeExecution(pushData);
}
function clearNodeExecutionData(nodeName: string): void {

View file

@ -15,6 +15,7 @@ vi.mock('vue-router', () => {
push,
}),
RouterLink: vi.fn(),
useRoute: vi.fn(),
};
});

View file

@ -2482,6 +2482,8 @@ export interface INodeGraphItem {
toolSettings?: IDataObject; //various langchain tool's settings
sql?: string; //merge node combineBySql, cloud only
workflow_id?: string; //@n8n/n8n-nodes-langchain.toolWorkflow and n8n-nodes-base.executeWorkflow
runs?: number;
items_total?: number;
}
export interface INodeNameIndex {

View file

@ -24,6 +24,8 @@ import type {
IWorkflowBase,
INodeTypes,
IDataObject,
IRunData,
ITaskData,
} from './Interfaces';
import { getNodeParameters } from './NodeHelpers';
@ -131,6 +133,21 @@ export function getDomainPath(raw: string, urlParts = URL_PARTS_REGEX): string {
}
}
function getNumberOfItemsInRuns(runs: ITaskData[]): number {
return runs.reduce((total, run) => {
const data = run.data ?? {};
let count = 0;
Object.keys(data).forEach((type) => {
const conn = data[type] ?? [];
conn.forEach((branch) => {
count += (branch ?? []).length;
});
});
return total + count;
}, 0);
}
export function generateNodesGraph(
workflow: Partial<IWorkflowBase>,
nodeTypes: INodeTypes,
@ -138,8 +155,10 @@ export function generateNodesGraph(
sourceInstanceId?: string;
nodeIdMap?: { [curr: string]: string };
isCloudDeployment?: boolean;
runData?: IRunData;
},
): INodesGraphResult {
const { runData } = options ?? {};
const nodeGraph: INodesGraph = {
node_types: [],
node_connections: [],
@ -200,6 +219,13 @@ export function generateNodesGraph(
position: node.position,
};
if (runData?.[node.name]) {
const runs = runData[node.name] ?? [];
nodeItem.runs = runs.length;
nodeItem.items_total = getNumberOfItemsInRuns(runs);
}
if (options?.sourceInstanceId) {
nodeItem.src_instance_id = options.sourceInstanceId;
}

View file

@ -3,6 +3,7 @@ import { v5 as uuidv5, v3 as uuidv3, v4 as uuidv4, v1 as uuidv1 } from 'uuid';
import { STICKY_NODE_TYPE } from '@/Constants';
import { ApplicationError } from '@/errors';
import type { IRunData } from '@/Interfaces';
import { NodeConnectionType, type IWorkflowBase } from '@/Interfaces';
import * as nodeHelpers from '@/NodeHelpers';
import {
@ -780,6 +781,108 @@ describe('generateNodesGraph', () => {
expect(() => generateNodesGraph(workflow, nodeTypes)).not.toThrow();
});
test('should add run and items count', () => {
const { workflow, runData } = generateTestWorkflowAndRunData();
expect(generateNodesGraph(workflow, nodeTypes, { runData })).toEqual({
nameIndices: {
DebugHelper: '4',
'Edit Fields': '1',
'Edit Fields1': '2',
'Edit Fields2': '3',
'Execute Workflow Trigger': '0',
Switch: '5',
},
nodeGraph: {
is_pinned: false,
node_connections: [
{
end: '1',
start: '0',
},
{
end: '4',
start: '0',
},
{
end: '5',
start: '1',
},
{
end: '1',
start: '4',
},
{
end: '2',
start: '5',
},
{
end: '3',
start: '5',
},
],
node_types: [
'n8n-nodes-base.executeWorkflowTrigger',
'n8n-nodes-base.set',
'n8n-nodes-base.set',
'n8n-nodes-base.set',
'n8n-nodes-base.debugHelper',
'n8n-nodes-base.switch',
],
nodes: {
'0': {
id: 'a2372c14-87de-42de-9f9e-1c499aa2c279',
items_total: 1,
position: [1000, 240],
runs: 1,
type: 'n8n-nodes-base.executeWorkflowTrigger',
version: 1,
},
'1': {
id: '0f7aa00e-248c-452c-8cd0-62cb55941633',
items_total: 4,
position: [1460, 640],
runs: 2,
type: 'n8n-nodes-base.set',
version: 3.1,
},
'2': {
id: '9165c185-9f1c-4ec1-87bf-76ca66dfae38',
items_total: 4,
position: [1860, 260],
runs: 2,
type: 'n8n-nodes-base.set',
version: 3.4,
},
'3': {
id: '7a915fd5-5987-4ff1-9509-06b24a0a4613',
position: [1940, 680],
type: 'n8n-nodes-base.set',
version: 3.4,
},
'4': {
id: '63050e7c-8ad5-4f44-8fdd-da555e40471b',
items_total: 3,
position: [1220, 240],
runs: 1,
type: 'n8n-nodes-base.debugHelper',
version: 1,
},
'5': {
id: 'fbf7525d-2d1d-4dcf-97a0-43b53d087ef3',
items_total: 4,
position: [1680, 640],
runs: 2,
type: 'n8n-nodes-base.switch',
version: 3.2,
},
},
notes: {},
},
webhookNodeNames: [],
});
});
});
function validUrls(idMaker: typeof alphanumericId | typeof email, char = CHAR) {
@ -886,3 +989,293 @@ function alphanumericId() {
}
const chooseRandomly = <T>(array: T[]) => array[randomInt(array.length)];
function generateTestWorkflowAndRunData(): { workflow: IWorkflowBase; runData: IRunData } {
const workflow: IWorkflowBase = {
meta: {
instanceId: 'a786b722078489c1fa382391a9f3476c2784761624deb2dfb4634827256d51a0',
},
nodes: [
{
parameters: {},
id: 'a2372c14-87de-42de-9f9e-1c499aa2c279',
name: 'Execute Workflow Trigger',
type: 'n8n-nodes-base.executeWorkflowTrigger',
typeVersion: 1,
position: [1000, 240],
},
{
parameters: {
options: {},
},
id: '0f7aa00e-248c-452c-8cd0-62cb55941633',
name: 'Edit Fields',
type: 'n8n-nodes-base.set',
typeVersion: 3.1,
position: [1460, 640],
},
{
parameters: {
options: {},
},
id: '9165c185-9f1c-4ec1-87bf-76ca66dfae38',
name: 'Edit Fields1',
type: 'n8n-nodes-base.set',
typeVersion: 3.4,
position: [1860, 260],
},
{
parameters: {
options: {},
},
id: '7a915fd5-5987-4ff1-9509-06b24a0a4613',
name: 'Edit Fields2',
type: 'n8n-nodes-base.set',
typeVersion: 3.4,
position: [1940, 680],
},
{
parameters: {
category: 'randomData',
randomDataSeed: '0',
randomDataCount: 3,
},
id: '63050e7c-8ad5-4f44-8fdd-da555e40471b',
name: 'DebugHelper',
type: 'n8n-nodes-base.debugHelper',
typeVersion: 1,
position: [1220, 240],
},
{
id: 'fbf7525d-2d1d-4dcf-97a0-43b53d087ef3',
name: 'Switch',
type: 'n8n-nodes-base.switch',
typeVersion: 3.2,
position: [1680, 640],
parameters: {},
},
],
connections: {
'Execute Workflow Trigger': {
main: [
[
{
node: 'Edit Fields',
type: 'main' as NodeConnectionType,
index: 0,
},
{
node: 'DebugHelper',
type: 'main' as NodeConnectionType,
index: 0,
},
],
],
},
'Edit Fields': {
main: [
[
{
node: 'Switch',
type: 'main' as NodeConnectionType,
index: 0,
},
],
],
},
DebugHelper: {
main: [
[
{
node: 'Edit Fields',
type: 'main' as NodeConnectionType,
index: 0,
},
],
],
},
Switch: {
main: [
// @ts-ignore
null,
// @ts-ignore
null,
[
{
node: 'Edit Fields1',
type: 'main' as NodeConnectionType,
index: 0,
},
],
[
{
node: 'Edit Fields2',
type: 'main' as NodeConnectionType,
index: 0,
},
],
],
},
},
pinData: {},
};
const runData: IRunData = {
'Execute Workflow Trigger': [
{
hints: [],
startTime: 1727793340927,
executionTime: 0,
source: [],
executionStatus: 'success',
data: { main: [[{ json: {}, pairedItem: { item: 0 } }]] },
},
],
DebugHelper: [
{
hints: [],
startTime: 1727793340928,
executionTime: 0,
source: [{ previousNode: 'Execute Workflow Trigger' }],
executionStatus: 'success',
data: {
main: [
[
{
json: {
test: 'abc',
},
pairedItem: { item: 0 },
},
{
json: {
test: 'abc',
},
pairedItem: { item: 0 },
},
{
json: {
test: 'abc',
},
pairedItem: { item: 0 },
},
],
],
},
},
],
'Edit Fields': [
{
hints: [],
startTime: 1727793340928,
executionTime: 1,
source: [{ previousNode: 'DebugHelper' }],
executionStatus: 'success',
data: {
main: [
[
{
json: {
test: 'abc',
},
pairedItem: { item: 0 },
},
{
json: {
test: 'abc',
},
pairedItem: { item: 1 },
},
{
json: {
test: 'abc',
},
pairedItem: { item: 2 },
},
],
],
},
},
{
hints: [],
startTime: 1727793340931,
executionTime: 0,
source: [{ previousNode: 'Execute Workflow Trigger' }],
executionStatus: 'success',
data: { main: [[{ json: {}, pairedItem: { item: 0 } }]] },
},
],
Switch: [
{
hints: [],
startTime: 1727793340929,
executionTime: 1,
source: [{ previousNode: 'Edit Fields' }],
executionStatus: 'success',
data: {
main: [
[],
[],
[
{
json: {
test: 'abc',
},
pairedItem: { item: 0 },
},
{
json: {
test: 'abc',
},
pairedItem: { item: 1 },
},
{
json: {
test: 'abc',
},
pairedItem: { item: 2 },
},
],
[],
],
},
},
{
hints: [],
startTime: 1727793340931,
executionTime: 0,
source: [{ previousNode: 'Edit Fields', previousNodeRun: 1 }],
executionStatus: 'success',
data: { main: [[], [], [{ json: {}, pairedItem: { item: 0 } }], []] },
},
],
'Edit Fields1': [
{
hints: [],
startTime: 1727793340930,
executionTime: 0,
source: [{ previousNode: 'Switch', previousNodeOutput: 2 }],
executionStatus: 'success',
data: {
main: [
[
{ json: {}, pairedItem: { item: 0 } },
{ json: {}, pairedItem: { item: 1 } },
{ json: {}, pairedItem: { item: 2 } },
],
],
},
},
{
hints: [],
startTime: 1727793340932,
executionTime: 1,
source: [{ previousNode: 'Switch', previousNodeOutput: 2, previousNodeRun: 1 }],
executionStatus: 'success',
data: { main: [[{ json: {}, pairedItem: { item: 0 } }]] },
},
],
};
return { workflow, runData };
}