feat(core): Add telemetry for workflow evaluation feature (no-changelog) (#13367)

packages/cli/src/evaluation.ee/metrics.controller.ts

@@ -8,6 +8,7 @@ import {
 	testMetricPatchRequestBodySchema,
 } from '@/evaluation.ee/metric.schema';
 import { getSharedWorkflowIds } from '@/public-api/v1/handlers/workflows/workflows.service';
+import { Telemetry } from '@/telemetry';
 
 import { TestDefinitionService } from './test-definition.service.ee';
 import { TestMetricsRequest } from './test-definitions.types.ee';
@@ -17,6 +18,7 @@ export class TestMetricsController {
 	constructor(
 		private readonly testDefinitionService: TestDefinitionService,
 		private readonly testMetricRepository: TestMetricRepository,
+		private readonly telemetry: Telemetry,
 	) {}
 
 	// This method is used in multiple places in the controller to get the test definition
@@ -105,7 +107,16 @@ export class TestMetricsController {
 
 		if (!metric) throw new NotFoundError('Metric not found');
 
-		await this.testMetricRepository.update(metricId, bodyParseResult.data);
+		const updateResult = await this.testMetricRepository.update(metricId, bodyParseResult.data);
+
+		// Send telemetry event if the metric was updated
+		if (updateResult.affected === 1 && metric.name !== bodyParseResult.data.name) {
+			this.telemetry.track('User added metrics to test', {
+				metric_id: metricId,
+				metric_name: bodyParseResult.data.name,
+				test_id: testDefinitionId,
+			});
+		}
 
 		// Respond with the updated metric
 		return await this.testMetricRepository.findOneBy({ id: metricId });

packages/cli/src/evaluation.ee/test-definition.service.ee.ts

@@ -7,6 +7,7 @@ import { BadRequestError } from '@/errors/response-errors/bad-request.error';
 import { NotFoundError } from '@/errors/response-errors/not-found.error';
 import { validateEntity } from '@/generic-helpers';
 import type { ListQuery } from '@/requests';
+import { Telemetry } from '@/telemetry';
 
 type TestDefinitionLike = Omit<
 	Partial<TestDefinition>,
@@ -22,6 +23,7 @@ export class TestDefinitionService {
 	constructor(
 		private testDefinitionRepository: TestDefinitionRepository,
 		private annotationTagRepository: AnnotationTagRepository,
+		private telemetry: Telemetry,
 	) {}
 
 	private toEntityLike(attrs: {
@@ -94,6 +96,13 @@ export class TestDefinitionService {
 	}
 
 	async update(id: string, attrs: TestDefinitionLike) {
+		const existingTestDefinition = await this.testDefinitionRepository.findOneOrFail({
+			where: {
+				id,
+			},
+			relations: ['workflow'],
+		});
+
 		if (attrs.name) {
 			const updatedTest = this.toEntity(attrs);
 			await validateEntity(updatedTest);
@@ -114,13 +123,6 @@ export class TestDefinitionService {
 
 		// If there are mocked nodes, validate them
 		if (attrs.mockedNodes && attrs.mockedNodes.length > 0) {
-			const existingTestDefinition = await this.testDefinitionRepository.findOneOrFail({
-				where: {
-					id,
-				},
-				relations: ['workflow'],
-			});
-
 			const existingNodeNames = new Map(
 				existingTestDefinition.workflow.nodes.map((n) => [n.name, n]),
 			);
@@ -146,6 +148,24 @@ export class TestDefinitionService {
 		if (queryResult.affected === 0) {
 			throw new NotFoundError('Test definition not found');
 		}
+
+		// Send the telemetry events
+		if (attrs.annotationTagId && attrs.annotationTagId !== existingTestDefinition.annotationTagId) {
+			this.telemetry.track('User added tag to test', {
+				test_id: id,
+				tag_id: attrs.annotationTagId,
+			});
+		}
+
+		if (
+			attrs.evaluationWorkflowId &&
+			existingTestDefinition.evaluationWorkflowId !== attrs.evaluationWorkflowId
+		) {
+			this.telemetry.track('User added evaluation workflow to test', {
+				test_id: id,
+				subworkflow_id: attrs.evaluationWorkflowId,
+			});
+		}
 	}
 
 	async delete(id: string, accessibleWorkflowIds: string[]) {
@@ -154,6 +174,8 @@ export class TestDefinitionService {
 		if (deleteResult.affected === 0) {
 			throw new NotFoundError('Test definition not found');
 		}
+
+		this.telemetry.track('User deleted a test', { test_id: id });
 	}
 
 	async getMany(options: ListQuery.Options, accessibleWorkflowIds: string[] = []) {

packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts

@@ -292,6 +292,8 @@ export class TestRunnerService {
 			userId: user.id,
 		};
 
+		let testRunEndStatusForTelemetry;
+
 		const abortSignal = abortController.signal;
 		try {
 			// Get the evaluation workflow
@@ -338,7 +340,7 @@ export class TestRunnerService {
 			// Update test run status
 			await this.testRunRepository.markAsRunning(testRun.id, pastExecutions.length);
 
-			this.telemetry.track('User runs test', {
+			this.telemetry.track('User ran test', {
 				user_id: user.id,
 				test_id: test.id,
 				run_id: testRun.id,
@@ -504,13 +506,17 @@ export class TestRunnerService {
 				await Db.transaction(async (trx) => {
 					await this.testRunRepository.markAsCancelled(testRun.id, trx);
 					await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx);
+
+					testRunEndStatusForTelemetry = 'cancelled';
 				});
 			} else {
 				const aggregatedMetrics = metrics.getAggregatedMetrics();
 
 				await this.testRunRepository.markAsCompleted(testRun.id, aggregatedMetrics);
 
-				this.logger.debug('Test run finished', { testId: test.id });
+				this.logger.debug('Test run finished', { testId: test.id, testRunId: testRun.id });
+
+				testRunEndStatusForTelemetry = 'completed';
 			}
 		} catch (e) {
 			if (e instanceof ExecutionCancelledError) {
@@ -523,15 +529,26 @@ export class TestRunnerService {
 					await this.testRunRepository.markAsCancelled(testRun.id, trx);
 					await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx);
 				});
+
+				testRunEndStatusForTelemetry = 'cancelled';
 			} else if (e instanceof TestRunError) {
 				await this.testRunRepository.markAsError(testRun.id, e.code, e.extra as IDataObject);
+				testRunEndStatusForTelemetry = 'error';
 			} else {
 				await this.testRunRepository.markAsError(testRun.id, 'UNKNOWN_ERROR');
+				testRunEndStatusForTelemetry = 'error';
 				throw e;
 			}
 		} finally {
 			// Clean up abort controller
 			this.abortControllers.delete(testRun.id);
+
+			// Send telemetry event
+			this.telemetry.track('Test run finished', {
+				test_id: test.id,
+				run_id: testRun.id,
+				status: testRunEndStatusForTelemetry,
+			});
 		}
 	}
 

packages/cli/src/evaluation.ee/test-runs.controller.ee.ts

@@ -11,6 +11,7 @@ import { TestRunsRequest } from '@/evaluation.ee/test-definitions.types.ee';
 import { TestRunnerService } from '@/evaluation.ee/test-runner/test-runner.service.ee';
 import { listQueryMiddleware } from '@/middlewares';
 import { getSharedWorkflowIds } from '@/public-api/v1/handlers/workflows/workflows.service';
+import { Telemetry } from '@/telemetry';
 
 import { TestDefinitionService } from './test-definition.service.ee';
 
@@ -22,6 +23,7 @@ export class TestRunsController {
 		private readonly testCaseExecutionRepository: TestCaseExecutionRepository,
 		private readonly testRunnerService: TestRunnerService,
 		private readonly instanceSettings: InstanceSettings,
+		private readonly telemetry: Telemetry,
 	) {}
 
 	/**
@@ -92,7 +94,7 @@ export class TestRunsController {
 
 	@Delete('/:testDefinitionId/runs/:id')
 	async delete(req: TestRunsRequest.Delete) {
-		const { id: testRunId } = req.params;
+		const { id: testRunId, testDefinitionId } = req.params;
 
 		// Check test definition and test run exist
 		await this.getTestDefinition(req);
@@ -100,6 +102,8 @@ export class TestRunsController {
 
 		await this.testRunRepository.delete({ id: testRunId });
 
+		this.telemetry.track('User deleted a run', { run_id: testRunId, test_id: testDefinitionId });
+
 		return { success: true };
 	}
 

packages/editor-ui/src/components/TestDefinition/EditDefinition/NodesPinning.vue

@@ -9,12 +9,14 @@ import { createEventBus, N8nTooltip } from 'n8n-design-system';
 import type { CanvasConnectionPort, CanvasEventBusEvents, CanvasNodeData } from '@/types';
 import { useVueFlow } from '@vue-flow/core';
 import { useI18n } from '@/composables/useI18n';
+import { useTelemetry } from '@/composables/useTelemetry';
 
 const workflowsStore = useWorkflowsStore();
 const nodeTypesStore = useNodeTypesStore();
 const route = useRoute();
 const router = useRouter();
 const locale = useI18n();
+const telemetry = useTelemetry();
 
 const { resetWorkspace, initializeWorkspace } = useCanvasOperations({ router });
 
@@ -101,6 +103,13 @@ function onPinButtonClick(data: CanvasNodeData) {
 
 	emit('update:modelValue', updatedNodes);
 	updateNodeClasses([data.id], !isPinned);
+
+	if (!isPinned) {
+		telemetry.track('User selected node to be mocked', {
+			node_id: data.id,
+			test_id: testId.value,
+		});
+	}
 }
 function isPinButtonVisible(outputs: CanvasConnectionPort[]) {
 	return outputs.length === 1;

packages/editor-ui/src/components/TestDefinition/EditDefinition/WorkflowSelector.vue

@@ -21,7 +21,10 @@ const props = withDefaults(defineProps<WorkflowSelectorProps>(), {
 	sampleWorkflowName: undefined,
 });
 
-defineEmits<{ 'update:modelValue': [value: WorkflowSelectorProps['modelValue']] }>();
+defineEmits<{
+	'update:modelValue': [value: WorkflowSelectorProps['modelValue']];
+	workflowCreated: [workflowId: string];
+}>();
 const locale = useI18n();
 
 const subworkflowName = computed(() => {
@@ -63,6 +66,7 @@ const sampleWorkflow = computed<IWorkflowDataCreate>(() => {
 				allow-new
 				:sample-workflow="sampleWorkflow"
 				@update:model-value="$emit('update:modelValue', $event)"
+				@workflow-created="$emit('workflowCreated', $event)"
 			/>
 		</n8n-input-label>
 	</div>

packages/editor-ui/src/components/TestDefinition/EditDefinition/sections/ConfigSection.vue

@@ -29,6 +29,7 @@ defineProps<{
 const emit = defineEmits<{
 	openPinningModal: [];
 	deleteMetric: [metric: Partial<TestMetricRecord>];
+	evaluationWorkflowCreated: [workflowId: string];
 }>();
 
 const locale = useI18n();
@@ -145,6 +146,7 @@ function showFieldIssues(fieldKey: string) {
 					:class="{ 'has-issues': getFieldIssues('evaluationWorkflow').length > 0 }"
 					:sample-workflow-name="sampleWorkflowName"
 					@update:model-value="updateChangedFieldsKeys('evaluationWorkflow')"
+					@workflow-created="$emit('evaluationWorkflowCreated', $event)"
 				/>
 			</template>
 		</EvaluationStep>

packages/editor-ui/src/components/WorkflowSelectorParameterInput/WorkflowSelectorParameterInput.vue

@@ -55,6 +55,7 @@ const emit = defineEmits<{
 	modalOpenerClick: [];
 	focus: [];
 	blur: [];
+	workflowCreated: [workflowId: string];
 }>();
 
 const workflowsStore = useWorkflowsStore();
@@ -232,6 +233,8 @@ const onAddResourceClicked = async () => {
 	hideDropdown();
 
 	window.open(href, '_blank');
+
+	emit('workflowCreated', newWorkflow.id);
 };
 </script>
 <template>

packages/editor-ui/src/components/executions/workflow/WorkflowExecutionAnnotationPanel.ee.vue

@@ -7,11 +7,13 @@ import { createEventBus } from 'n8n-design-system';
 import VoteButtons from '@/components/executions/workflow/VoteButtons.vue';
 import { useToast } from '@/composables/useToast';
 import { useI18n } from '@/composables/useI18n';
+import { useTelemetry } from '@/composables/useTelemetry';
 
 const executionsStore = useExecutionsStore();
 
 const { showError } = useToast();
 const i18n = useI18n();
+const telemetry = useTelemetry();
 
 const tagsEventBus = createEventBus();
 const isTagsEditEnabled = ref(false);
@@ -81,6 +83,13 @@ const onTagsBlur = async () => {
 
 	try {
 		await executionsStore.annotateExecution(activeExecution.value.id, { tags: newTagIds });
+
+		if (newTagIds.length > 0) {
+			telemetry.track('User added execution annotation tag', {
+				tag_ids: newTagIds,
+				execution_id: activeExecution.value.id,
+			});
+		}
 	} catch (e) {
 		showError(e, 'executionAnnotationView.tag.error');
 	}

packages/editor-ui/src/views/TestDefinition/TestDefinitionEditView.vue

@@ -186,7 +186,7 @@ async function getExamplePinnedDataForTags() {
 // Debounced watchers for auto-saving
 watch(
 	() => state.value.metrics,
-	debounce(async () => await updateMetrics(testId.value), { debounceTime: 400 }),
+	debounce(async () => await updateMetrics(testId.value), { debounceTime: 400, trailing: true }),
 	{ deep: true },
 );
 watch(() => state.value.tags.value, getExamplePinnedDataForTags);
@@ -198,9 +198,16 @@ watch(
 		state.value.evaluationWorkflow,
 		state.value.mockedNodes,
 	],
-	debounce(onSaveTest, { debounceTime: 400 }),
+	debounce(onSaveTest, { debounceTime: 400, trailing: true }),
 	{ deep: true },
 );
+
+function onEvaluationWorkflowCreated(workflowId: string) {
+	telemetry.track('User created evaluation workflow from test', {
+		test_id: testId.value,
+		subworkflow_id: workflowId,
+	});
+}
 </script>
 
 <template>
@@ -261,6 +268,7 @@ watch(
 				:sample-workflow-name="workflowName"
 				@open-pinning-modal="openPinningModal"
 				@delete-metric="onDeleteMetric"
+				@evaluation-workflow-created="onEvaluationWorkflowCreated($event)"
 			/>
 		</div>
 	</div>