chore: Add basic telemetry events to workflow evaluation (no-changelog) (#12890)

Co-authored-by: Oleg Ivaniv <me@olegivaniv.com>

packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts

@@ -18,6 +18,7 @@ import type { TestRunRepository } from '@/databases/repositories/test-run.reposi
 import type { WorkflowRepository } from '@/databases/repositories/workflow.repository';
 import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials';
 import { NodeTypes } from '@/node-types';
+import type { Telemetry } from '@/telemetry';
 import type { WorkflowRunner } from '@/workflow-runner';
 import { mockInstance, mockLogger } from '@test/mocking';
 import { mockNodeTypesData } from '@test-integration/utils/node-types-data';
@@ -131,6 +132,7 @@ function mockEvaluationExecutionData(metrics: Record<string, GenericValue>) {
 
 const errorReporter = mock<ErrorReporter>();
 const logger = mockLogger();
+const telemetry = mock<Telemetry>();
 
 async function mockLongExecutionPromise(data: IRun, delay: number): Promise<IRun> {
 	return await new Promise((resolve) => {
@@ -182,6 +184,7 @@ describe('TestRunnerService', () => {
 	test('should create an instance of TestRunnerService', async () => {
 		const testRunnerService = new TestRunnerService(
 			logger,
+			telemetry,
 			workflowRepository,
 			workflowRunner,
 			executionRepository,
@@ -198,6 +201,7 @@ describe('TestRunnerService', () => {
 	test('should create and run test cases from past executions', async () => {
 		const testRunnerService = new TestRunnerService(
 			logger,
+			telemetry,
 			workflowRepository,
 			workflowRunner,
 			executionRepository,
@@ -237,6 +241,7 @@ describe('TestRunnerService', () => {
 	test('should run both workflow under test and evaluation workflow', async () => {
 		const testRunnerService = new TestRunnerService(
 			logger,
+			telemetry,
 			workflowRepository,
 			workflowRunner,
 			executionRepository,
@@ -339,6 +344,7 @@ describe('TestRunnerService', () => {
 	test('should properly count passed and failed executions', async () => {
 		const testRunnerService = new TestRunnerService(
 			logger,
+			telemetry,
 			workflowRepository,
 			workflowRunner,
 			executionRepository,
@@ -398,6 +404,7 @@ describe('TestRunnerService', () => {
 	test('should properly count failed test executions', async () => {
 		const testRunnerService = new TestRunnerService(
 			logger,
+			telemetry,
 			workflowRepository,
 			workflowRunner,
 			executionRepository,
@@ -453,6 +460,7 @@ describe('TestRunnerService', () => {
 	test('should properly count failed evaluations', async () => {
 		const testRunnerService = new TestRunnerService(
 			logger,
+			telemetry,
 			workflowRepository,
 			workflowRunner,
 			executionRepository,
@@ -512,6 +520,7 @@ describe('TestRunnerService', () => {
 	test('should specify correct start nodes when running workflow under test', async () => {
 		const testRunnerService = new TestRunnerService(
 			logger,
+			telemetry,
 			workflowRepository,
 			workflowRunner,
 			executionRepository,
@@ -587,6 +596,7 @@ describe('TestRunnerService', () => {
 	test('should properly choose trigger and start nodes', async () => {
 		const testRunnerService = new TestRunnerService(
 			logger,
+			telemetry,
 			workflowRepository,
 			workflowRunner,
 			executionRepository,
@@ -613,6 +623,7 @@ describe('TestRunnerService', () => {
 	test('should properly choose trigger and start nodes 2', async () => {
 		const testRunnerService = new TestRunnerService(
 			logger,
+			telemetry,
 			workflowRepository,
 			workflowRunner,
 			executionRepository,
@@ -644,6 +655,7 @@ describe('TestRunnerService', () => {
 		test('should cancel test run', async () => {
 			const testRunnerService = new TestRunnerService(
 				logger,
+				telemetry,
 				workflowRepository,
 				workflowRunner,
 				executionRepository,

packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts

@@ -23,6 +23,7 @@ import { TestMetricRepository } from '@/databases/repositories/test-metric.repos
 import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee';
 import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
 import { NodeTypes } from '@/node-types';
+import { Telemetry } from '@/telemetry';
 import { getRunData } from '@/workflow-execute-additional-data';
 import { WorkflowRunner } from '@/workflow-runner';
 
@@ -43,6 +44,7 @@ export class TestRunnerService {
 
 	constructor(
 		private readonly logger: Logger,
+		private readonly telemetry: Telemetry,
 		private readonly workflowRepository: WorkflowRepository,
 		private readonly workflowRunner: WorkflowRunner,
 		private readonly executionRepository: ExecutionRepository,
@@ -268,12 +270,22 @@ export class TestRunnerService {
 			const testMetricNames = await this.getTestMetricNames(test.id);
 
 			// 2. Run over all the test cases
+			const pastExecutionIds = pastExecutions.map((e) => e.id);
+
 			await this.testRunRepository.markAsRunning(testRun.id, pastExecutions.length);
+			this.telemetry.track('User runs test', {
+				user_id: user.id,
+				test_id: test.id,
+				run_id: testRun.id,
+				executions_ids: pastExecutionIds,
+				workflow_id: test.workflowId,
+				evaluation_workflow_id: test.evaluationWorkflowId,
+			});
 
 			// Object to collect the results of the evaluation workflow executions
 			const metrics = new EvaluationMetrics(testMetricNames);
 
-			for (const { id: pastExecutionId } of pastExecutions) {
+			for (const pastExecutionId of pastExecutionIds) {
 				if (abortSignal.aborted) {
 					this.logger.debug('Test run was cancelled', {
 						testId: test.id,

packages/editor-ui/src/views/TestDefinition/TestDefinitionEditView.vue

@@ -14,6 +14,8 @@ import type { TestMetricRecord, TestRunRecord } from '@/api/testDefinition.ee';
 import { useUIStore } from '@/stores/ui.store';
 import { useTestDefinitionStore } from '@/stores/testDefinition.store.ee';
 import ConfigSection from '@/components/TestDefinition/EditDefinition/sections/ConfigSection.vue';
+import { useTelemetry } from '@/composables/useTelemetry';
+import { useRootStore } from '@/stores/root.store';
 import { useExecutionsStore } from '@/stores/executions.store';
 import { useWorkflowsStore } from '@/stores/workflows.store';
 import type { IPinData } from 'n8n-workflow';
@@ -30,6 +32,7 @@ const toast = useToast();
 const testDefinitionStore = useTestDefinitionStore();
 const tagsStore = useAnnotationTagsStore();
 const uiStore = useUIStore();
+const telemetry = useTelemetry();
 const executionsStore = useExecutionsStore();
 const workflowStore = useWorkflowsStore();
 
@@ -97,6 +100,18 @@ async function onSaveTest() {
 				name: VIEWS.TEST_DEFINITION_EDIT,
 				params: { testId: savedTest.id },
 			});
+
+			telemetry.track(
+				'User created test',
+				{
+					test_id: savedTest.id,
+					workflow_id: currentWorkflowId.value,
+					session_id: useRootStore().pushRef,
+				},
+				{
+					withPostHog: true,
+				},
+			);
 		}
 	} catch (e: unknown) {
 		toast.showError(e, locale.baseText('testDefinition.edit.testSaveFailed'));