diff --git a/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts b/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts index 026b5d2eb8..f82b09a8e1 100644 --- a/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts +++ b/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts @@ -18,6 +18,7 @@ import type { TestRunRepository } from '@/databases/repositories/test-run.reposi import type { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { NodeTypes } from '@/node-types'; +import type { Telemetry } from '@/telemetry'; import type { WorkflowRunner } from '@/workflow-runner'; import { mockInstance, mockLogger } from '@test/mocking'; import { mockNodeTypesData } from '@test-integration/utils/node-types-data'; @@ -131,6 +132,7 @@ function mockEvaluationExecutionData(metrics: Record) { const errorReporter = mock(); const logger = mockLogger(); +const telemetry = mock(); async function mockLongExecutionPromise(data: IRun, delay: number): Promise { return await new Promise((resolve) => { @@ -182,6 +184,7 @@ describe('TestRunnerService', () => { test('should create an instance of TestRunnerService', async () => { const testRunnerService = new TestRunnerService( logger, + telemetry, workflowRepository, workflowRunner, executionRepository, @@ -198,6 +201,7 @@ describe('TestRunnerService', () => { test('should create and run test cases from past executions', async () => { const testRunnerService = new TestRunnerService( logger, + telemetry, workflowRepository, workflowRunner, executionRepository, @@ -237,6 +241,7 @@ describe('TestRunnerService', () => { test('should run both workflow under test and evaluation workflow', async () => { const testRunnerService = new TestRunnerService( logger, + telemetry, workflowRepository, workflowRunner, executionRepository, @@ -339,6 +344,7 @@ describe('TestRunnerService', () => { test('should properly count passed and failed executions', async () => { const testRunnerService = new TestRunnerService( logger, + telemetry, workflowRepository, workflowRunner, executionRepository, @@ -398,6 +404,7 @@ describe('TestRunnerService', () => { test('should properly count failed test executions', async () => { const testRunnerService = new TestRunnerService( logger, + telemetry, workflowRepository, workflowRunner, executionRepository, @@ -453,6 +460,7 @@ describe('TestRunnerService', () => { test('should properly count failed evaluations', async () => { const testRunnerService = new TestRunnerService( logger, + telemetry, workflowRepository, workflowRunner, executionRepository, @@ -512,6 +520,7 @@ describe('TestRunnerService', () => { test('should specify correct start nodes when running workflow under test', async () => { const testRunnerService = new TestRunnerService( logger, + telemetry, workflowRepository, workflowRunner, executionRepository, @@ -587,6 +596,7 @@ describe('TestRunnerService', () => { test('should properly choose trigger and start nodes', async () => { const testRunnerService = new TestRunnerService( logger, + telemetry, workflowRepository, workflowRunner, executionRepository, @@ -613,6 +623,7 @@ describe('TestRunnerService', () => { test('should properly choose trigger and start nodes 2', async () => { const testRunnerService = new TestRunnerService( logger, + telemetry, workflowRepository, workflowRunner, executionRepository, @@ -644,6 +655,7 @@ describe('TestRunnerService', () => { test('should cancel test run', async () => { const testRunnerService = new TestRunnerService( logger, + telemetry, workflowRepository, workflowRunner, executionRepository, diff --git a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts index a594e15c05..f2928f0b91 100644 --- a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts +++ b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts @@ -23,6 +23,7 @@ import { TestMetricRepository } from '@/databases/repositories/test-metric.repos import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; import { NodeTypes } from '@/node-types'; +import { Telemetry } from '@/telemetry'; import { getRunData } from '@/workflow-execute-additional-data'; import { WorkflowRunner } from '@/workflow-runner'; @@ -43,6 +44,7 @@ export class TestRunnerService { constructor( private readonly logger: Logger, + private readonly telemetry: Telemetry, private readonly workflowRepository: WorkflowRepository, private readonly workflowRunner: WorkflowRunner, private readonly executionRepository: ExecutionRepository, @@ -268,12 +270,22 @@ export class TestRunnerService { const testMetricNames = await this.getTestMetricNames(test.id); // 2. Run over all the test cases + const pastExecutionIds = pastExecutions.map((e) => e.id); + await this.testRunRepository.markAsRunning(testRun.id, pastExecutions.length); + this.telemetry.track('User runs test', { + user_id: user.id, + test_id: test.id, + run_id: testRun.id, + executions_ids: pastExecutionIds, + workflow_id: test.workflowId, + evaluation_workflow_id: test.evaluationWorkflowId, + }); // Object to collect the results of the evaluation workflow executions const metrics = new EvaluationMetrics(testMetricNames); - for (const { id: pastExecutionId } of pastExecutions) { + for (const pastExecutionId of pastExecutionIds) { if (abortSignal.aborted) { this.logger.debug('Test run was cancelled', { testId: test.id, diff --git a/packages/editor-ui/src/views/TestDefinition/TestDefinitionEditView.vue b/packages/editor-ui/src/views/TestDefinition/TestDefinitionEditView.vue index 55d0b92297..95a49ea2aa 100644 --- a/packages/editor-ui/src/views/TestDefinition/TestDefinitionEditView.vue +++ b/packages/editor-ui/src/views/TestDefinition/TestDefinitionEditView.vue @@ -14,6 +14,8 @@ import type { TestMetricRecord, TestRunRecord } from '@/api/testDefinition.ee'; import { useUIStore } from '@/stores/ui.store'; import { useTestDefinitionStore } from '@/stores/testDefinition.store.ee'; import ConfigSection from '@/components/TestDefinition/EditDefinition/sections/ConfigSection.vue'; +import { useTelemetry } from '@/composables/useTelemetry'; +import { useRootStore } from '@/stores/root.store'; import { useExecutionsStore } from '@/stores/executions.store'; import { useWorkflowsStore } from '@/stores/workflows.store'; import type { IPinData } from 'n8n-workflow'; @@ -30,6 +32,7 @@ const toast = useToast(); const testDefinitionStore = useTestDefinitionStore(); const tagsStore = useAnnotationTagsStore(); const uiStore = useUIStore(); +const telemetry = useTelemetry(); const executionsStore = useExecutionsStore(); const workflowStore = useWorkflowsStore(); @@ -97,6 +100,18 @@ async function onSaveTest() { name: VIEWS.TEST_DEFINITION_EDIT, params: { testId: savedTest.id }, }); + + telemetry.track( + 'User created test', + { + test_id: savedTest.id, + workflow_id: currentWorkflowId.value, + session_id: useRootStore().pushRef, + }, + { + withPostHog: true, + }, + ); } } catch (e: unknown) { toast.showError(e, locale.baseText('testDefinition.edit.testSaveFailed'));