diff --git a/packages/cli/src/evaluation.ee/metrics.controller.ts b/packages/cli/src/evaluation.ee/metrics.controller.ts index 2072b978b1..5d27931166 100644 --- a/packages/cli/src/evaluation.ee/metrics.controller.ts +++ b/packages/cli/src/evaluation.ee/metrics.controller.ts @@ -8,6 +8,7 @@ import { testMetricPatchRequestBodySchema, } from '@/evaluation.ee/metric.schema'; import { getSharedWorkflowIds } from '@/public-api/v1/handlers/workflows/workflows.service'; +import { Telemetry } from '@/telemetry'; import { TestDefinitionService } from './test-definition.service.ee'; import { TestMetricsRequest } from './test-definitions.types.ee'; @@ -17,6 +18,7 @@ export class TestMetricsController { constructor( private readonly testDefinitionService: TestDefinitionService, private readonly testMetricRepository: TestMetricRepository, + private readonly telemetry: Telemetry, ) {} // This method is used in multiple places in the controller to get the test definition @@ -105,7 +107,16 @@ export class TestMetricsController { if (!metric) throw new NotFoundError('Metric not found'); - await this.testMetricRepository.update(metricId, bodyParseResult.data); + const updateResult = await this.testMetricRepository.update(metricId, bodyParseResult.data); + + // Send telemetry event if the metric was updated + if (updateResult.affected === 1 && metric.name !== bodyParseResult.data.name) { + this.telemetry.track('User added metrics to test', { + metric_id: metricId, + metric_name: bodyParseResult.data.name, + test_id: testDefinitionId, + }); + } // Respond with the updated metric return await this.testMetricRepository.findOneBy({ id: metricId }); diff --git a/packages/cli/src/evaluation.ee/test-definition.service.ee.ts b/packages/cli/src/evaluation.ee/test-definition.service.ee.ts index 682223e3b2..5129f3b389 100644 --- a/packages/cli/src/evaluation.ee/test-definition.service.ee.ts +++ b/packages/cli/src/evaluation.ee/test-definition.service.ee.ts @@ -7,6 +7,7 @@ import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { validateEntity } from '@/generic-helpers'; import type { ListQuery } from '@/requests'; +import { Telemetry } from '@/telemetry'; type TestDefinitionLike = Omit< Partial, @@ -22,6 +23,7 @@ export class TestDefinitionService { constructor( private testDefinitionRepository: TestDefinitionRepository, private annotationTagRepository: AnnotationTagRepository, + private telemetry: Telemetry, ) {} private toEntityLike(attrs: { @@ -94,6 +96,13 @@ export class TestDefinitionService { } async update(id: string, attrs: TestDefinitionLike) { + const existingTestDefinition = await this.testDefinitionRepository.findOneOrFail({ + where: { + id, + }, + relations: ['workflow'], + }); + if (attrs.name) { const updatedTest = this.toEntity(attrs); await validateEntity(updatedTest); @@ -114,13 +123,6 @@ export class TestDefinitionService { // If there are mocked nodes, validate them if (attrs.mockedNodes && attrs.mockedNodes.length > 0) { - const existingTestDefinition = await this.testDefinitionRepository.findOneOrFail({ - where: { - id, - }, - relations: ['workflow'], - }); - const existingNodeNames = new Map( existingTestDefinition.workflow.nodes.map((n) => [n.name, n]), ); @@ -146,6 +148,24 @@ export class TestDefinitionService { if (queryResult.affected === 0) { throw new NotFoundError('Test definition not found'); } + + // Send the telemetry events + if (attrs.annotationTagId && attrs.annotationTagId !== existingTestDefinition.annotationTagId) { + this.telemetry.track('User added tag to test', { + test_id: id, + tag_id: attrs.annotationTagId, + }); + } + + if ( + attrs.evaluationWorkflowId && + existingTestDefinition.evaluationWorkflowId !== attrs.evaluationWorkflowId + ) { + this.telemetry.track('User added evaluation workflow to test', { + test_id: id, + subworkflow_id: attrs.evaluationWorkflowId, + }); + } } async delete(id: string, accessibleWorkflowIds: string[]) { @@ -154,6 +174,8 @@ export class TestDefinitionService { if (deleteResult.affected === 0) { throw new NotFoundError('Test definition not found'); } + + this.telemetry.track('User deleted a test', { test_id: id }); } async getMany(options: ListQuery.Options, accessibleWorkflowIds: string[] = []) { diff --git a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts index 2c6c45ba68..433c86cbcf 100644 --- a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts +++ b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts @@ -292,6 +292,8 @@ export class TestRunnerService { userId: user.id, }; + let testRunEndStatusForTelemetry; + const abortSignal = abortController.signal; try { // Get the evaluation workflow @@ -338,7 +340,7 @@ export class TestRunnerService { // Update test run status await this.testRunRepository.markAsRunning(testRun.id, pastExecutions.length); - this.telemetry.track('User runs test', { + this.telemetry.track('User ran test', { user_id: user.id, test_id: test.id, run_id: testRun.id, @@ -504,13 +506,17 @@ export class TestRunnerService { await Db.transaction(async (trx) => { await this.testRunRepository.markAsCancelled(testRun.id, trx); await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx); + + testRunEndStatusForTelemetry = 'cancelled'; }); } else { const aggregatedMetrics = metrics.getAggregatedMetrics(); await this.testRunRepository.markAsCompleted(testRun.id, aggregatedMetrics); - this.logger.debug('Test run finished', { testId: test.id }); + this.logger.debug('Test run finished', { testId: test.id, testRunId: testRun.id }); + + testRunEndStatusForTelemetry = 'completed'; } } catch (e) { if (e instanceof ExecutionCancelledError) { @@ -523,15 +529,26 @@ export class TestRunnerService { await this.testRunRepository.markAsCancelled(testRun.id, trx); await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx); }); + + testRunEndStatusForTelemetry = 'cancelled'; } else if (e instanceof TestRunError) { await this.testRunRepository.markAsError(testRun.id, e.code, e.extra as IDataObject); + testRunEndStatusForTelemetry = 'error'; } else { await this.testRunRepository.markAsError(testRun.id, 'UNKNOWN_ERROR'); + testRunEndStatusForTelemetry = 'error'; throw e; } } finally { // Clean up abort controller this.abortControllers.delete(testRun.id); + + // Send telemetry event + this.telemetry.track('Test run finished', { + test_id: test.id, + run_id: testRun.id, + status: testRunEndStatusForTelemetry, + }); } } diff --git a/packages/cli/src/evaluation.ee/test-runs.controller.ee.ts b/packages/cli/src/evaluation.ee/test-runs.controller.ee.ts index 00f4ead191..333cbb9fae 100644 --- a/packages/cli/src/evaluation.ee/test-runs.controller.ee.ts +++ b/packages/cli/src/evaluation.ee/test-runs.controller.ee.ts @@ -11,6 +11,7 @@ import { TestRunsRequest } from '@/evaluation.ee/test-definitions.types.ee'; import { TestRunnerService } from '@/evaluation.ee/test-runner/test-runner.service.ee'; import { listQueryMiddleware } from '@/middlewares'; import { getSharedWorkflowIds } from '@/public-api/v1/handlers/workflows/workflows.service'; +import { Telemetry } from '@/telemetry'; import { TestDefinitionService } from './test-definition.service.ee'; @@ -22,6 +23,7 @@ export class TestRunsController { private readonly testCaseExecutionRepository: TestCaseExecutionRepository, private readonly testRunnerService: TestRunnerService, private readonly instanceSettings: InstanceSettings, + private readonly telemetry: Telemetry, ) {} /** @@ -92,7 +94,7 @@ export class TestRunsController { @Delete('/:testDefinitionId/runs/:id') async delete(req: TestRunsRequest.Delete) { - const { id: testRunId } = req.params; + const { id: testRunId, testDefinitionId } = req.params; // Check test definition and test run exist await this.getTestDefinition(req); @@ -100,6 +102,8 @@ export class TestRunsController { await this.testRunRepository.delete({ id: testRunId }); + this.telemetry.track('User deleted a run', { run_id: testRunId, test_id: testDefinitionId }); + return { success: true }; } diff --git a/packages/editor-ui/src/components/TestDefinition/EditDefinition/NodesPinning.vue b/packages/editor-ui/src/components/TestDefinition/EditDefinition/NodesPinning.vue index 2e2c2a6d9c..c92932249c 100644 --- a/packages/editor-ui/src/components/TestDefinition/EditDefinition/NodesPinning.vue +++ b/packages/editor-ui/src/components/TestDefinition/EditDefinition/NodesPinning.vue @@ -9,12 +9,14 @@ import { createEventBus, N8nTooltip } from 'n8n-design-system'; import type { CanvasConnectionPort, CanvasEventBusEvents, CanvasNodeData } from '@/types'; import { useVueFlow } from '@vue-flow/core'; import { useI18n } from '@/composables/useI18n'; +import { useTelemetry } from '@/composables/useTelemetry'; const workflowsStore = useWorkflowsStore(); const nodeTypesStore = useNodeTypesStore(); const route = useRoute(); const router = useRouter(); const locale = useI18n(); +const telemetry = useTelemetry(); const { resetWorkspace, initializeWorkspace } = useCanvasOperations({ router }); @@ -101,6 +103,13 @@ function onPinButtonClick(data: CanvasNodeData) { emit('update:modelValue', updatedNodes); updateNodeClasses([data.id], !isPinned); + + if (!isPinned) { + telemetry.track('User selected node to be mocked', { + node_id: data.id, + test_id: testId.value, + }); + } } function isPinButtonVisible(outputs: CanvasConnectionPort[]) { return outputs.length === 1; diff --git a/packages/editor-ui/src/components/TestDefinition/EditDefinition/WorkflowSelector.vue b/packages/editor-ui/src/components/TestDefinition/EditDefinition/WorkflowSelector.vue index 45b1080fa2..84757c7672 100644 --- a/packages/editor-ui/src/components/TestDefinition/EditDefinition/WorkflowSelector.vue +++ b/packages/editor-ui/src/components/TestDefinition/EditDefinition/WorkflowSelector.vue @@ -21,7 +21,10 @@ const props = withDefaults(defineProps(), { sampleWorkflowName: undefined, }); -defineEmits<{ 'update:modelValue': [value: WorkflowSelectorProps['modelValue']] }>(); +defineEmits<{ + 'update:modelValue': [value: WorkflowSelectorProps['modelValue']]; + workflowCreated: [workflowId: string]; +}>(); const locale = useI18n(); const subworkflowName = computed(() => { @@ -63,6 +66,7 @@ const sampleWorkflow = computed(() => { allow-new :sample-workflow="sampleWorkflow" @update:model-value="$emit('update:modelValue', $event)" + @workflow-created="$emit('workflowCreated', $event)" /> diff --git a/packages/editor-ui/src/components/TestDefinition/EditDefinition/sections/ConfigSection.vue b/packages/editor-ui/src/components/TestDefinition/EditDefinition/sections/ConfigSection.vue index 36c9fefe3c..3591ad7831 100644 --- a/packages/editor-ui/src/components/TestDefinition/EditDefinition/sections/ConfigSection.vue +++ b/packages/editor-ui/src/components/TestDefinition/EditDefinition/sections/ConfigSection.vue @@ -29,6 +29,7 @@ defineProps<{ const emit = defineEmits<{ openPinningModal: []; deleteMetric: [metric: Partial]; + evaluationWorkflowCreated: [workflowId: string]; }>(); const locale = useI18n(); @@ -145,6 +146,7 @@ function showFieldIssues(fieldKey: string) { :class="{ 'has-issues': getFieldIssues('evaluationWorkflow').length > 0 }" :sample-workflow-name="sampleWorkflowName" @update:model-value="updateChangedFieldsKeys('evaluationWorkflow')" + @workflow-created="$emit('evaluationWorkflowCreated', $event)" /> diff --git a/packages/editor-ui/src/components/WorkflowSelectorParameterInput/WorkflowSelectorParameterInput.vue b/packages/editor-ui/src/components/WorkflowSelectorParameterInput/WorkflowSelectorParameterInput.vue index 686adcafc6..85e68c9f95 100644 --- a/packages/editor-ui/src/components/WorkflowSelectorParameterInput/WorkflowSelectorParameterInput.vue +++ b/packages/editor-ui/src/components/WorkflowSelectorParameterInput/WorkflowSelectorParameterInput.vue @@ -55,6 +55,7 @@ const emit = defineEmits<{ modalOpenerClick: []; focus: []; blur: []; + workflowCreated: [workflowId: string]; }>(); const workflowsStore = useWorkflowsStore(); @@ -232,6 +233,8 @@ const onAddResourceClicked = async () => { hideDropdown(); window.open(href, '_blank'); + + emit('workflowCreated', newWorkflow.id); };