diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml index 90a69438ab..2df829f7c6 100644 --- a/.github/workflows/test-workflows.yml +++ b/.github/workflows/test-workflows.yml @@ -17,7 +17,9 @@ jobs: build: name: Install & Build runs-on: ubuntu-latest - if: github.event_name != 'pull_request_review' || startsWith(github.event.pull_request.base.ref, 'release/') + if: | + (github.event_name != 'pull_request_review' || startsWith(github.event.pull_request.base.ref, 'release/')) && + !contains(github.event.pull_request.labels.*.name, 'community') steps: - uses: actions/checkout@v4.1.1 - run: corepack enable diff --git a/cypress/composables/workflow.ts b/cypress/composables/workflow.ts index 29b871f560..e8e402dd5f 100644 --- a/cypress/composables/workflow.ts +++ b/cypress/composables/workflow.ts @@ -46,7 +46,14 @@ export function getNodes() { } export function getNodeByName(name: string) { - return cy.getByTestId('canvas-node').filter(`[data-name="${name}"]`).eq(0); + return cy.ifCanvasVersion( + () => cy.getByTestId('canvas-node').filter(`[data-name="${name}"]`).eq(0), + () => cy.getByTestId('canvas-node').filter(`[data-node-name="${name}"]`).eq(0), + ); +} + +export function getWorkflowHistoryCloseButton() { + return cy.getByTestId('workflow-history-close-button'); } export function disableNode(name: string) { diff --git a/cypress/e2e/27-cloud.cy.ts b/cypress/e2e/27-cloud.cy.ts index e9b814597d..bcf9750ecb 100644 --- a/cypress/e2e/27-cloud.cy.ts +++ b/cypress/e2e/27-cloud.cy.ts @@ -1,15 +1,17 @@ import planData from '../fixtures/Plan_data_opt_in_trial.json'; import { - BannerStack, MainSidebar, WorkflowPage, visitPublicApiPage, getPublicApiUpgradeCTA, + WorkflowsPage, } from '../pages'; +const NUMBER_OF_AI_CREDITS = 100; + const mainSidebar = new MainSidebar(); -const bannerStack = new BannerStack(); const workflowPage = new WorkflowPage(); +const workflowsPage = new WorkflowsPage(); describe('Cloud', () => { before(() => { @@ -22,6 +24,10 @@ describe('Cloud', () => { cy.overrideSettings({ deployment: { type: 'cloud' }, n8nMetadata: { userId: '1' }, + aiCredits: { + enabled: true, + credits: NUMBER_OF_AI_CREDITS, + }, }); cy.intercept('GET', '/rest/admin/cloud-plan', planData).as('getPlanData'); cy.intercept('GET', '/rest/cloud/proxy/user/me', {}).as('getCloudUserInfo'); @@ -40,11 +46,11 @@ describe('Cloud', () => { it('should render trial banner for opt-in cloud user', () => { visitWorkflowPage(); - bannerStack.getters.banner().should('be.visible'); + cy.getByTestId('banner-stack').should('be.visible'); mainSidebar.actions.signout(); - bannerStack.getters.banner().should('not.be.visible'); + cy.getByTestId('banner-stack').should('not.be.visible'); }); }); @@ -64,4 +70,66 @@ describe('Cloud', () => { getPublicApiUpgradeCTA().should('be.visible'); }); }); + + describe('Easy AI workflow experiment', () => { + it('should not show option to take you to the easy AI workflow if experiment is control', () => { + window.localStorage.setItem( + 'N8N_EXPERIMENT_OVERRIDES', + JSON.stringify({ '026_easy_ai_workflow': 'control' }), + ); + + cy.visit(workflowsPage.url); + + cy.getByTestId('easy-ai-workflow-card').should('not.exist'); + }); + + it('should show option to take you to the easy AI workflow if experiment is variant', () => { + window.localStorage.setItem( + 'N8N_EXPERIMENT_OVERRIDES', + JSON.stringify({ '026_easy_ai_workflow': 'variant' }), + ); + + cy.visit(workflowsPage.url); + + cy.getByTestId('easy-ai-workflow-card').should('to.exist'); + }); + + it('should show default instructions if free AI credits experiment is control', () => { + window.localStorage.setItem( + 'N8N_EXPERIMENT_OVERRIDES', + JSON.stringify({ '027_free_openai_calls': 'control', '026_easy_ai_workflow': 'variant' }), + ); + + cy.visit(workflowsPage.url); + + cy.getByTestId('easy-ai-workflow-card').click(); + + workflowPage.getters + .stickies() + .eq(0) + .should(($el) => { + expect($el).contains.text('Set up your OpenAI credentials in the OpenAI Model node'); + }); + }); + + it('should show updated instructions if free AI credits experiment is variant', () => { + window.localStorage.setItem( + 'N8N_EXPERIMENT_OVERRIDES', + JSON.stringify({ '027_free_openai_calls': 'variant', '026_easy_ai_workflow': 'variant' }), + ); + + cy.visit(workflowsPage.url); + + cy.getByTestId('easy-ai-workflow-card').click(); + + workflowPage.getters + .stickies() + .eq(0) + .should(($el) => { + expect($el).contains.text( + `Claim your free ${NUMBER_OF_AI_CREDITS} OpenAI calls in the OpenAI model node`, + ); + }); + }); + }); }); diff --git a/cypress/e2e/30-editor-after-route-changes.cy.ts b/cypress/e2e/30-editor-after-route-changes.cy.ts index f0381a32a2..307c4a9537 100644 --- a/cypress/e2e/30-editor-after-route-changes.cy.ts +++ b/cypress/e2e/30-editor-after-route-changes.cy.ts @@ -1,18 +1,14 @@ +import { getWorkflowHistoryCloseButton } from '../composables/workflow'; import { CODE_NODE_NAME, EDIT_FIELDS_SET_NODE_NAME, IF_NODE_NAME, SCHEDULE_TRIGGER_NODE_NAME, } from '../constants'; -import { - WorkflowExecutionsTab, - WorkflowPage as WorkflowPageClass, - WorkflowHistoryPage, -} from '../pages'; +import { WorkflowExecutionsTab, WorkflowPage as WorkflowPageClass } from '../pages'; const workflowPage = new WorkflowPageClass(); const executionsTab = new WorkflowExecutionsTab(); -const workflowHistoryPage = new WorkflowHistoryPage(); const createNewWorkflowAndActivate = () => { workflowPage.actions.visit(); @@ -92,7 +88,7 @@ const switchBetweenEditorAndHistory = () => { cy.wait(['@getVersion']); cy.intercept('GET', '/rest/workflows/*').as('workflowGet'); - workflowHistoryPage.getters.workflowHistoryCloseButton().click(); + getWorkflowHistoryCloseButton().click(); cy.wait(['@workflowGet']); cy.wait(1000); @@ -168,7 +164,7 @@ describe('Editor actions should work', () => { cy.wait(['@getVersion']); cy.intercept('GET', '/rest/workflows/*').as('workflowGet'); - workflowHistoryPage.getters.workflowHistoryCloseButton().click(); + getWorkflowHistoryCloseButton().click(); cy.wait(['@workflowGet']); cy.wait(1000); diff --git a/cypress/e2e/34-template-credentials-setup.cy.ts b/cypress/e2e/34-template-credentials-setup.cy.ts index 386c83eb0a..8d372acac0 100644 --- a/cypress/e2e/34-template-credentials-setup.cy.ts +++ b/cypress/e2e/34-template-credentials-setup.cy.ts @@ -3,6 +3,7 @@ import * as formStep from '../composables/setup-template-form-step'; import { getSetupWorkflowCredentialsButton } from '../composables/setup-workflow-credentials-button'; import TestTemplate1 from '../fixtures/Test_Template_1.json'; import TestTemplate2 from '../fixtures/Test_Template_2.json'; +import { clearNotifications } from '../pages/notifications'; import { clickUseWorkflowButtonByTitle, visitTemplateCollectionPage, @@ -111,16 +112,19 @@ describe('Template credentials setup', () => { templateCredentialsSetupPage.fillInDummyCredentialsForAppWithConfirm('X (Formerly Twitter)'); templateCredentialsSetupPage.fillInDummyCredentialsForApp('Telegram'); + clearNotifications(); + templateCredentialsSetupPage.finishCredentialSetup(); workflowPage.getters.canvasNodes().should('have.length', 3); + cy.grantBrowserPermissions('clipboardReadWrite', 'clipboardSanitizedWrite'); + // Focus the canvas so the copy to clipboard works workflowPage.getters.canvasNodes().eq(0).realClick(); workflowPage.actions.hitSelectAll(); workflowPage.actions.hitCopy(); - cy.grantBrowserPermissions('clipboardReadWrite', 'clipboardSanitizedWrite'); // Check workflow JSON by copying it to clipboard cy.readClipboard().then((workflowJSON) => { const workflow = JSON.parse(workflowJSON); @@ -154,6 +158,8 @@ describe('Template credentials setup', () => { templateCredentialsSetupPage.fillInDummyCredentialsForApp('Email (IMAP)'); templateCredentialsSetupPage.fillInDummyCredentialsForApp('Nextcloud'); + clearNotifications(); + templateCredentialsSetupPage.finishCredentialSetup(); workflowPage.getters.canvasNodes().should('have.length', 3); @@ -176,6 +182,8 @@ describe('Template credentials setup', () => { templateCredentialsSetupPage.visitTemplateCredentialSetupPage(testTemplate.id); templateCredentialsSetupPage.fillInDummyCredentialsForApp('Shopify'); + clearNotifications(); + templateCredentialsSetupPage.finishCredentialSetup(); getSetupWorkflowCredentialsButton().should('be.visible'); @@ -192,6 +200,8 @@ describe('Template credentials setup', () => { templateCredentialsSetupPage.fillInDummyCredentialsForAppWithConfirm('X (Formerly Twitter)'); templateCredentialsSetupPage.fillInDummyCredentialsForApp('Telegram'); + clearNotifications(); + setupCredsModal.closeModalFromContinueButton(); setupCredsModal.getWorkflowCredentialsModal().should('not.exist'); diff --git a/cypress/e2e/35-admin-user-smoke-test.cy.ts b/cypress/e2e/35-admin-user-smoke-test.cy.ts index c8585118e7..6bb31ae1c2 100644 --- a/cypress/e2e/35-admin-user-smoke-test.cy.ts +++ b/cypress/e2e/35-admin-user-smoke-test.cy.ts @@ -1,22 +1,20 @@ -import { SettingsPage } from '../pages/settings'; - -const settingsPage = new SettingsPage(); +const url = '/settings'; describe('Admin user', { disableAutoLogin: true }, () => { it('should see same Settings sub menu items as instance owner', () => { cy.signinAsOwner(); - cy.visit(settingsPage.url); + cy.visit(url); let ownerMenuItems = 0; - settingsPage.getters.menuItems().then(($el) => { + cy.getByTestId('menu-item').then(($el) => { ownerMenuItems = $el.length; }); cy.signout(); cy.signinAsAdmin(); - cy.visit(settingsPage.url); + cy.visit(url); - settingsPage.getters.menuItems().should('have.length', ownerMenuItems); + cy.getByTestId('menu-item').should('have.length', ownerMenuItems); }); }); diff --git a/cypress/e2e/4-node-creator.cy.ts b/cypress/e2e/4-node-creator.cy.ts index e841605863..5e32d5568c 100644 --- a/cypress/e2e/4-node-creator.cy.ts +++ b/cypress/e2e/4-node-creator.cy.ts @@ -517,7 +517,7 @@ describe('Node Creator', () => { const actions = [ 'Get ranked documents from vector store', 'Add documents to vector store', - 'Retrieve documents for AI processing', + 'Retrieve documents for Chain/Tool as Vector Store', ]; nodeCreatorFeature.actions.openNodeCreator(); diff --git a/cypress/e2e/47-subworkflow-debugging.cy.ts b/cypress/e2e/47-subworkflow-debugging.cy.ts index f808bdd044..725b6b32c4 100644 --- a/cypress/e2e/47-subworkflow-debugging.cy.ts +++ b/cypress/e2e/47-subworkflow-debugging.cy.ts @@ -40,7 +40,7 @@ describe('Subworkflow debugging', () => { openNode('Execute Workflow with param'); getOutputPanelItemsCount().should('contain.text', '2 items, 1 sub-execution'); - getOutputPanelRelatedExecutionLink().should('contain.text', 'Inspect Sub-Execution'); + getOutputPanelRelatedExecutionLink().should('contain.text', 'View sub-execution'); getOutputPanelRelatedExecutionLink().should('have.attr', 'href'); // ensure workflow executed and waited on output @@ -64,7 +64,7 @@ describe('Subworkflow debugging', () => { openNode('Execute Workflow with param2'); getOutputPanelItemsCount().should('not.exist'); - getOutputPanelRelatedExecutionLink().should('contain.text', 'Inspect Sub-Execution'); + getOutputPanelRelatedExecutionLink().should('contain.text', 'View sub-execution'); getOutputPanelRelatedExecutionLink().should('have.attr', 'href'); // ensure workflow executed but returned same data as input @@ -109,7 +109,7 @@ describe('Subworkflow debugging', () => { openNode('Execute Workflow with param'); getOutputPanelItemsCount().should('contain.text', '2 items, 1 sub-execution'); - getOutputPanelRelatedExecutionLink().should('contain.text', 'Inspect Sub-Execution'); + getOutputPanelRelatedExecutionLink().should('contain.text', 'View sub-execution'); getOutputPanelRelatedExecutionLink().should('have.attr', 'href'); // ensure workflow executed and waited on output @@ -125,7 +125,7 @@ describe('Subworkflow debugging', () => { getExecutionPreviewOutputPanelRelatedExecutionLink().should( 'include.text', - 'Inspect Parent Execution', + 'View parent execution', ); getExecutionPreviewOutputPanelRelatedExecutionLink() diff --git a/cypress/e2e/6-code-node.cy.ts b/cypress/e2e/6-code-node.cy.ts index 5bc7d05ee2..674d91af18 100644 --- a/cypress/e2e/6-code-node.cy.ts +++ b/cypress/e2e/6-code-node.cy.ts @@ -57,7 +57,7 @@ for (const item of $input.all()) { return `); - getParameter().get('.cm-lint-marker-error').should('have.length', 6); + getParameter().get('.cm-lintRange-error').should('have.length', 6); getParameter().contains('itemMatching').realHover(); cy.get('.cm-tooltip-lint').should( 'have.text', @@ -81,7 +81,7 @@ $input.item() return [] `); - getParameter().get('.cm-lint-marker-error').should('have.length', 5); + getParameter().get('.cm-lintRange-error').should('have.length', 5); getParameter().contains('all').realHover(); cy.get('.cm-tooltip-lint').should( 'have.text', diff --git a/cypress/e2e/7-workflow-actions.cy.ts b/cypress/e2e/7-workflow-actions.cy.ts index 8571b174d9..f0f3ae019a 100644 --- a/cypress/e2e/7-workflow-actions.cy.ts +++ b/cypress/e2e/7-workflow-actions.cy.ts @@ -171,9 +171,16 @@ describe('Workflow Actions', () => { cy.get('#node-creator').should('not.exist'); WorkflowPage.actions.hitSelectAll(); - cy.get('.jtk-drag-selected').should('have.length', 2); WorkflowPage.actions.hitCopy(); successToast().should('exist'); + // Both nodes should be copied + cy.window() + .its('navigator.clipboard') + .then((clip) => clip.readText()) + .then((text) => { + const copiedWorkflow = JSON.parse(text); + expect(copiedWorkflow.nodes).to.have.length(2); + }); }); it('should paste nodes (both current and old node versions)', () => { @@ -345,7 +352,15 @@ describe('Workflow Actions', () => { WorkflowPage.actions.hitDeleteAllNodes(); WorkflowPage.getters.canvasNodes().should('have.length', 0); // Button should be disabled - WorkflowPage.getters.executeWorkflowButton().should('be.disabled'); + cy.ifCanvasVersion( + () => { + WorkflowPage.getters.executeWorkflowButton().should('be.disabled'); + }, + () => { + // In new canvas, button does not exist when there are no nodes + WorkflowPage.getters.executeWorkflowButton().should('not.exist'); + }, + ); // Keyboard shortcut should not work WorkflowPage.actions.hitExecuteWorkflow(); successToast().should('not.exist'); diff --git a/cypress/fixtures/Test_Subworkflow-Inputs.json b/cypress/fixtures/Test_Subworkflow-Inputs.json index aeb4d601fd..ee2b34513e 100644 --- a/cypress/fixtures/Test_Subworkflow-Inputs.json +++ b/cypress/fixtures/Test_Subworkflow-Inputs.json @@ -19,7 +19,6 @@ "value": {}, "matchingColumns": [], "schema": [], - "ignoreTypeMismatchErrors": false, "attemptToConvertTypes": false, "convertFieldsToString": true }, diff --git a/cypress/package.json b/cypress/package.json index 26b585408b..6725c46bc6 100644 --- a/cypress/package.json +++ b/cypress/package.json @@ -12,7 +12,7 @@ "format:check": "biome ci .", "lint": "eslint . --quiet", "lintfix": "eslint . --fix", - "develop": "cd ..; pnpm dev", + "develop": "cd ..; pnpm dev:e2e:server", "start": "cd ..; pnpm start" }, "devDependencies": { diff --git a/cypress/pages/bannerStack.ts b/cypress/pages/bannerStack.ts deleted file mode 100644 index c4936891ae..0000000000 --- a/cypress/pages/bannerStack.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { BasePage } from './base'; - -export class BannerStack extends BasePage { - getters = { - banner: () => cy.getByTestId('banner-stack'), - }; - - actions = {}; -} diff --git a/cypress/pages/base.ts b/cypress/pages/base.ts index abd7a210a8..dbcb65e5bd 100644 --- a/cypress/pages/base.ts +++ b/cypress/pages/base.ts @@ -1,5 +1,13 @@ import type { IE2ETestPage } from '../types'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class BasePage implements IE2ETestPage { getters = {}; diff --git a/cypress/pages/credentials.ts b/cypress/pages/credentials.ts index 08b2fee9c7..b7b68504f9 100644 --- a/cypress/pages/credentials.ts +++ b/cypress/pages/credentials.ts @@ -1,5 +1,13 @@ import { BasePage } from './base'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class CredentialsPage extends BasePage { url = '/home/credentials'; diff --git a/cypress/pages/features/ai-assistant.ts b/cypress/pages/features/ai-assistant.ts index 6ff48851f3..f2e747817b 100644 --- a/cypress/pages/features/ai-assistant.ts +++ b/cypress/pages/features/ai-assistant.ts @@ -8,6 +8,14 @@ const AI_ASSISTANT_FEATURE = { disabledFor: 'control', }; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class AIAssistant extends BasePage { url = '/workflows/new'; diff --git a/cypress/pages/features/node-creator.ts b/cypress/pages/features/node-creator.ts index a0d3995160..38c932468f 100644 --- a/cypress/pages/features/node-creator.ts +++ b/cypress/pages/features/node-creator.ts @@ -1,5 +1,13 @@ import { BasePage } from '../base'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class NodeCreator extends BasePage { url = '/workflow/new'; diff --git a/cypress/pages/index.ts b/cypress/pages/index.ts index 39c9be3b56..93dd165621 100644 --- a/cypress/pages/index.ts +++ b/cypress/pages/index.ts @@ -7,9 +7,7 @@ export * from './settings-users'; export * from './settings-log-streaming'; export * from './sidebar'; export * from './ndv'; -export * from './bannerStack'; export * from './workflow-executions-tab'; export * from './signin'; -export * from './workflow-history'; export * from './workerView'; export * from './settings-public-api'; diff --git a/cypress/pages/mfa-login.ts b/cypress/pages/mfa-login.ts index 7e679804ff..884be7d75d 100644 --- a/cypress/pages/mfa-login.ts +++ b/cypress/pages/mfa-login.ts @@ -3,6 +3,14 @@ import { SigninPage } from './signin'; import { WorkflowsPage } from './workflows'; import { N8N_AUTH_COOKIE } from '../constants'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class MfaLoginPage extends BasePage { url = '/mfa'; diff --git a/cypress/pages/modals/change-password-modal.ts b/cypress/pages/modals/change-password-modal.ts index 3e9ebc8697..28c4d01d86 100644 --- a/cypress/pages/modals/change-password-modal.ts +++ b/cypress/pages/modals/change-password-modal.ts @@ -1,5 +1,13 @@ import { BasePage } from './../base'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class ChangePasswordModal extends BasePage { getters = { modalContainer: () => cy.getByTestId('changePassword-modal').last(), diff --git a/cypress/pages/modals/credentials-modal.ts b/cypress/pages/modals/credentials-modal.ts index b8907386a0..592a396161 100644 --- a/cypress/pages/modals/credentials-modal.ts +++ b/cypress/pages/modals/credentials-modal.ts @@ -2,6 +2,14 @@ import { getCredentialSaveButton, saveCredential } from '../../composables/modal import { getVisibleSelect } from '../../utils'; import { BasePage } from '../base'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class CredentialsModal extends BasePage { getters = { newCredentialModal: () => cy.getByTestId('selectCredential-modal', { timeout: 5000 }), @@ -61,6 +69,7 @@ export class CredentialsModal extends BasePage { this.getters .credentialInputs() .find('input[type=text], input[type=password]') + .filter(':not([readonly])') .each(($el) => { cy.wrap($el).type('test'); }); diff --git a/cypress/pages/modals/message-box.ts b/cypress/pages/modals/message-box.ts index a40c2d1a88..42f83d9a15 100644 --- a/cypress/pages/modals/message-box.ts +++ b/cypress/pages/modals/message-box.ts @@ -1,5 +1,13 @@ import { BasePage } from '../base'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class MessageBox extends BasePage { getters = { modal: () => cy.get('.el-message-box', { withinSubject: null }), diff --git a/cypress/pages/modals/mfa-setup-modal.ts b/cypress/pages/modals/mfa-setup-modal.ts index d127731be2..baa37f5f3a 100644 --- a/cypress/pages/modals/mfa-setup-modal.ts +++ b/cypress/pages/modals/mfa-setup-modal.ts @@ -1,5 +1,13 @@ import { BasePage } from './../base'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class MfaSetupModal extends BasePage { getters = { modalContainer: () => cy.getByTestId('changePassword-modal').last(), diff --git a/cypress/pages/modals/workflow-sharing-modal.ts b/cypress/pages/modals/workflow-sharing-modal.ts index 02e183fc81..176cd84a7b 100644 --- a/cypress/pages/modals/workflow-sharing-modal.ts +++ b/cypress/pages/modals/workflow-sharing-modal.ts @@ -1,5 +1,13 @@ import { BasePage } from '../base'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class WorkflowSharingModal extends BasePage { getters = { modal: () => cy.getByTestId('workflowShare-modal', { timeout: 5000 }), diff --git a/cypress/pages/ndv.ts b/cypress/pages/ndv.ts index 1926ef0ad1..91ece23122 100644 --- a/cypress/pages/ndv.ts +++ b/cypress/pages/ndv.ts @@ -1,6 +1,14 @@ import { BasePage } from './base'; import { getVisiblePopper, getVisibleSelect } from '../utils'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class NDV extends BasePage { getters = { container: () => cy.getByTestId('ndv'), diff --git a/cypress/pages/notifications.ts b/cypress/pages/notifications.ts index 162c536007..af2a51e369 100644 --- a/cypress/pages/notifications.ts +++ b/cypress/pages/notifications.ts @@ -13,5 +13,10 @@ export const infoToast = () => cy.get('.el-notification:has(.el-notification--in * Actions */ export const clearNotifications = () => { - successToast().find('.el-notification__closeBtn').click({ multiple: true }); + const buttons = successToast().find('.el-notification__closeBtn'); + buttons.then(($buttons) => { + if ($buttons.length) { + buttons.click({ multiple: true }); + } + }); }; diff --git a/cypress/pages/settings-log-streaming.ts b/cypress/pages/settings-log-streaming.ts index 9063b8dc41..959b18bce5 100644 --- a/cypress/pages/settings-log-streaming.ts +++ b/cypress/pages/settings-log-streaming.ts @@ -1,6 +1,14 @@ import { BasePage } from './base'; import { getVisibleSelect } from '../utils'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class SettingsLogStreamingPage extends BasePage { url = '/settings/log-streaming'; diff --git a/cypress/pages/settings-personal.ts b/cypress/pages/settings-personal.ts index 5602bd7e92..49c7de1283 100644 --- a/cypress/pages/settings-personal.ts +++ b/cypress/pages/settings-personal.ts @@ -7,6 +7,14 @@ import { MfaSetupModal } from './modals/mfa-setup-modal'; const changePasswordModal = new ChangePasswordModal(); const mfaSetupModal = new MfaSetupModal(); +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class PersonalSettingsPage extends BasePage { url = '/settings/personal'; diff --git a/cypress/pages/settings-usage.ts b/cypress/pages/settings-usage.ts deleted file mode 100644 index 85300fe05f..0000000000 --- a/cypress/pages/settings-usage.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { BasePage } from './base'; - -export class SettingsUsagePage extends BasePage { - url = '/settings/usage'; - - getters = {}; - - actions = {}; -} diff --git a/cypress/pages/settings-users.ts b/cypress/pages/settings-users.ts index 1eaebc911a..f442377fad 100644 --- a/cypress/pages/settings-users.ts +++ b/cypress/pages/settings-users.ts @@ -9,6 +9,14 @@ const workflowsPage = new WorkflowsPage(); const mainSidebar = new MainSidebar(); const settingsSidebar = new SettingsSidebar(); +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class SettingsUsersPage extends BasePage { url = '/settings/users'; diff --git a/cypress/pages/settings.ts b/cypress/pages/settings.ts deleted file mode 100644 index 74c3b0fe76..0000000000 --- a/cypress/pages/settings.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { BasePage } from './base'; - -export class SettingsPage extends BasePage { - url = '/settings'; - - getters = { - menuItems: () => cy.getByTestId('menu-item'), - }; - - actions = {}; -} diff --git a/cypress/pages/sidebar/main-sidebar.ts b/cypress/pages/sidebar/main-sidebar.ts index 4266b93688..7824a6bebb 100644 --- a/cypress/pages/sidebar/main-sidebar.ts +++ b/cypress/pages/sidebar/main-sidebar.ts @@ -1,6 +1,14 @@ import { BasePage } from '../base'; import { WorkflowsPage } from '../workflows'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class MainSidebar extends BasePage { getters = { menuItem: (id: string) => cy.getByTestId('menu-item').get('#' + id), diff --git a/cypress/pages/sidebar/settings-sidebar.ts b/cypress/pages/sidebar/settings-sidebar.ts index 17d43b65e7..17bf5d10b7 100644 --- a/cypress/pages/sidebar/settings-sidebar.ts +++ b/cypress/pages/sidebar/settings-sidebar.ts @@ -1,5 +1,13 @@ import { BasePage } from '../base'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class SettingsSidebar extends BasePage { getters = { menuItem: (id: string) => cy.getByTestId('menu-item').get('#' + id), diff --git a/cypress/pages/signin.ts b/cypress/pages/signin.ts index a97fe4888e..bc0d7196a3 100644 --- a/cypress/pages/signin.ts +++ b/cypress/pages/signin.ts @@ -2,6 +2,14 @@ import { BasePage } from './base'; import { WorkflowsPage } from './workflows'; import { N8N_AUTH_COOKIE } from '../constants'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class SigninPage extends BasePage { url = '/signin'; diff --git a/cypress/pages/templates.ts b/cypress/pages/templates.ts index a17da87ba2..8d8fdc15c5 100644 --- a/cypress/pages/templates.ts +++ b/cypress/pages/templates.ts @@ -1,5 +1,13 @@ import { BasePage } from './base'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class TemplatesPage extends BasePage { url = '/templates'; diff --git a/cypress/pages/variables.ts b/cypress/pages/variables.ts index 6ac9a939b2..5fb0a64d9a 100644 --- a/cypress/pages/variables.ts +++ b/cypress/pages/variables.ts @@ -2,6 +2,14 @@ import { BasePage } from './base'; import Chainable = Cypress.Chainable; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class VariablesPage extends BasePage { url = '/variables'; diff --git a/cypress/pages/workerView.ts b/cypress/pages/workerView.ts index f442468c52..ff56ab1ec5 100644 --- a/cypress/pages/workerView.ts +++ b/cypress/pages/workerView.ts @@ -1,5 +1,13 @@ import { BasePage } from './base'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class WorkerViewPage extends BasePage { url = '/settings/workers'; diff --git a/cypress/pages/workflow-executions-tab.ts b/cypress/pages/workflow-executions-tab.ts index be022e6cdf..da91e99a98 100644 --- a/cypress/pages/workflow-executions-tab.ts +++ b/cypress/pages/workflow-executions-tab.ts @@ -3,6 +3,14 @@ import { WorkflowPage } from './workflow'; const workflowPage = new WorkflowPage(); +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class WorkflowExecutionsTab extends BasePage { getters = { executionsTabButton: () => cy.getByTestId('radio-button-executions'), diff --git a/cypress/pages/workflow-history.ts b/cypress/pages/workflow-history.ts deleted file mode 100644 index 1b9d7328b1..0000000000 --- a/cypress/pages/workflow-history.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { BasePage } from './base'; - -export class WorkflowHistoryPage extends BasePage { - getters = { - workflowHistoryCloseButton: () => cy.getByTestId('workflow-history-close-button'), - }; -} diff --git a/cypress/pages/workflow.ts b/cypress/pages/workflow.ts index e99b01aa46..4d6702b082 100644 --- a/cypress/pages/workflow.ts +++ b/cypress/pages/workflow.ts @@ -6,6 +6,15 @@ import { getVisibleSelect } from '../utils'; import { getUniqueWorkflowName, isCanvasV2 } from '../utils/workflowUtils'; const nodeCreator = new NodeCreator(); + +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class WorkflowPage extends BasePage { url = '/workflow/new'; diff --git a/cypress/pages/workflows.ts b/cypress/pages/workflows.ts index a58911a355..7441bfa256 100644 --- a/cypress/pages/workflows.ts +++ b/cypress/pages/workflows.ts @@ -1,5 +1,13 @@ import { BasePage } from './base'; +/** + * @deprecated Use functional composables from @composables instead. + * If a composable doesn't exist for your use case, please create a new one in: + * cypress/composables + * + * This class-based approach is being phased out in favor of more modular functional composables. + * Each getter and action in this class should be moved to individual composable functions. + */ export class WorkflowsPage extends BasePage { url = '/home/workflows'; diff --git a/package.json b/package.json index 90e48e9bd8..7b6c9027df 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,11 @@ "dev": "turbo run dev --parallel --env-mode=loose --filter=!n8n-design-system --filter=!@n8n/chat --filter=!@n8n/task-runner", "dev:be": "turbo run dev --parallel --env-mode=loose --filter=!n8n-design-system --filter=!@n8n/chat --filter=!@n8n/task-runner --filter=!n8n-editor-ui", "dev:ai": "turbo run dev --parallel --env-mode=loose --filter=@n8n/nodes-langchain --filter=n8n --filter=n8n-core", + "dev:fe": "run-p start \"dev:fe:editor --filter=n8n-design-system\"", + "dev:fe:editor": "turbo run dev --parallel --env-mode=loose --filter=n8n-editor-ui", + "dev:e2e": "cd cypress && pnpm run test:e2e:dev", + "dev:e2e:v2": "cd cypress && pnpm run test:e2e:dev:v2", + "dev:e2e:server": "run-p start dev:fe:editor", "clean": "turbo run clean --parallel", "reset": "node scripts/ensure-zx.mjs && zx scripts/reset.mjs", "format": "turbo run format && node scripts/format.mjs", @@ -55,6 +60,7 @@ "lefthook": "^1.7.15", "nock": "^13.3.2", "nodemon": "^3.0.1", + "npm-run-all2": "^7.0.2", "p-limit": "^3.1.0", "rimraf": "^5.0.1", "run-script-os": "^1.0.7", diff --git a/packages/@n8n/config/src/configs/runners.config.ts b/packages/@n8n/config/src/configs/runners.config.ts index 733e724408..02ebdf5df9 100644 --- a/packages/@n8n/config/src/configs/runners.config.ts +++ b/packages/@n8n/config/src/configs/runners.config.ts @@ -39,13 +39,23 @@ export class TaskRunnersConfig { @Env('N8N_RUNNERS_MAX_OLD_SPACE_SIZE') maxOldSpaceSize: string = ''; - /** How many concurrent tasks can a runner execute at a time */ + /** + * How many concurrent tasks can a runner execute at a time + * + * Kept high for backwards compatibility - n8n v2 will reduce this to `5` + */ @Env('N8N_RUNNERS_MAX_CONCURRENCY') - maxConcurrency: number = 5; + maxConcurrency: number = 10; - /** How long (in seconds) a task is allowed to take for completion, else the task will be aborted. (In internal mode, the runner will also be restarted.) Must be greater than 0. */ + /** + * How long (in seconds) a task is allowed to take for completion, else the + * task will be aborted. (In internal mode, the runner will also be + * restarted.) Must be greater than 0. + * + * Kept high for backwards compatibility - n8n v2 will reduce this to `60` + */ @Env('N8N_RUNNERS_TASK_TIMEOUT') - taskTimeout: number = 60; + taskTimeout: number = 300; // 5 minutes /** How often (in seconds) the runner must send a heartbeat to the broker, else the task will be aborted. (In internal mode, the runner will also be restarted.) Must be greater than 0. */ @Env('N8N_RUNNERS_HEARTBEAT_INTERVAL') diff --git a/packages/@n8n/config/test/config.test.ts b/packages/@n8n/config/test/config.test.ts index 9fd0a35d5a..ed386fee6b 100644 --- a/packages/@n8n/config/test/config.test.ts +++ b/packages/@n8n/config/test/config.test.ts @@ -229,8 +229,8 @@ describe('GlobalConfig', () => { maxPayload: 1024 * 1024 * 1024, port: 5679, maxOldSpaceSize: '', - maxConcurrency: 5, - taskTimeout: 60, + maxConcurrency: 10, + taskTimeout: 300, heartbeatInterval: 30, }, sentry: { diff --git a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.ts b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.ts index 5ca6091378..ab3320c2ba 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/LmChatGoogleVertex/LmChatGoogleVertex.node.ts @@ -131,6 +131,7 @@ export class LmChatGoogleVertex implements INodeType { const credentials = await this.getCredentials('googleApi'); const privateKey = formatPrivateKey(credentials.privateKey as string); const email = (credentials.email as string).trim(); + const region = credentials.region as string; const modelName = this.getNodeParameter('modelName', itemIndex) as string; @@ -165,6 +166,7 @@ export class LmChatGoogleVertex implements INodeType { private_key: privateKey, }, }, + location: region, model: modelName, topK: options.topK, topP: options.topP, diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/__snapshots__/createVectorStoreNode.test.ts.snap b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/__snapshots__/createVectorStoreNode.test.ts.snap index 91da891842..2eff69800b 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/__snapshots__/createVectorStoreNode.test.ts.snap +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/__snapshots__/createVectorStoreNode.test.ts.snap @@ -89,14 +89,14 @@ exports[`createVectorStoreNode retrieve mode supplies vector store as data 1`] = "value": "insert", }, { - "action": "Retrieve documents for AI processing as Vector Store", + "action": "Retrieve documents for Chain/Tool as Vector Store", "description": "Retrieve documents from vector store to be used as vector store with AI nodes", - "name": "Retrieve Documents (As Vector Store for AI Agent)", + "name": "Retrieve Documents (As Vector Store for Chain/Tool)", "outputConnectionType": "ai_vectorStore", "value": "retrieve", }, { - "action": "Retrieve documents for AI processing as Tool", + "action": "Retrieve documents for AI Agent as Tool", "description": "Retrieve documents from vector store to be used as tool with AI nodes", "name": "Retrieve Documents (As Tool for AI Agent)", "outputConnectionType": "ai_tool", diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts index f8e11cadf1..441126c985 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts @@ -111,17 +111,17 @@ function getOperationModeOptions(args: VectorStoreNodeConstructorArgs): INodePro action: 'Add documents to vector store', }, { - name: 'Retrieve Documents (As Vector Store for AI Agent)', + name: 'Retrieve Documents (As Vector Store for Chain/Tool)', value: 'retrieve', description: 'Retrieve documents from vector store to be used as vector store with AI nodes', - action: 'Retrieve documents for AI processing as Vector Store', + action: 'Retrieve documents for Chain/Tool as Vector Store', outputConnectionType: NodeConnectionType.AiVectorStore, }, { name: 'Retrieve Documents (As Tool for AI Agent)', value: 'retrieve-as-tool', description: 'Retrieve documents from vector store to be used as tool with AI nodes', - action: 'Retrieve documents for AI processing as Tool', + action: 'Retrieve documents for AI Agent as Tool', outputConnectionType: NodeConnectionType.AiTool, }, { diff --git a/packages/@n8n/task-runner/src/config/base-runner-config.ts b/packages/@n8n/task-runner/src/config/base-runner-config.ts index d08056c5ae..f9032dda9f 100644 --- a/packages/@n8n/task-runner/src/config/base-runner-config.ts +++ b/packages/@n8n/task-runner/src/config/base-runner-config.ts @@ -23,8 +23,13 @@ export class BaseRunnerConfig { @Env('N8N_RUNNERS_MAX_PAYLOAD') maxPayloadSize: number = 1024 * 1024 * 1024; + /** + * How many concurrent tasks can a runner execute at a time + * + * Kept high for backwards compatibility - n8n v2 will reduce this to `5` + */ @Env('N8N_RUNNERS_MAX_CONCURRENCY') - maxConcurrency: number = 5; + maxConcurrency: number = 10; /** * How long (in seconds) a runner may be idle for before exit. Intended @@ -37,8 +42,15 @@ export class BaseRunnerConfig { @Env('GENERIC_TIMEZONE') timezone: string = 'America/New_York'; + /** + * How long (in seconds) a task is allowed to take for completion, else the + * task will be aborted. (In internal mode, the runner will also be + * restarted.) Must be greater than 0. + * + * Kept high for backwards compatibility - n8n v2 will reduce this to `60` + */ @Env('N8N_RUNNERS_TASK_TIMEOUT') - taskTimeout: number = 60; + taskTimeout: number = 300; // 5 minutes @Nested healthcheckServer!: HealthcheckServerConfig; diff --git a/packages/cli/src/constants.ts b/packages/cli/src/constants.ts index 4bd1890c4e..abcf298d3d 100644 --- a/packages/cli/src/constants.ts +++ b/packages/cli/src/constants.ts @@ -195,4 +195,3 @@ export const WsStatusCodes = { } as const; export const FREE_AI_CREDITS_CREDENTIAL_NAME = 'n8n free OpenAI API credits'; -export const OPEN_AI_API_CREDENTIAL_TYPE = 'openAiApi'; diff --git a/packages/cli/src/controllers/ai.controller.ts b/packages/cli/src/controllers/ai.controller.ts index 791c02bec3..e82ecd77ef 100644 --- a/packages/cli/src/controllers/ai.controller.ts +++ b/packages/cli/src/controllers/ai.controller.ts @@ -6,10 +6,11 @@ import { } from '@n8n/api-types'; import type { AiAssistantSDK } from '@n8n_io/ai-assistant-sdk'; import { Response } from 'express'; +import { OPEN_AI_API_CREDENTIAL_TYPE } from 'n8n-workflow'; import { strict as assert } from 'node:assert'; import { WritableStream } from 'node:stream/web'; -import { FREE_AI_CREDITS_CREDENTIAL_NAME, OPEN_AI_API_CREDENTIAL_TYPE } from '@/constants'; +import { FREE_AI_CREDITS_CREDENTIAL_NAME } from '@/constants'; import { CredentialsService } from '@/credentials/credentials.service'; import { Body, Post, RestController } from '@/decorators'; import { InternalServerError } from '@/errors/response-errors/internal-server.error'; diff --git a/packages/cli/src/databases/entities/test-run.ee.ts b/packages/cli/src/databases/entities/test-run.ee.ts index ab5f041f11..39d8e16ddd 100644 --- a/packages/cli/src/databases/entities/test-run.ee.ts +++ b/packages/cli/src/databases/entities/test-run.ee.ts @@ -35,4 +35,23 @@ export class TestRun extends WithTimestampsAndStringId { @Column(jsonColumnType, { nullable: true }) metrics: AggregatedTestRunMetrics; + + /** + * Total number of the test cases, matching the filter condition of the test definition (specified annotationTag) + */ + @Column('integer', { nullable: true }) + totalCases: number; + + /** + * Number of test cases that passed (evaluation workflow was executed successfully) + */ + @Column('integer', { nullable: true }) + passedCases: number; + + /** + * Number of failed test cases + * (any unexpected exception happened during the execution or evaluation workflow ended with an error) + */ + @Column('integer', { nullable: true }) + failedCases: number; } diff --git a/packages/cli/src/databases/migrations/common/1736172058779-AddStatsColumnsToTestRun.ts b/packages/cli/src/databases/migrations/common/1736172058779-AddStatsColumnsToTestRun.ts new file mode 100644 index 0000000000..8d6a5cdc20 --- /dev/null +++ b/packages/cli/src/databases/migrations/common/1736172058779-AddStatsColumnsToTestRun.ts @@ -0,0 +1,31 @@ +import type { MigrationContext, ReversibleMigration } from '@/databases/types'; + +const columns = ['totalCases', 'passedCases', 'failedCases'] as const; + +export class AddStatsColumnsToTestRun1736172058779 implements ReversibleMigration { + async up({ escape, runQuery }: MigrationContext) { + const tableName = escape.tableName('test_run'); + const columnNames = columns.map((name) => escape.columnName(name)); + + // Values can be NULL only if the test run is new, otherwise they must be non-negative integers. + // Test run might be cancelled or interrupted by unexpected error at any moment, so values can be either NULL or non-negative integers. + for (const name of columnNames) { + await runQuery(`ALTER TABLE ${tableName} ADD COLUMN ${name} INT CHECK( + CASE + WHEN status = 'new' THEN ${name} IS NULL + WHEN status in ('cancelled', 'error') THEN ${name} IS NULL OR ${name} >= 0 + ELSE ${name} >= 0 + END + )`); + } + } + + async down({ escape, runQuery }: MigrationContext) { + const tableName = escape.tableName('test_run'); + const columnNames = columns.map((name) => escape.columnName(name)); + + for (const name of columnNames) { + await runQuery(`ALTER TABLE ${tableName} DROP COLUMN ${name}`); + } + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index 89df273472..b76409c0c1 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -76,6 +76,7 @@ import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-Crea import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable'; import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable'; +import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun'; export const mysqlMigrations: Migration[] = [ InitialMigration1588157391238, @@ -154,4 +155,5 @@ export const mysqlMigrations: Migration[] = [ AddMockedNodesColumnToTestDefinition1733133775640, AddManagedColumnToCredentialsTable1734479635324, AddProjectIcons1729607673469, + AddStatsColumnsToTestRun1736172058779, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index d5d72282f4..7cf90bde5b 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -76,6 +76,7 @@ import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-Crea import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable'; import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable'; +import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun'; export const postgresMigrations: Migration[] = [ InitialMigration1587669153312, @@ -154,4 +155,5 @@ export const postgresMigrations: Migration[] = [ AddMockedNodesColumnToTestDefinition1733133775640, AddManagedColumnToCredentialsTable1734479635324, AddProjectIcons1729607673469, + AddStatsColumnsToTestRun1736172058779, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index 7fec59baf2..363a6e47c3 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -73,6 +73,7 @@ import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-Crea import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable'; import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable'; +import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun'; const sqliteMigrations: Migration[] = [ InitialMigration1588102412422, @@ -148,6 +149,7 @@ const sqliteMigrations: Migration[] = [ AddMockedNodesColumnToTestDefinition1733133775640, AddManagedColumnToCredentialsTable1734479635324, AddProjectIcons1729607673469, + AddStatsColumnsToTestRun1736172058779, ]; export { sqliteMigrations }; diff --git a/packages/cli/src/databases/repositories/test-run.repository.ee.ts b/packages/cli/src/databases/repositories/test-run.repository.ee.ts index f0f235c551..037844734f 100644 --- a/packages/cli/src/databases/repositories/test-run.repository.ee.ts +++ b/packages/cli/src/databases/repositories/test-run.repository.ee.ts @@ -21,14 +21,28 @@ export class TestRunRepository extends Repository { return await this.save(testRun); } - async markAsRunning(id: string) { - return await this.update(id, { status: 'running', runAt: new Date() }); + async markAsRunning(id: string, totalCases: number) { + return await this.update(id, { + status: 'running', + runAt: new Date(), + totalCases, + passedCases: 0, + failedCases: 0, + }); } async markAsCompleted(id: string, metrics: AggregatedTestRunMetrics) { return await this.update(id, { status: 'completed', completedAt: new Date(), metrics }); } + async incrementPassed(id: string) { + return await this.increment({ id }, 'passedCases', 1); + } + + async incrementFailed(id: string) { + return await this.increment({ id }, 'failedCases', 1); + } + async getMany(testDefinitionId: string, options: ListQuery.Options) { const findManyOptions: FindManyOptions = { where: { testDefinition: { id: testDefinitionId } }, diff --git a/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts b/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts index 06b9653374..cc3c3bc33b 100644 --- a/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts +++ b/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts @@ -2,7 +2,8 @@ import type { SelectQueryBuilder } from '@n8n/typeorm'; import { stringify } from 'flatted'; import { readFileSync } from 'fs'; import { mock, mockDeep } from 'jest-mock-extended'; -import type { GenericValue, IRun } from 'n8n-workflow'; +import type { ErrorReporter } from 'n8n-core'; +import type { ExecutionError, GenericValue, IRun } from 'n8n-workflow'; import path from 'path'; import type { ActiveExecutions } from '@/active-executions'; @@ -90,6 +91,16 @@ function mockExecutionData() { }); } +function mockErrorExecutionData() { + return mock({ + data: { + resultData: { + error: mock(), + }, + }, + }); +} + function mockEvaluationExecutionData(metrics: Record) { return mock({ data: { @@ -110,6 +121,9 @@ function mockEvaluationExecutionData(metrics: Record) { }, ], }, + // error is an optional prop, but jest-mock-extended will mock it by default, + // which affects the code logic. So, we need to explicitly set it to undefined. + error: undefined, }, }, }); @@ -156,6 +170,8 @@ describe('TestRunnerService', () => { testRunRepository.createTestRun.mockClear(); testRunRepository.markAsRunning.mockClear(); testRunRepository.markAsCompleted.mockClear(); + testRunRepository.incrementFailed.mockClear(); + testRunRepository.incrementPassed.mockClear(); }); test('should create an instance of TestRunnerService', async () => { @@ -167,6 +183,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, + mock(), ); expect(testRunnerService).toBeInstanceOf(TestRunnerService); @@ -181,6 +198,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, + mock(), ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -218,6 +236,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, + mock(), ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -298,12 +317,185 @@ describe('TestRunnerService', () => { // Check Test Run status was updated correctly expect(testRunRepository.createTestRun).toHaveBeenCalledTimes(1); expect(testRunRepository.markAsRunning).toHaveBeenCalledTimes(1); - expect(testRunRepository.markAsRunning).toHaveBeenCalledWith('test-run-id'); + expect(testRunRepository.markAsRunning).toHaveBeenCalledWith('test-run-id', expect.any(Number)); expect(testRunRepository.markAsCompleted).toHaveBeenCalledTimes(1); expect(testRunRepository.markAsCompleted).toHaveBeenCalledWith('test-run-id', { metric1: 0.75, metric2: 0, }); + + expect(testRunRepository.incrementPassed).toHaveBeenCalledTimes(2); + expect(testRunRepository.incrementFailed).not.toHaveBeenCalled(); + }); + + test('should properly count passed and failed executions', async () => { + const testRunnerService = new TestRunnerService( + workflowRepository, + workflowRunner, + executionRepository, + activeExecutions, + testRunRepository, + testMetricRepository, + mockNodeTypes, + mock(), + ); + + workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ + id: 'workflow-under-test-id', + ...wfUnderTestJson, + }); + + workflowRepository.findById.calledWith('evaluation-workflow-id').mockResolvedValueOnce({ + id: 'evaluation-workflow-id', + ...wfEvaluationJson, + }); + + workflowRunner.run.mockResolvedValueOnce('some-execution-id'); + workflowRunner.run.mockResolvedValueOnce('some-execution-id-2'); + workflowRunner.run.mockResolvedValueOnce('some-execution-id-3'); + workflowRunner.run.mockResolvedValueOnce('some-execution-id-4'); + + // Mock executions of workflow under test + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id') + .mockResolvedValue(mockExecutionData()); + + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id-3') + .mockResolvedValue(mockExecutionData()); + + // Mock executions of evaluation workflow + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id-2') + .mockResolvedValue(mockEvaluationExecutionData({ metric1: 1, metric2: 0 })); + + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id-4') + .mockRejectedValue(new Error('Some error')); + + await testRunnerService.runTest( + mock(), + mock({ + workflowId: 'workflow-under-test-id', + evaluationWorkflowId: 'evaluation-workflow-id', + mockedNodes: [], + }), + ); + + expect(testRunRepository.incrementPassed).toHaveBeenCalledTimes(1); + expect(testRunRepository.incrementFailed).toHaveBeenCalledTimes(1); + }); + + test('should properly count failed test executions', async () => { + const testRunnerService = new TestRunnerService( + workflowRepository, + workflowRunner, + executionRepository, + activeExecutions, + testRunRepository, + testMetricRepository, + mockNodeTypes, + mock(), + ); + + workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ + id: 'workflow-under-test-id', + ...wfUnderTestJson, + }); + + workflowRepository.findById.calledWith('evaluation-workflow-id').mockResolvedValueOnce({ + id: 'evaluation-workflow-id', + ...wfEvaluationJson, + }); + + workflowRunner.run.mockResolvedValueOnce('some-execution-id'); + workflowRunner.run.mockResolvedValueOnce('some-execution-id-2'); + workflowRunner.run.mockResolvedValueOnce('some-execution-id-3'); + workflowRunner.run.mockResolvedValueOnce('some-execution-id-4'); + + // Mock executions of workflow under test + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id') + .mockResolvedValue(mockExecutionData()); + + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id-3') + .mockResolvedValue(mockErrorExecutionData()); + + // Mock executions of evaluation workflow + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id-2') + .mockResolvedValue(mockEvaluationExecutionData({ metric1: 1, metric2: 0 })); + + await testRunnerService.runTest( + mock(), + mock({ + workflowId: 'workflow-under-test-id', + evaluationWorkflowId: 'evaluation-workflow-id', + mockedNodes: [], + }), + ); + + expect(testRunRepository.incrementPassed).toHaveBeenCalledTimes(1); + expect(testRunRepository.incrementFailed).toHaveBeenCalledTimes(1); + }); + + test('should properly count failed evaluations', async () => { + const testRunnerService = new TestRunnerService( + workflowRepository, + workflowRunner, + executionRepository, + activeExecutions, + testRunRepository, + testMetricRepository, + mockNodeTypes, + mock(), + ); + + workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ + id: 'workflow-under-test-id', + ...wfUnderTestJson, + }); + + workflowRepository.findById.calledWith('evaluation-workflow-id').mockResolvedValueOnce({ + id: 'evaluation-workflow-id', + ...wfEvaluationJson, + }); + + workflowRunner.run.mockResolvedValueOnce('some-execution-id'); + workflowRunner.run.mockResolvedValueOnce('some-execution-id-2'); + workflowRunner.run.mockResolvedValueOnce('some-execution-id-3'); + workflowRunner.run.mockResolvedValueOnce('some-execution-id-4'); + + // Mock executions of workflow under test + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id') + .mockResolvedValue(mockExecutionData()); + + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id-3') + .mockResolvedValue(mockExecutionData()); + + // Mock executions of evaluation workflow + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id-2') + .mockResolvedValue(mockEvaluationExecutionData({ metric1: 1, metric2: 0 })); + + activeExecutions.getPostExecutePromise + .calledWith('some-execution-id-4') + .mockResolvedValue(mockErrorExecutionData()); + + await testRunnerService.runTest( + mock(), + mock({ + workflowId: 'workflow-under-test-id', + evaluationWorkflowId: 'evaluation-workflow-id', + mockedNodes: [], + }), + ); + + expect(testRunRepository.incrementPassed).toHaveBeenCalledTimes(1); + expect(testRunRepository.incrementFailed).toHaveBeenCalledTimes(1); }); test('should specify correct start nodes when running workflow under test', async () => { @@ -315,6 +507,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, + mock(), ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -388,6 +581,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, + mock(), ); const startNodesData = (testRunnerService as any).getStartNodesData( @@ -412,6 +606,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, + mock(), ); const startNodesData = (testRunnerService as any).getStartNodesData( diff --git a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts index 926bc29b70..5a054a3527 100644 --- a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts +++ b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts @@ -1,5 +1,6 @@ import { Service } from '@n8n/di'; import { parse } from 'flatted'; +import { ErrorReporter } from 'n8n-core'; import { NodeConnectionType, Workflow } from 'n8n-workflow'; import type { IDataObject, @@ -45,6 +46,7 @@ export class TestRunnerService { private readonly testRunRepository: TestRunRepository, private readonly testMetricRepository: TestMetricRepository, private readonly nodeTypes: NodeTypes, + private readonly errorReporter: ErrorReporter, ) {} /** @@ -134,6 +136,7 @@ export class TestRunnerService { evaluationWorkflow: WorkflowEntity, expectedData: IRunData, actualData: IRunData, + testRunId?: string, ) { // Prepare the evaluation wf input data. // Provide both the expected data and the actual data @@ -146,7 +149,13 @@ export class TestRunnerService { // Prepare the data to run the evaluation workflow const data = await getRunData(evaluationWorkflow, [evaluationInputData]); - + // FIXME: This is a hack to add the testRunId to the evaluation workflow execution data + // So that we can fetch all execution runs for a test run + if (testRunId && data.executionData) { + data.executionData.resultData.metadata = { + testRunId, + }; + } data.executionMode = 'evaluation'; // Trigger the evaluation workflow @@ -223,52 +232,66 @@ export class TestRunnerService { // 2. Run over all the test cases - await this.testRunRepository.markAsRunning(testRun.id); + await this.testRunRepository.markAsRunning(testRun.id, pastExecutions.length); // Object to collect the results of the evaluation workflow executions const metrics = new EvaluationMetrics(testMetricNames); for (const { id: pastExecutionId } of pastExecutions) { - // Fetch past execution with data - const pastExecution = await this.executionRepository.findOne({ - where: { id: pastExecutionId }, - relations: ['executionData', 'metadata'], - }); - assert(pastExecution, 'Execution not found'); + try { + // Fetch past execution with data + const pastExecution = await this.executionRepository.findOne({ + where: { id: pastExecutionId }, + relations: ['executionData', 'metadata'], + }); + assert(pastExecution, 'Execution not found'); - const executionData = parse(pastExecution.executionData.data) as IRunExecutionData; + const executionData = parse(pastExecution.executionData.data) as IRunExecutionData; - // Run the test case and wait for it to finish - const testCaseExecution = await this.runTestCase( - workflow, - executionData, - pastExecution.executionData.workflowData, - test.mockedNodes, - user.id, - ); + // Run the test case and wait for it to finish + const testCaseExecution = await this.runTestCase( + workflow, + executionData, + pastExecution.executionData.workflowData, + test.mockedNodes, + user.id, + ); - // In case of a permission check issue, the test case execution will be undefined. - // Skip them and continue with the next test case - if (!testCaseExecution) { - continue; + // In case of a permission check issue, the test case execution will be undefined. + // Skip them, increment the failed count and continue with the next test case + if (!testCaseExecution) { + await this.testRunRepository.incrementFailed(testRun.id); + continue; + } + + // Collect the results of the test case execution + const testCaseRunData = testCaseExecution.data.resultData.runData; + + // Get the original runData from the test case execution data + const originalRunData = executionData.resultData.runData; + + // Run the evaluation workflow with the original and new run data + const evalExecution = await this.runTestCaseEvaluation( + evaluationWorkflow, + originalRunData, + testCaseRunData, + testRun.id, + ); + assert(evalExecution); + + metrics.addResults(this.extractEvaluationResult(evalExecution)); + + if (evalExecution.data.resultData.error) { + await this.testRunRepository.incrementFailed(testRun.id); + } else { + await this.testRunRepository.incrementPassed(testRun.id); + } + } catch (e) { + // In case of an unexpected error, increment the failed count and continue with the next test case + await this.testRunRepository.incrementFailed(testRun.id); + + this.errorReporter.error(e); } - - // Collect the results of the test case execution - const testCaseRunData = testCaseExecution.data.resultData.runData; - - // Get the original runData from the test case execution data - const originalRunData = executionData.resultData.runData; - - // Run the evaluation workflow with the original and new run data - const evalExecution = await this.runTestCaseEvaluation( - evaluationWorkflow, - originalRunData, - testCaseRunData, - ); - assert(evalExecution); - - // Extract the output of the last node executed in the evaluation workflow - metrics.addResults(this.extractEvaluationResult(evalExecution)); } const aggregatedMetrics = metrics.getAggregatedMetrics(); diff --git a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts index 9b4d8aecd2..01673cd375 100644 --- a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts +++ b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts @@ -5,7 +5,9 @@ import type { INode, INodesGraphResult } from 'n8n-workflow'; import { NodeApiError, TelemetryHelpers, type IRun, type IWorkflowBase } from 'n8n-workflow'; import { N8N_VERSION } from '@/constants'; +import type { CredentialsEntity } from '@/databases/entities/credentials-entity'; import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; +import type { CredentialsRepository } from '@/databases/repositories/credentials.repository'; import type { ProjectRelationRepository } from '@/databases/repositories/project-relation.repository'; import type { SharedWorkflowRepository } from '@/databases/repositories/shared-workflow.repository'; import type { WorkflowRepository } from '@/databases/repositories/workflow.repository'; @@ -52,6 +54,7 @@ describe('TelemetryEventRelay', () => { const nodeTypes = mock(); const sharedWorkflowRepository = mock(); const projectRelationRepository = mock(); + const credentialsRepository = mock(); const eventService = new EventService(); let telemetryEventRelay: TelemetryEventRelay; @@ -67,6 +70,7 @@ describe('TelemetryEventRelay', () => { nodeTypes, sharedWorkflowRepository, projectRelationRepository, + credentialsRepository, ); await telemetryEventRelay.init(); @@ -90,6 +94,7 @@ describe('TelemetryEventRelay', () => { nodeTypes, sharedWorkflowRepository, projectRelationRepository, + credentialsRepository, ); // @ts-expect-error Private method const setupListenersSpy = jest.spyOn(telemetryEventRelay, 'setupListeners'); @@ -112,6 +117,7 @@ describe('TelemetryEventRelay', () => { nodeTypes, sharedWorkflowRepository, projectRelationRepository, + credentialsRepository, ); // @ts-expect-error Private method const setupListenersSpy = jest.spyOn(telemetryEventRelay, 'setupListeners'); @@ -1197,6 +1203,9 @@ describe('TelemetryEventRelay', () => { it('should call telemetry.track when manual node execution finished', async () => { sharedWorkflowRepository.findSharingRole.mockResolvedValue('workflow:editor'); + credentialsRepository.findOneBy.mockResolvedValue( + mock({ type: 'openAiApi', isManaged: false }), + ); const runData = { status: 'error', @@ -1276,6 +1285,8 @@ describe('TelemetryEventRelay', () => { error_node_id: '1', node_id: '1', node_type: 'n8n-nodes-base.jira', + is_managed: false, + credential_type: null, node_graph_string: JSON.stringify(nodeGraph.nodeGraph), }), ); @@ -1498,5 +1509,187 @@ describe('TelemetryEventRelay', () => { }), ); }); + + it('should call telemetry.track when manual node execution finished with is_managed and credential_type properties', async () => { + sharedWorkflowRepository.findSharingRole.mockResolvedValue('workflow:editor'); + credentialsRepository.findOneBy.mockResolvedValue( + mock({ type: 'openAiApi', isManaged: true }), + ); + + const runData = { + status: 'error', + mode: 'manual', + data: { + executionData: { + nodeExecutionStack: [{ node: { credentials: { openAiApi: { id: 'nhu-l8E4hX' } } } }], + }, + startData: { + destinationNode: 'OpenAI', + runNodeFilter: ['OpenAI'], + }, + resultData: { + runData: {}, + lastNodeExecuted: 'OpenAI', + error: new NodeApiError( + { + id: '1', + typeVersion: 1, + name: 'Jira', + type: 'n8n-nodes-base.jira', + parameters: {}, + position: [100, 200], + }, + { + message: 'Error message', + description: 'Incorrect API key provided', + httpCode: '401', + stack: '', + }, + { + message: 'Error message', + description: 'Error description', + level: 'warning', + functionality: 'regular', + }, + ), + }, + }, + } as unknown as IRun; + + const nodeGraph: INodesGraphResult = { + nodeGraph: { node_types: [], node_connections: [], webhookNodeNames: [] }, + nameIndices: { + Jira: '1', + OpenAI: '1', + }, + } as unknown as INodesGraphResult; + + jest.spyOn(TelemetryHelpers, 'generateNodesGraph').mockImplementation(() => nodeGraph); + + jest + .spyOn(TelemetryHelpers, 'getNodeTypeForName') + .mockImplementation( + () => ({ type: 'n8n-nodes-base.jira', version: 1, name: 'Jira' }) as unknown as INode, + ); + + const event: RelayEventMap['workflow-post-execute'] = { + workflow: mockWorkflowBase, + executionId: 'execution123', + userId: 'user123', + runData, + }; + + eventService.emit('workflow-post-execute', event); + + await flushPromises(); + + expect(credentialsRepository.findOneBy).toHaveBeenCalledWith({ + id: 'nhu-l8E4hX', + }); + + expect(telemetry.track).toHaveBeenCalledWith( + 'Manual node exec finished', + expect.objectContaining({ + webhook_domain: null, + user_id: 'user123', + workflow_id: 'workflow123', + status: 'error', + executionStatus: 'error', + sharing_role: 'sharee', + error_message: 'Error message', + error_node_type: 'n8n-nodes-base.jira', + error_node_id: '1', + node_id: '1', + node_type: 'n8n-nodes-base.jira', + + is_managed: true, + credential_type: 'openAiApi', + node_graph_string: JSON.stringify(nodeGraph.nodeGraph), + }), + ); + + expect(telemetry.trackWorkflowExecution).toHaveBeenCalledWith( + expect.objectContaining({ + workflow_id: 'workflow123', + success: false, + is_manual: true, + execution_mode: 'manual', + version_cli: N8N_VERSION, + error_message: 'Error message', + error_node_type: 'n8n-nodes-base.jira', + node_graph_string: JSON.stringify(nodeGraph.nodeGraph), + error_node_id: '1', + }), + ); + }); + + it('should call telemetry.track when user ran out of free AI credits', async () => { + sharedWorkflowRepository.findSharingRole.mockResolvedValue('workflow:editor'); + credentialsRepository.findOneBy.mockResolvedValue( + mock({ type: 'openAiApi', isManaged: true }), + ); + + const runData = { + status: 'error', + mode: 'trigger', + data: { + startData: { + destinationNode: 'OpenAI', + runNodeFilter: ['OpenAI'], + }, + executionData: { + nodeExecutionStack: [{ node: { credentials: { openAiApi: { id: 'nhu-l8E4hX' } } } }], + }, + resultData: { + runData: {}, + lastNodeExecuted: 'OpenAI', + error: new NodeApiError( + { + id: '1', + typeVersion: 1, + name: 'OpenAI', + type: 'n8n-nodes-base.openAi', + parameters: {}, + position: [100, 200], + }, + { + message: `400 - ${JSON.stringify({ + error: { + message: 'error message', + type: 'error_type', + code: 200, + }, + })}`, + error: { + message: 'error message', + type: 'error_type', + code: 200, + }, + }, + { + httpCode: '400', + }, + ), + }, + }, + } as unknown as IRun; + + jest + .spyOn(TelemetryHelpers, 'userInInstanceRanOutOfFreeAiCredits') + .mockImplementation(() => true); + + const event: RelayEventMap['workflow-post-execute'] = { + workflow: mockWorkflowBase, + executionId: 'execution123', + userId: 'user123', + runData, + }; + + eventService.emit('workflow-post-execute', event); + + await flushPromises(); + + expect(telemetry.track).toHaveBeenCalledWith('User ran out of free AI credits'); + }); }); }); diff --git a/packages/cli/src/events/relays/telemetry.event-relay.ts b/packages/cli/src/events/relays/telemetry.event-relay.ts index 221449bbab..67fbacb107 100644 --- a/packages/cli/src/events/relays/telemetry.event-relay.ts +++ b/packages/cli/src/events/relays/telemetry.event-relay.ts @@ -9,6 +9,7 @@ import { get as pslGet } from 'psl'; import config from '@/config'; import { N8N_VERSION } from '@/constants'; +import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; import { ProjectRelationRepository } from '@/databases/repositories/project-relation.repository'; import { SharedWorkflowRepository } from '@/databases/repositories/shared-workflow.repository'; import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; @@ -34,6 +35,7 @@ export class TelemetryEventRelay extends EventRelay { private readonly nodeTypes: NodeTypes, private readonly sharedWorkflowRepository: SharedWorkflowRepository, private readonly projectRelationRepository: ProjectRelationRepository, + private readonly credentialsRepository: CredentialsRepository, ) { super(eventService); } @@ -632,6 +634,10 @@ export class TelemetryEventRelay extends EventRelay { let nodeGraphResult: INodesGraphResult | null = null; if (!telemetryProperties.success && runData?.data.resultData.error) { + if (TelemetryHelpers.userInInstanceRanOutOfFreeAiCredits(runData)) { + this.telemetry.track('User ran out of free AI credits'); + } + telemetryProperties.error_message = runData?.data.resultData.error.message; let errorNodeName = 'node' in runData?.data.resultData.error @@ -693,6 +699,8 @@ export class TelemetryEventRelay extends EventRelay { error_node_id: telemetryProperties.error_node_id as string, webhook_domain: null, sharing_role: userRole, + credential_type: null, + is_managed: false, }; if (!manualExecEventProperties.node_graph_string) { @@ -703,7 +711,18 @@ export class TelemetryEventRelay extends EventRelay { } if (runData.data.startData?.destinationNode) { - const telemetryPayload = { + const credentialsData = TelemetryHelpers.extractLastExecutedNodeCredentialData(runData); + if (credentialsData) { + manualExecEventProperties.credential_type = credentialsData.credentialType; + const credential = await this.credentialsRepository.findOneBy({ + id: credentialsData.credentialId, + }); + if (credential) { + manualExecEventProperties.is_managed = credential.isManaged; + } + } + + const telemetryPayload: ITelemetryTrackProperties = { ...manualExecEventProperties, node_type: TelemetryHelpers.getNodeTypeForName( workflow, diff --git a/packages/cli/src/task-runners/task-runner-server.ts b/packages/cli/src/task-runners/task-runner-server.ts index 6e68c4fb32..80679f8c41 100644 --- a/packages/cli/src/task-runners/task-runner-server.ts +++ b/packages/cli/src/task-runners/task-runner-server.ts @@ -2,6 +2,7 @@ import { GlobalConfig } from '@n8n/config'; import { Service } from '@n8n/di'; import compression from 'compression'; import express from 'express'; +import { rateLimit as expressRateLimit } from 'express-rate-limit'; import { Logger } from 'n8n-core'; import * as a from 'node:assert/strict'; import { randomBytes } from 'node:crypto'; @@ -147,8 +148,16 @@ export class TaskRunnerServer { } private configureRoutes() { + const createRateLimiter = () => + expressRateLimit({ + windowMs: 1000, + limit: 5, + message: { message: 'Too many requests' }, + }); + this.app.use( this.upgradeEndpoint, + createRateLimiter(), // eslint-disable-next-line @typescript-eslint/unbound-method this.taskRunnerAuthController.authMiddleware, (req: TaskRunnerServerInitRequest, res: TaskRunnerServerInitResponse) => @@ -158,6 +167,7 @@ export class TaskRunnerServer { const authEndpoint = `${this.getEndpointBasePath()}/auth`; this.app.post( authEndpoint, + createRateLimiter(), send(async (req) => await this.taskRunnerAuthController.createGrantToken(req)), ); diff --git a/packages/cli/src/workflows/workflow-execution.service.ts b/packages/cli/src/workflows/workflow-execution.service.ts index 3b384f4c53..bccd5c1f13 100644 --- a/packages/cli/src/workflows/workflow-execution.service.ts +++ b/packages/cli/src/workflows/workflow-execution.service.ts @@ -147,6 +147,12 @@ export class WorkflowExecutionService { triggerToStartFrom, }; + const hasRunData = (node: INode) => runData !== undefined && !!runData[node.name]; + + if (pinnedTrigger && !hasRunData(pinnedTrigger)) { + data.startNodes = [{ name: pinnedTrigger.name, sourceData: null }]; + } + /** * Historically, manual executions in scaling mode ran in the main process, * so some execution details were never persisted in the database. @@ -160,7 +166,7 @@ export class WorkflowExecutionService { ) { data.executionData = { startData: { - startNodes, + startNodes: data.startNodes, destinationNode, }, resultData: { @@ -176,12 +182,6 @@ export class WorkflowExecutionService { }; } - const hasRunData = (node: INode) => runData !== undefined && !!runData[node.name]; - - if (pinnedTrigger && !hasRunData(pinnedTrigger)) { - data.startNodes = [{ name: pinnedTrigger.name, sourceData: null }]; - } - const executionId = await this.workflowRunner.run(data); return { diff --git a/packages/cli/test/integration/ai/ai.api.test.ts b/packages/cli/test/integration/ai/ai.api.test.ts index 721f2296ed..6741930b2e 100644 --- a/packages/cli/test/integration/ai/ai.api.test.ts +++ b/packages/cli/test/integration/ai/ai.api.test.ts @@ -1,8 +1,9 @@ import { Container } from '@n8n/di'; import { randomUUID } from 'crypto'; import { mock } from 'jest-mock-extended'; +import { OPEN_AI_API_CREDENTIAL_TYPE } from 'n8n-workflow'; -import { FREE_AI_CREDITS_CREDENTIAL_NAME, OPEN_AI_API_CREDENTIAL_TYPE } from '@/constants'; +import { FREE_AI_CREDITS_CREDENTIAL_NAME } from '@/constants'; import type { Project } from '@/databases/entities/project'; import type { User } from '@/databases/entities/user'; import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; diff --git a/packages/cli/test/integration/evaluation/test-runs.api.test.ts b/packages/cli/test/integration/evaluation/test-runs.api.test.ts index 8f01029ad2..3fcc321cc9 100644 --- a/packages/cli/test/integration/evaluation/test-runs.api.test.ts +++ b/packages/cli/test/integration/evaluation/test-runs.api.test.ts @@ -93,7 +93,7 @@ describe('GET /evaluation/test-definitions/:testDefinitionId/runs', () => { const testRunRepository = Container.get(TestRunRepository); const testRun1 = await testRunRepository.createTestRun(testDefinition.id); // Mark as running just to make a slight delay between the runs - await testRunRepository.markAsRunning(testRun1.id); + await testRunRepository.markAsRunning(testRun1.id, 10); const testRun2 = await testRunRepository.createTestRun(testDefinition.id); // Fetch the first page diff --git a/packages/cli/test/integration/runners/task-runner-server.test.ts b/packages/cli/test/integration/runners/task-runner-server.test.ts index 11d77b53fc..6088af3525 100644 --- a/packages/cli/test/integration/runners/task-runner-server.test.ts +++ b/packages/cli/test/integration/runners/task-runner-server.test.ts @@ -19,4 +19,26 @@ describe('TaskRunnerServer', () => { await agent.get('/healthz').expect(200); }); }); + + describe('/runners/_ws', () => { + it('should return 429 when too many requests are made', async () => { + await agent.post('/runners/_ws').send({}).expect(401); + await agent.post('/runners/_ws').send({}).expect(401); + await agent.post('/runners/_ws').send({}).expect(401); + await agent.post('/runners/_ws').send({}).expect(401); + await agent.post('/runners/_ws').send({}).expect(401); + await agent.post('/runners/_ws').send({}).expect(429); + }); + }); + + describe('/runners/auth', () => { + it('should return 429 when too many requests are made', async () => { + await agent.post('/runners/auth').send({ token: 'invalid' }).expect(403); + await agent.post('/runners/auth').send({ token: 'invalid' }).expect(403); + await agent.post('/runners/auth').send({ token: 'invalid' }).expect(403); + await agent.post('/runners/auth').send({ token: 'invalid' }).expect(403); + await agent.post('/runners/auth').send({ token: 'invalid' }).expect(403); + await agent.post('/runners/auth').send({ token: 'invalid' }).expect(429); + }); + }); }); diff --git a/packages/core/src/node-execution-context/utils/validateValueAgainstSchema.ts b/packages/core/src/node-execution-context/utils/validateValueAgainstSchema.ts index adac8c3a78..d058c50a52 100644 --- a/packages/core/src/node-execution-context/utils/validateValueAgainstSchema.ts +++ b/packages/core/src/node-execution-context/utils/validateValueAgainstSchema.ts @@ -61,11 +61,7 @@ const validateResourceMapperValue = ( }); if (!validationResult.valid) { - if (!resourceMapperField.ignoreTypeMismatchErrors) { - return { ...validationResult, fieldName: key }; - } else { - paramValues[key] = resolvedValue; - } + return { ...validationResult, fieldName: key }; } else { // If it's valid, set the casted value paramValues[key] = validationResult.newValue; diff --git a/packages/design-system/package.json b/packages/design-system/package.json index 2ecd0513ac..68e47f908a 100644 --- a/packages/design-system/package.json +++ b/packages/design-system/package.json @@ -11,7 +11,7 @@ "test": "vitest run", "test:dev": "vitest", "build:storybook": "storybook build", - "storybook": "storybook dev -p 6006", + "storybook": "storybook dev -p 6006 --no-open", "chromatic": "chromatic", "format": "biome format --write . && prettier --write . --ignore-path ../../.prettierignore", "format:check": "biome ci . && prettier --check . --ignore-path ../../.prettierignore", diff --git a/packages/design-system/src/components/N8nSelect/Select.vue b/packages/design-system/src/components/N8nSelect/Select.vue index 7cb4fafcc9..80a065d429 100644 --- a/packages/design-system/src/components/N8nSelect/Select.vue +++ b/packages/design-system/src/components/N8nSelect/Select.vue @@ -31,6 +31,10 @@ const props = defineProps({ multiple: { type: Boolean, }, + multipleLimit: { + type: Number, + default: 0, + }, filterMethod: { type: Function, }, @@ -120,6 +124,7 @@ defineExpose({ ; } + interface CreateTestDefinitionParams { name: string; workflowId: string; @@ -21,31 +25,63 @@ export interface UpdateTestDefinitionParams { evaluationWorkflowId?: string | null; annotationTagId?: string | null; description?: string | null; + mockedNodes?: Array<{ name: string }>; } + export interface UpdateTestResponse { createdAt: string; updatedAt: string; id: string; name: string; workflowId: string; - description: string | null; - annotationTag: string | null; - evaluationWorkflowId: string | null; - annotationTagId: string | null; + description?: string | null; + annotationTag?: string | null; + evaluationWorkflowId?: string | null; + annotationTagId?: string | null; +} + +export interface TestRunRecord { + id: string; + testDefinitionId: string; + status: 'new' | 'running' | 'completed' | 'error'; + metrics?: Record; + createdAt: string; + updatedAt: string; + runAt: string; + completedAt: string; +} + +interface GetTestRunParams { + testDefinitionId: string; + runId: string; +} + +interface DeleteTestRunParams { + testDefinitionId: string; + runId: string; } const endpoint = '/evaluation/test-definitions'; +const getMetricsEndpoint = (testDefinitionId: string, metricId?: string) => + `${endpoint}/${testDefinitionId}/metrics${metricId ? `/${metricId}` : ''}`; -export async function getTestDefinitions(context: IRestApiContext) { +export async function getTestDefinitions( + context: IRestApiContext, + params?: { workflowId?: string }, +) { + let url = endpoint; + if (params?.workflowId) { + url += `?filter=${JSON.stringify({ workflowId: params.workflowId })}`; + } return await makeRestApiRequest<{ count: number; testDefinitions: TestDefinitionRecord[] }>( context, 'GET', - endpoint, + url, ); } export async function getTestDefinition(context: IRestApiContext, id: string) { - return await makeRestApiRequest<{ id: string }>(context, 'GET', `${endpoint}/${id}`); + return await makeRestApiRequest(context, 'GET', `${endpoint}/${id}`); } export async function createTestDefinition( @@ -71,3 +107,125 @@ export async function updateTestDefinition( export async function deleteTestDefinition(context: IRestApiContext, id: string) { return await makeRestApiRequest<{ success: boolean }>(context, 'DELETE', `${endpoint}/${id}`); } + +// Metrics +export interface TestMetricRecord { + id: string; + name: string; + testDefinitionId: string; + createdAt?: string; + updatedAt?: string; +} + +export interface CreateTestMetricParams { + testDefinitionId: string; + name: string; +} + +export interface UpdateTestMetricParams { + name: string; + id: string; + testDefinitionId: string; +} + +export interface DeleteTestMetricParams { + testDefinitionId: string; + id: string; +} + +export const getTestMetrics = async (context: IRestApiContext, testDefinitionId: string) => { + return await makeRestApiRequest( + context, + 'GET', + getMetricsEndpoint(testDefinitionId), + ); +}; + +export const getTestMetric = async ( + context: IRestApiContext, + testDefinitionId: string, + id: string, +) => { + return await makeRestApiRequest( + context, + 'GET', + getMetricsEndpoint(testDefinitionId, id), + ); +}; + +export const createTestMetric = async ( + context: IRestApiContext, + params: CreateTestMetricParams, +) => { + return await makeRestApiRequest( + context, + 'POST', + getMetricsEndpoint(params.testDefinitionId), + { name: params.name }, + ); +}; + +export const updateTestMetric = async ( + context: IRestApiContext, + params: UpdateTestMetricParams, +) => { + return await makeRestApiRequest( + context, + 'PATCH', + getMetricsEndpoint(params.testDefinitionId, params.id), + { name: params.name }, + ); +}; + +export const deleteTestMetric = async ( + context: IRestApiContext, + params: DeleteTestMetricParams, +) => { + return await makeRestApiRequest( + context, + 'DELETE', + getMetricsEndpoint(params.testDefinitionId, params.id), + ); +}; + +const getRunsEndpoint = (testDefinitionId: string, runId?: string) => + `${endpoint}/${testDefinitionId}/runs${runId ? `/${runId}` : ''}`; + +// Get all test runs for a test definition +export const getTestRuns = async (context: IRestApiContext, testDefinitionId: string) => { + return await makeRestApiRequest( + context, + 'GET', + getRunsEndpoint(testDefinitionId), + ); +}; + +// Get specific test run +export const getTestRun = async (context: IRestApiContext, params: GetTestRunParams) => { + return await makeRestApiRequest( + context, + 'GET', + getRunsEndpoint(params.testDefinitionId, params.runId), + ); +}; + +// Start a new test run +export const startTestRun = async (context: IRestApiContext, testDefinitionId: string) => { + const response = await request({ + method: 'POST', + baseURL: context.baseUrl, + endpoint: `${endpoint}/${testDefinitionId}/run`, + headers: { 'push-ref': context.pushRef }, + }); + // CLI is returning the response without wrapping it in `data` key + return response as { success: boolean }; +}; + +// Delete a test run +export const deleteTestRun = async (context: IRestApiContext, params: DeleteTestRunParams) => { + return await makeRestApiRequest<{ success: boolean }>( + context, + 'DELETE', + getRunsEndpoint(params.testDefinitionId, params.runId), + ); +}; diff --git a/packages/editor-ui/src/components/CodeNodeEditor/CodeNodeEditor.vue b/packages/editor-ui/src/components/CodeNodeEditor/CodeNodeEditor.vue index 3f74bfe7a0..c17ff804c0 100644 --- a/packages/editor-ui/src/components/CodeNodeEditor/CodeNodeEditor.vue +++ b/packages/editor-ui/src/components/CodeNodeEditor/CodeNodeEditor.vue @@ -1,32 +1,24 @@